├── .cramrc ├── .flake8 ├── .github ├── actions │ ├── run-integration-tests │ │ └── action.yaml │ └── setup-integration-tests │ │ └── action.yaml ├── dependabot.yml └── workflows │ ├── ci.yaml │ ├── standalone-installers.yaml │ └── sync-rtd-redirects.yaml ├── .gitignore ├── .readthedocs.yml ├── CHANGES.md ├── LICENSE ├── LICENSE.cognito-srp ├── LICENSE.id3c ├── LICENSE.sphinx ├── MANIFEST.in ├── README.md ├── bin ├── nextstrain ├── standalone-installer-unix └── standalone-installer-windows ├── devel ├── changes ├── create-github-release ├── generate-changes-doc ├── generate-command-doc ├── generate-doc ├── pyoxidizer ├── pytest ├── read-version ├── reflow-markdown ├── release ├── rtd-pre-build ├── setup-venv ├── update-version ├── venv-run └── within-container ├── doc ├── .gitignore ├── Makefile ├── _static │ └── big-picture.svg ├── aws-batch.md ├── changes.md ├── commands │ ├── authorization.rst │ ├── build.rst │ ├── check-setup.rst │ ├── debugger.rst │ ├── deploy.rst │ ├── index.rst │ ├── init-shell.rst │ ├── login.rst │ ├── logout.rst │ ├── remote │ │ ├── delete.rst │ │ ├── download.rst │ │ ├── index.rst │ │ ├── list.rst │ │ └── upload.rst │ ├── run.rst │ ├── setup.rst │ ├── shell.rst │ ├── update.rst │ ├── version.rst │ ├── view.rst │ └── whoami.rst ├── conf.py ├── config │ ├── file.rst │ └── paths.rst ├── development.md ├── glossary.rst ├── index.rst ├── installation.rst ├── make.bat ├── redirects.yaml ├── remotes │ ├── nextstrain.org.rst │ └── s3.rst ├── runtimes │ ├── ambient.rst │ ├── aws-batch.rst │ ├── comparison.csv │ ├── conda.rst │ ├── docker.rst │ ├── index.rst │ └── singularity.rst └── upgrading.rst ├── nextstrain └── cli │ ├── __init__.py │ ├── __main__.py │ ├── __version__.py │ ├── argparse.py │ ├── authn │ ├── __init__.py │ ├── configuration.py │ ├── errors.py │ └── session.py │ ├── aws │ ├── __init__.py │ └── cognito │ │ ├── __init__.py │ │ └── srp.py │ ├── browser.py │ ├── command │ ├── __init__.py │ ├── authorization.py │ ├── build.py │ ├── check_setup.py │ ├── debugger.py │ ├── deploy.py │ ├── init_shell.py │ ├── login.py │ ├── logout.py │ ├── remote │ │ ├── __init__.py │ │ ├── delete.py │ │ ├── download.py │ │ ├── ls.py │ │ └── upload.py │ ├── run.py │ ├── setup.py │ ├── shell.py │ ├── update.py │ ├── version.py │ ├── view.py │ └── whoami.py │ ├── config.py │ ├── console.py │ ├── debug.py │ ├── env.py │ ├── errors.py │ ├── gzip.py │ ├── hostenv.py │ ├── markdown.py │ ├── net.py │ ├── pathogens.py │ ├── paths.py │ ├── remote │ ├── __init__.py │ ├── nextstrain_dot_org.py │ └── s3.py │ ├── requests.py │ ├── resources │ ├── __init__.py │ └── bashrc │ ├── rst │ ├── __init__.py │ └── sphinx.py │ ├── runner │ ├── __init__.py │ ├── ambient.py │ ├── aws_batch │ │ ├── __init__.py │ │ ├── jobs.py │ │ ├── logs.py │ │ └── s3.py │ ├── conda.py │ ├── docker.py │ └── singularity.py │ ├── types.py │ ├── url.py │ ├── util.py │ └── volume.py ├── pyoxidizer.bzl ├── pyproject.toml ├── pyrightconfig.json ├── pytest.ini ├── setup.py └── tests ├── command-build.py ├── cram.py ├── data ├── home │ ├── .gitignore │ ├── config │ └── pathogens │ │ ├── with-implicit-default │ │ └── 1.2.3=GEXDELRT │ │ │ └── .gitignore │ │ └── with-no-implicit-default │ │ ├── 1.2.3=GEXDELRT │ │ └── .gitignore │ │ └── 4.5.6=GQXDKLRW │ │ └── .gitignore ├── markdown │ ├── embed-images-001.md │ ├── embed-images-002.md │ ├── embed-images-003.md │ ├── image.png │ └── roundtrip-001.md └── pathogen-repo │ ├── ingest │ └── .gitignore │ ├── nextstrain-pathogen.yaml │ └── phylogenetic │ └── .gitignore ├── doc.py ├── env ├── flake8.py ├── help.py ├── markdown.py ├── open_browser.py ├── pyright.py ├── remote.py └── version.cram /.cramrc: -------------------------------------------------------------------------------- 1 | [cram] 2 | shell = bash 3 | indent = 4 4 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | # Right now we use Flake8 only for a few static checks focusing on runtime 2 | # safety and correctness. We don't use it for style checks. 3 | 4 | [flake8] 5 | select = 6 | # syntax errors 7 | E9, 8 | 9 | # all pyflakes correctness issues 10 | F, 11 | 12 | extend-ignore = 13 | # allow f-strings without any placeholders 14 | F541, 15 | 16 | exclude = 17 | .git, 18 | .venv*, 19 | __pycache__, 20 | build, 21 | dist, 22 | -------------------------------------------------------------------------------- /.github/actions/run-integration-tests/action.yaml: -------------------------------------------------------------------------------- 1 | name: Run integration tests 2 | description: >- 3 | Runs integration tests excercising common commands and behaviour. 4 | 5 | Jobs which use this action must have 6 | 7 | 1. Called the setup-integration-tests action. 8 | 2. Installed Nextstrain CLI on the PATH. 9 | 10 | before calling this action. 11 | 12 | runs: 13 | using: composite 14 | steps: 15 | - shell: bash -l -eo pipefail {0} 16 | run: nextstrain version --verbose 17 | 18 | - if: runner.os != 'macOS' && runner.os != 'Windows' 19 | shell: bash -l -eo pipefail {0} 20 | run: nextstrain setup docker 21 | 22 | - if: runner.os != 'Windows' 23 | shell: bash -l -eo pipefail {0} 24 | run: nextstrain setup conda 25 | 26 | - if: runner.os != 'macOS' && runner.os != 'Windows' 27 | shell: bash -l -eo pipefail {0} 28 | run: nextstrain setup singularity 29 | 30 | - shell: bash -l -eo pipefail {0} 31 | run: nextstrain check-setup --set-default 32 | 33 | - shell: bash -l -eo pipefail {0} 34 | run: nextstrain version --verbose 35 | 36 | - if: runner.os != 'macOS' && runner.os != 'Windows' 37 | name: Build zika-tutorial with --docker 38 | shell: bash -l -eo pipefail {0} 39 | run: | 40 | git -C zika-tutorial clean -dfqx 41 | nextstrain build --docker --cpus 2 zika-tutorial 42 | 43 | - if: runner.os != 'Windows' 44 | name: Build zika-tutorial with --conda 45 | shell: bash -l -eo pipefail {0} 46 | run: | 47 | git -C zika-tutorial clean -dfqx 48 | nextstrain build --conda --cpus 2 zika-tutorial 49 | 50 | - if: runner.os != 'macOS' && runner.os != 'Windows' 51 | name: Build zika-tutorial with --singularity 52 | shell: bash -l -eo pipefail {0} 53 | run: | 54 | git -C zika-tutorial clean -dfqx 55 | nextstrain build --singularity --cpus 2 zika-tutorial 56 | 57 | - if: runner.os != 'Windows' 58 | name: Build zika-tutorial with --ambient 59 | shell: bash -l -eo pipefail {0} 60 | run: | 61 | git -C zika-tutorial clean -dfqx 62 | nextstrain build --ambient --cpus 2 zika-tutorial 63 | -------------------------------------------------------------------------------- /.github/actions/setup-integration-tests/action.yaml: -------------------------------------------------------------------------------- 1 | name: Setup integration tests 2 | description: >- 3 | Sets up prerequisites for run-integration-tests, namely installing software 4 | needed by the "ambient" runner and cloning zika-tutorial for use as an test 5 | build. 6 | 7 | Jobs which use this action must also set 8 | 9 | defaults: 10 | run: 11 | shell: bash -l -eo pipefail {0} 12 | 13 | at the job level (or the equivalent "shell:" key at the step level) to 14 | activate the integration Conda environment by default. 15 | 16 | inputs: 17 | python-version: 18 | description: Version of Python to use for conda-incubator/setup-miniconda. 19 | type: string 20 | required: true 21 | 22 | runs: 23 | using: composite 24 | steps: 25 | - uses: conda-incubator/setup-miniconda@v3 26 | with: 27 | python-version: ${{ inputs.python-version }} 28 | miniforge-version: latest 29 | channels: conda-forge,bioconda 30 | 31 | - run: cat ~/.profile || true 32 | shell: bash -l -eo pipefail {0} 33 | 34 | - run: cat ~/.bash_profile || true 35 | shell: bash -l -eo pipefail {0} 36 | 37 | - run: cat ~/.bashrc || true 38 | shell: bash -l -eo pipefail {0} 39 | 40 | # Install software for the "ambient" runner; not supported on Windows. 41 | - if: runner.os != 'Windows' 42 | run: mamba install augur auspice 'snakemake !=7.30.2' 43 | shell: bash -l -eo pipefail {0} 44 | 45 | - run: conda info 46 | shell: bash -l -eo pipefail {0} 47 | 48 | - run: conda list 49 | shell: bash -l -eo pipefail {0} 50 | 51 | - if: runner.os == 'Windows' 52 | name: Fix python vs. python3 mismatch on Windows 53 | shell: bash -l -eo pipefail {0} 54 | run: | 55 | python="$(type -p python)" 56 | cp -v "$python" "$(dirname "$python")"/python3 57 | 58 | - name: Check python version 59 | shell: bash -l -eo pipefail {0} 60 | run: | 61 | # Assert that we're on the expected Python version, in case the GH 62 | # Actions environment is messed up. 63 | type python 64 | python --version 65 | type python3 66 | python3 --version 67 | python --version | grep -F 'Python ${{ inputs.python-version }}.' 68 | python3 --version | grep -F 'Python ${{ inputs.python-version }}.' 69 | [[ "$(python --version)" == "$(python3 --version)" ]] 70 | 71 | # Install Singularity on Linux. 72 | # 73 | # We don't install it with Conda because Conda Forge provides a non-suid 74 | # build of Singularity. We're compatible with Singularity's non-suid mode, 75 | # but production usages of Singularity are likely to use its suid mode, so 76 | # I'd rather test against that. 77 | # -trs, 6 Jan 2023 78 | - if: runner.os == 'Linux' 79 | shell: bash -l -eo pipefail {0} 80 | env: 81 | GITHUB_TOKEN: ${{ github.token }} 82 | run: | 83 | # Work in a temp dir to avoid cluttering the caller's working dir 84 | pushd "$(mktemp -d)" 85 | export "$(grep UBUNTU_CODENAME /etc/os-release)" 86 | 87 | # Download latest SingularityCE 3.x series .deb for this version of Ubuntu 88 | # 89 | # XXX TODO: Start testing the SingularityCE 4.x series. 90 | # -trs, 19 Sept 2023 91 | url="$( 92 | curl -fsSL --proto '=https' -H "Authorization: Bearer $GITHUB_TOKEN" \ 93 | "https://api.github.com/repos/sylabs/singularity/releases?per_page=100" \ 94 | | jq -r ' 95 | map(select(.tag_name | startswith("v3."))) 96 | | .[0].assets 97 | | (map(select(.name | endswith("\(env.UBUNTU_CODENAME)_amd64.deb"))) | .[0].browser_download_url) 98 | // (map(select(.name | endswith("jammy_amd64.deb"))) | .[0].browser_download_url)')" 99 | 100 | curl -fsSL --proto '=https' "$url" > singularity.deb 101 | 102 | # Install and check that it runs 103 | sudo dpkg -i singularity.deb 104 | singularity --version 105 | 106 | # Clone the small build we'll use as an integration test case. 107 | - run: git clone https://github.com/nextstrain/zika-tutorial 108 | shell: bash -l -eo pipefail {0} 109 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Dependabot configuration file 2 | # 3 | # 4 | # Each ecosystem is checked on a scheduled interval defined below. To trigger 5 | # a check manually, go to 6 | # 7 | # https://github.com/nextstrain/cli/network/updates 8 | # 9 | # and look for a "Check for updates" button. You may need to click around a 10 | # bit first. 11 | --- 12 | version: 2 13 | updates: 14 | - package-ecosystem: "github-actions" 15 | directory: "/" 16 | schedule: 17 | interval: "weekly" 18 | -------------------------------------------------------------------------------- /.github/workflows/standalone-installers.yaml: -------------------------------------------------------------------------------- 1 | name: Standalone installers 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | paths: 8 | - bin/standalone-installer-unix 9 | - bin/standalone-installer-windows 10 | 11 | pull_request: 12 | paths: 13 | - bin/standalone-installer-unix 14 | - bin/standalone-installer-windows 15 | 16 | # Routinely check that the external resources the installers rely on keep 17 | # functioning as expected. 18 | schedule: 19 | # Every day at 17:42 UTC / 9:42 Seattle (winter) / 10:42 Seattle (summer) 20 | - cron: "42 17 * * *" 21 | 22 | workflow_dispatch: 23 | 24 | jobs: 25 | # The goal here is to make sure the installers run successfully on a variety 26 | # of OS versions. We're _not_ testing unreleased standalone builds here—the 27 | # installation archives are downloaded from the latest release on GitHub via 28 | # nextstrain.org endpoints—which is why this isn't part of CI. That is, this 29 | # is akin to testing `pip install nextstrain-cli` if we wanted to make sure 30 | # `pip` worked. 31 | # -trs, 29 August 2022 32 | test: 33 | name: test (os=${{ matrix.os }}) 34 | strategy: 35 | fail-fast: false 36 | matrix: 37 | os: 38 | - ubuntu-22.04 39 | - ubuntu-24.04 40 | - macos-13 41 | - macos-14 # (aarch64) 42 | - macos-15 # (aarch64) 43 | - windows-2022 44 | - windows-2025 45 | 46 | runs-on: ${{matrix.os}} 47 | steps: 48 | - uses: actions/checkout@v4 49 | 50 | # Use pipes like are used in the real instructions when the installer is 51 | # fetched from nextstrain.org. This tests that the shell programs work 52 | # when they're specifically executed this way. 53 | - if: runner.os != 'Windows' 54 | run: | 55 | # shellcheck disable=SC2002 56 | cat ./bin/standalone-installer-unix | bash 57 | shell: bash 58 | 59 | - if: runner.os == 'Windows' 60 | run: Get-Content -Raw ./bin/standalone-installer-windows | Invoke-Expression 61 | shell: pwsh 62 | -------------------------------------------------------------------------------- /.github/workflows/sync-rtd-redirects.yaml: -------------------------------------------------------------------------------- 1 | name: Sync RTD redirects 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | paths: 8 | - doc/redirects.yaml 9 | - .github/workflows/sync-rtd-redirects.yaml 10 | 11 | # The reusable workflow will only actually make changes when running on the 12 | # default branch (e.g. refs/heads/main); every other run will be a dry run. 13 | pull_request: 14 | 15 | # Manually triggered using GitHub's UI 16 | workflow_dispatch: 17 | 18 | jobs: 19 | sync: 20 | name: rtd redirects 21 | uses: nextstrain/.github/.github/workflows/sync-rtd-redirects.yaml@master 22 | with: 23 | project: nextstrain-cli 24 | file: doc/redirects.yaml 25 | secrets: 26 | RTD_TOKEN: ${{ secrets.RTD_TOKEN }} 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | __pycache__/ 3 | environment* 4 | 5 | # Automatically-managed PyOxidizer binaries (e.g. pyoxidizer-0.22.0-py3-none-manylinux2010_x86_64). 6 | /devel/pyoxidizer-[0-9]* 7 | 8 | # Package build dirs 9 | /build/ 10 | /dist/ 11 | /nextstrain_cli.egg-info/ 12 | 13 | # OS generated files # 14 | ###################### 15 | .DS_Store 16 | .DS_Store? 17 | ._* 18 | .Spotlight-V100 19 | .Trashes 20 | Icon? 21 | ehthumbs.db 22 | Thumbs.db 23 | 24 | # cram test run outputs 25 | /tests/*.cram.err 26 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | build: 4 | os: ubuntu-20.04 5 | tools: 6 | python: "3.10" 7 | jobs: 8 | pre_build: 9 | - ./devel/rtd-pre-build 10 | sphinx: 11 | configuration: doc/conf.py 12 | builder: dirhtml 13 | python: 14 | install: 15 | - method: pip 16 | path: . 17 | extra_requirements: 18 | - dev 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018–2021 Trevor Bedford and Richard Neher 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LICENSE.id3c: -------------------------------------------------------------------------------- 1 | This license applies to the original copy of "prose_list" from the ID3C project 2 | into this project, incorporated as part of "nextstrain/cli/util.py". Any 3 | subsequent modifications to this project's copy of "prose_list" are licensed 4 | under the MIT license of this project, not of ID3C. 5 | 6 | MIT License 7 | 8 | Copyright (c) 2018 Brotman Baty Institute 9 | 10 | Permission is hereby granted, free of charge, to any person obtaining a copy 11 | of this software and associated documentation files (the "Software"), to deal 12 | in the Software without restriction, including without limitation the rights 13 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | copies of the Software, and to permit persons to whom the Software is 15 | furnished to do so, subject to the following conditions: 16 | 17 | The above copyright notice and this permission notice shall be included in all 18 | copies or substantial portions of the Software. 19 | 20 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 | SOFTWARE. 27 | -------------------------------------------------------------------------------- /LICENSE.sphinx: -------------------------------------------------------------------------------- 1 | This license applies to the original copy of code from the Sphinx project 2 | (version 4.3.2) into this project as "nextstrain/cli/rst/sphinx.py". 3 | Subsequent modifications to this project's copy are licensed under the MIT 4 | license of this project. 5 | 6 | Copyright (c) 2007-2022 by the Sphinx team (see AUTHORS file). 7 | All rights reserved. 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions are 11 | met: 12 | 13 | * Redistributions of source code must retain the above copyright 14 | notice, this list of conditions and the following disclaimer. 15 | 16 | * Redistributions in binary form must reproduce the above copyright 17 | notice, this list of conditions and the following disclaimer in the 18 | documentation and/or other materials provided with the distribution. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include LICENSE.cognito-srp 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nextstrain CLI 2 | 3 | This is the source code repository for a program called `nextstrain`, the 4 | Nextstrain command-line interface (CLI). It aims to provide a consistent way 5 | to run and visualize pathogen builds and access Nextstrain components like 6 | [Augur][] and [Auspice][] across computing platforms such as [Docker][], 7 | [Conda][], [Singularity][], and [AWS Batch][]. 8 | 9 | Get started using the Nextstrain CLI by reading the [documentation][], which 10 | includes installation and usage information. 11 | 12 | If you'd like to contribute to development, the [development docs][] should 13 | help you get going. We're glad you'd like to contribute! 14 | 15 | If you need help (or just want to say hi!), [open an issue][] or send us an 16 | email to . 17 | 18 | 19 | [Augur]: https://docs.nextstrain.org/projects/augur/ 20 | [Auspice]: https://docs.nextstrain.org/projects/auspice/ 21 | [Docker]: https://docs.nextstrain.org/projects/cli/page/runtimes/docker/ 22 | [Conda]: https://docs.nextstrain.org/projects/cli/page/runtimes/conda/ 23 | [Singularity]: https://docs.nextstrain.org/projects/cli/page/runtimes/singularity/ 24 | [AWS Batch]: https://docs.nextstrain.org/projects/cli/page/runtimes/aws-batch/ 25 | [documentation]: https://docs.nextstrain.org/projects/cli/ 26 | [development docs]: https://docs.nextstrain.org/projects/cli/page/development/ 27 | [open an issue]: https://github.com/nextstrain/cli/issues/new 28 | -------------------------------------------------------------------------------- /bin/nextstrain: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # This script is only used in development. Package installations will use a 4 | # similar "nextstrain" script automatically created using the entry point 5 | # feature of setuptools. 6 | # 7 | from sys import path, exit 8 | from pathlib import Path 9 | 10 | # Try to add our containing package source directory to the Python module 11 | # search path so that we load nextstrain.cli from there instead of any 12 | # installed system-wide. 13 | try: 14 | dev_path = Path(__file__).parent.parent 15 | 16 | # Raises an exception if the path doesn't exist. 17 | (dev_path / "nextstrain/cli/__init__.py").resolve(strict = True) 18 | except: 19 | pass 20 | else: 21 | path.insert(0, str(dev_path)) 22 | 23 | from nextstrain.cli.__main__ import main 24 | exit( main() ) 25 | -------------------------------------------------------------------------------- /bin/standalone-installer-windows: -------------------------------------------------------------------------------- 1 | # 2 | # PowerShell 5+ program to download the latest Nextstrain CLI standalone 3 | # installation archive for Windows, extract it into the current user's app 4 | # data directory, and ensure PATH includes the installation destination. 5 | # 6 | # It maintains rough parity with the Bash program for Linux and macOS, 7 | # standalone-installer-unix. 8 | # 9 | # Set $env:DESTINATION to change the installation location. 10 | # 11 | # Set $env:VERSION to change the version downloaded and installed, or pass the 12 | # desired version as the first argument to this program. 13 | # 14 | Set-StrictMode -Version 3.0 15 | $ErrorActionPreference = "Stop" 16 | 17 | # Wrap everything in a function which we call at the end to avoid execution of 18 | # a partially-downloaded program. 19 | function main([string]$version) { 20 | $destination = & { 21 | if ($env:DESTINATION) { 22 | return $env:DESTINATION 23 | } 24 | return "${HOME}\.nextstrain\cli-standalone" 25 | } 26 | 27 | $nextstrain_dot_org = & { 28 | if ($env:NEXTSTRAIN_DOT_ORG) { return $env:NEXTSTRAIN_DOT_ORG } 29 | return "https://nextstrain.org" 30 | } 31 | 32 | if (!$version) { 33 | $version = & { 34 | if ($env:VERSION) { return $env:VERSION } 35 | return "latest" 36 | } 37 | } 38 | 39 | # XXX TODO: Check for x86_64 arch; i.e. don't pass on 32-bit systems or 40 | # non-Intel 64-bit (e.g. arm64). 41 | 42 | $archive = "standalone-x86_64-pc-windows-msvc.zip" 43 | $archive_url = "${nextstrain_dot_org}/cli/download/${version}/${archive}" 44 | 45 | # Move into a temporary working dir 46 | $tmp = New-Item -Type Directory (Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName())) 47 | Push-Location $tmp 48 | if (!$env:DEBUG) { 49 | trap { 50 | if ($tmp) { 51 | Pop-Location 52 | Remove-Item -Recurse -Force $tmp 53 | } 54 | break 55 | } 56 | } 57 | log "Temporary working directory: $tmp" 58 | 59 | # curl is built into PowerShell Core since version 6, but Windows 10 ships 60 | # with Windows PowerShell 5. orz 61 | log "Downloading $archive_url" 62 | Invoke-WebRequest $archive_url -OutFile $archive 63 | 64 | log "Extracting $archive" 65 | New-Item -Type Directory standalone | Out-Null 66 | Expand-Archive -Path $archive -DestinationPath standalone 67 | 68 | if (Test-Path $destination) { 69 | log "Removing existing $destination" 70 | Remove-Item -Recurse -Force $destination 71 | } 72 | 73 | log "Installing to $destination" 74 | New-Item -Type Directory -Force $(Split-Path $destination) | Out-Null 75 | Move-Item standalone $destination 76 | 77 | # Naively splitting is wrong in the general case, but fine for this check as 78 | # long as $destination itself doesn't contain a semi-colon. 79 | if ($destination -notin ($env:PATH -split ";")) { 80 | log "Prepending $destination to PATH for current user" 81 | 82 | # Update it for this session 83 | $env:PATH = "${destination};${env:PATH}" 84 | 85 | # Make it stick for new sessions. 86 | # 87 | # Note that this intentionally doesn't use $env:PATH to get the 88 | # previous value because $env:PATH is a dynamic per-process value 89 | # constructed from multiple sources, including the sticky per-user 90 | # environment we're modifying here. 91 | # 92 | # XXX TODO: This expands %VARS% in PATH, e.g. entries like 93 | # %SystemRoot%\system32 → C:\Windows\system32, when it roundtrips the 94 | # current value. I think this is basically harmless, and most users 95 | # probably have empty user environment PATHs anyway, but worth noting 96 | # as a future improvement in case it's not so harmless. I think to 97 | # avoid it we'd have to query and manipulate the registry directly 98 | # (instead of using this nice API), like 99 | # https://aka.ms/install-powershell.ps1 does. 100 | # -trs, 24 Aug 2022 101 | [Environment]::SetEnvironmentVariable("PATH", "$destination;" + [Environment]::GetEnvironmentVariable("PATH", "User"), "User") 102 | } 103 | 104 | Pop-Location 105 | 106 | if (!$env:DEBUG) { 107 | log "Cleaning up" 108 | Remove-Item -Recurse -Force $tmp 109 | } 110 | 111 | $version = & "$destination\nextstrain" --version 112 | 113 | echo @" 114 | ______________________________________________________________________________ 115 | 116 | Nextstrain CLI ($version) installed to $destination. 117 | "@ 118 | } 119 | 120 | function log { 121 | echo "--> $Args" 122 | } 123 | 124 | main @args 125 | 126 | # vim: set ft=ps1 : 127 | -------------------------------------------------------------------------------- /devel/changes: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Extract changelog for a specified version (or __NEXT__ if no version 4 | specified). 5 | """ 6 | from pathlib import Path 7 | from sys import argv, exit, stdout, stderr 8 | 9 | 10 | CHANGELOG = Path(__file__).parent / "../CHANGES.md" 11 | 12 | 13 | def main(version = "__NEXT__"): 14 | with CHANGELOG.open(encoding = "utf-8") as file: 15 | for line in file: 16 | # Find the heading for this version 17 | if line.rstrip("\n") == f"# {version}" or line.startswith(f"# {version} "): 18 | 19 | # Print subsequent lines until we reach the next version heading 20 | for line in file: 21 | if line.startswith("# "): 22 | break 23 | stdout.write(line) 24 | return 0 25 | 26 | print(f"No changes found for {version!r}.", file = stderr) 27 | return 1 28 | 29 | 30 | if __name__ == "__main__": 31 | exit(main(*argv[1:])) 32 | -------------------------------------------------------------------------------- /devel/create-github-release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | shopt -s extglob 4 | 5 | devel="$(dirname "$0")" 6 | 7 | main() { 8 | local version="${1:?version is required}" 9 | local commit pre_release 10 | shift 11 | local -a assets=("$@") 12 | 13 | if [[ ${#assets[@]} -eq 0 ]]; then 14 | echo "ERROR: no assets provided" >&2 15 | exit 1 16 | fi 17 | 18 | # Asserts that $version is an actual tag, not a branch name or other ref. 19 | git rev-parse --verify "$version^{tag}" >/dev/null 20 | 21 | # Translate from tag into commit for `gh release create --target`. 22 | commit="$(git rev-parse --verify "$version^{commit}")" 23 | 24 | if is-pre-release "$version"; then 25 | pre_release=1 26 | fi 27 | 28 | gh release create \ 29 | "$version" \ 30 | --repo "${GITHUB_REPOSITORY:-nextstrain/cli}" \ 31 | --title "$version" \ 32 | --target "$commit" \ 33 | ${pre_release:+--prerelease} \ 34 | --notes-file <(preamble; "$devel"/changes "$version" | "$devel"/reflow-markdown) \ 35 | "${assets[@]}" 36 | } 37 | 38 | is-pre-release() { 39 | # See https://peps.python.org/pep-0440/ 40 | local version="$1" 41 | case "$version" in 42 | *@(a|b|rc|c)+([0-9])*) # alpha, beta, release candidate (PEP 440 pre-releases) 43 | return 0;; 44 | *.dev+([0-9])*) # dev release 45 | return 0;; 46 | *+*) # local version 47 | return 0;; 48 | *) 49 | return 1;; 50 | esac 51 | } 52 | 53 | preamble() { 54 | cat <<~~ 55 | 56 | _These release notes are automatically extracted from the full [changelog][]._ 57 | 58 | [changelog]: https://github.com/nextstrain/cli/blob/master/CHANGES.md#readme 59 | 60 | ~~ 61 | } 62 | 63 | main "$@" 64 | -------------------------------------------------------------------------------- /devel/generate-changes-doc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Transform our :file:`CHANGES.md` into :file:`docs/changes.md` for use with 4 | Sphinx + MyST. 5 | 6 | We keep the former free of Sphinx/MyST-isms since it is used in several other 7 | contexts as well. 8 | """ 9 | import re 10 | import os 11 | from argparse import ArgumentParser 12 | from difflib import unified_diff 13 | from docutils.nodes import make_id 14 | from hashlib import md5 15 | from pathlib import Path 16 | from sys import exit, stdout, stderr 17 | from tempfile import NamedTemporaryFile 18 | from packaging.version import VERSION_PATTERN 19 | 20 | from nextstrain.cli.argparse import HelpFormatter 21 | from nextstrain.cli.debug import debug 22 | 23 | 24 | repo = Path(__file__).resolve().parent.parent 25 | 26 | 27 | version_heading = re.compile(r'^#\s+(?P(?ix:' + VERSION_PATTERN + r')|__NEXT__)(\s|$)').search 28 | subheading = re.compile(r'^##+\s+(?P.+)').search 29 | 30 | 31 | argparser = ArgumentParser( 32 | prog = "./devel/generate-changes-doc", 33 | usage = "./devel/generate-changes-doc [--check] [--diff]", 34 | description = __doc__, 35 | formatter_class = HelpFormatter) 36 | 37 | argparser.add_argument("--check", action = "store_true", help = "Only check if the generated contents need updating; do not actually update any files. Exits 1 if there are updates, 0 if not.") 38 | argparser.add_argument("--diff", action = "store_true", help = "Show a diff of updates to the generated contents (or would-be-updates, if --check is also specified).") 39 | 40 | 41 | def main(*, check = False, diff = False): 42 | src = repo / "CHANGES.md" 43 | dst = repo / "doc/changes.md" 44 | 45 | debug(f"Converting {src} → {dst}…") 46 | 47 | with src.open(encoding = "utf-8") as CHANGES: 48 | new = "".join(generate(CHANGES)).encode("utf-8") 49 | 50 | old = dst.read_bytes() if dst.exists() else None 51 | 52 | # Any updates? 53 | new_md5 = md5(new).hexdigest() 54 | old_md5 = md5(old).hexdigest() if old is not None else "0" * 32 55 | 56 | debug(f"Old MD5: {old_md5}") 57 | debug(f"New MD5: {new_md5}") 58 | 59 | if old_md5 != new_md5: 60 | if check: 61 | check_failed = True 62 | else: 63 | dst.write_bytes(new) 64 | debug(f"wrote {len(new):,} bytes ({new_md5}) to {dst}") 65 | print(dst, file = stderr) 66 | 67 | if diff: 68 | stdout.writelines( 69 | unified_diff( 70 | old.decode("utf-8").splitlines(keepends = True) if old is not None else [], 71 | new.decode("utf-8").splitlines(keepends = True), 72 | str(dst), 73 | str(dst), 74 | old_md5, 75 | new_md5)) 76 | 77 | else: 78 | if check: 79 | check_failed = False 80 | debug(f"{dst} unchanged") 81 | 82 | return 1 if check and check_failed else 0 83 | 84 | 85 | def generate(lines): 86 | # Title the document 87 | yield "# Changelog\n\n" 88 | 89 | version = None 90 | 91 | for line in lines: 92 | # Add targets for version headings and subheadings 93 | if match := version_heading(line): 94 | version = match["version"] 95 | version_id = make_id("v" + version) 96 | yield f"({version_id})=\n" 97 | 98 | elif version and (match := subheading(line)): 99 | heading = match["heading"] 100 | heading_id = make_id("v" + version + "-" + heading) 101 | yield f"({heading_id})=\n" 102 | 103 | # Offset heading levels by 1 104 | if line.startswith("#"): 105 | line = "#" + line 106 | 107 | # Rewrite relative links into doc/… to correct them. 108 | # 109 | # XXX TODO: This is a relatively crude approach, but we can always 110 | # improve it if necessary. For example, we could extend 111 | # nextstrain.cli.markdown to parse Link and LinkReference nodes and 112 | # then parse/rewrite/generate. 113 | # -trs, 28 May 2025 114 | line = re.sub(r'(?<=\]\()doc/', '', line) 115 | 116 | # Rewrite __NEXT__ links under the __NEXT__ version for RTD builds so 117 | # they work on PR previews and the "latest" version. Note that 118 | # CHANGES.md in released versions (and the "stable" version) should not 119 | # have any __NEXT__ version heading nor __NEXT__ links as both are 120 | # removed/rewritten by devel/release. 121 | # 122 | # 123 | if version == "__NEXT__" and (RTD_URL := os.environ.get("READTHEDOCS_CANONICAL_URL")): 124 | if not RTD_URL.endswith("/"): 125 | RTD_URL += "/" 126 | line = line.replace("https://docs.nextstrain.org/projects/cli/en/__NEXT__/", RTD_URL) 127 | 128 | yield line 129 | 130 | 131 | if __name__ == "__main__": 132 | exit(main(**vars(argparser.parse_args()))) 133 | -------------------------------------------------------------------------------- /devel/generate-doc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | devel="$(dirname "$0")" 5 | 6 | set -x 7 | "$devel"/generate-command-doc 8 | "$devel"/generate-changes-doc 9 | -------------------------------------------------------------------------------- /devel/pyoxidizer: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | shopt -s extglob failglob 4 | 5 | devel="$(dirname "$0")" 6 | 7 | main() { 8 | local pyoxidizer 9 | 10 | if pyoxidizer="$(locate 2>/dev/null)"; then 11 | echo "--> Located existing PyOxidizer binary at $pyoxidizer" >&2 12 | else 13 | echo "--> Downloading PyOxidizer binary…" >&2 14 | pyoxidizer="$(download)" 15 | echo "--> Downloaded PyOxidizer binary to $pyoxidizer" >&2 16 | fi 17 | 18 | # On Linux, exec into a manylinux container with an old glibc version. 19 | # These manylinux images are used in the Python packaging ecosystem to 20 | # build widely-compatible binary packages (wheels). 21 | # 22 | # https://github.com/pypa/manylinux#readme 23 | if [[ "$(platform-system)" == Linux ]]; then 24 | exec "$devel"/within-container quay.io/pypa/manylinux2014_x86_64 "$pyoxidizer" "$@" 25 | else 26 | exec "$pyoxidizer" "$@" 27 | fi 28 | } 29 | 30 | locate() { 31 | # Locate existing local copy of pyoxidizer. 32 | printf '%s\n' "$devel"/pyoxidizer-[0-9]*_"$(platform-machine)"?(.exe) | sort --reverse --version-sort | head -n1 33 | } 34 | 35 | download() { 36 | # Download pyoxidizer and return the path to it. 37 | # 38 | # Even though it's a Rust project, the easiest cross-platform way to 39 | # download it is via pip since they publish wheels with the binaries. :-) 40 | # Using Pip also allows us to piggy back on the version specification 41 | # system and platform-specific bits of a package manager/registry 42 | # ecosystem. 43 | local tmp 44 | tmp="$(mktemp -d)" 45 | 46 | # shellcheck disable=SC2064 47 | trap "rm -rf \"$tmp\"" EXIT 48 | 49 | python3 -m pip download \ 50 | --disable-pip-version-check \ 51 | --quiet \ 52 | --no-deps \ 53 | --dest "$tmp" \ 54 | 'pyoxidizer !=0.23.0' \ 55 | >&2 56 | 57 | local wheels wheel binary 58 | wheels=("$tmp"/pyoxidizer-[0-9]*.whl) 59 | wheel="${wheels[0]}" 60 | binary="$(basename "$wheel" .whl)" 61 | 62 | if [[ "$(platform-system)" == Windows ]]; then 63 | binary+=.exe 64 | fi 65 | 66 | unzip -p "$wheel" '*/scripts/pyoxidizer*' > "$devel/$binary" 67 | chmod +x "$devel/$binary" 68 | 69 | echo "$devel/$binary" 70 | } 71 | 72 | platform-machine() { 73 | python3 -c 'import platform; print(platform.machine())' 74 | } 75 | 76 | platform-system() { 77 | python3 -c 'import platform; print(platform.system())' 78 | } 79 | 80 | main "$@" 81 | -------------------------------------------------------------------------------- /devel/pytest: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | cd "$(dirname "$0")/.." 5 | 6 | set -x 7 | source tests/env 8 | exec pytest "$@" 9 | -------------------------------------------------------------------------------- /devel/read-version: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from pathlib import Path 3 | 4 | devel = Path(__file__).parent 5 | repo = devel.parent 6 | version_file = repo / "nextstrain/cli/__version__.py" 7 | 8 | exec(version_file.read_text()) 9 | print(__version__) 10 | -------------------------------------------------------------------------------- /devel/reflow-markdown: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Reflows paragraphs in Markdown to single long lines while preserving verbatim 3 | # code blocks, lists, and what not. 4 | set -euo pipefail 5 | 6 | devel="$(dirname "$0")" 7 | 8 | main() { 9 | pandoc --wrap none --from markdown --to markdown 10 | } 11 | 12 | pandoc() { 13 | # XXX TODO: This relies on Docker being available, which it typically is in 14 | # our development environments (local and CI). If Docker ever poses a 15 | # burden, we could switch to just-in-time downloading of static binaries 16 | # from and exec-ing those à 17 | # la what our devel/pyoxidizer does. 18 | # -trs, 18 Jan 2024 19 | "$devel"/within-container --interactive pandoc/core "$@" 20 | } 21 | 22 | main "$@" 23 | -------------------------------------------------------------------------------- /devel/release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | devel="$(dirname "$0")" 5 | repo="$devel/.." 6 | version_file="$repo/nextstrain/cli/__version__.py" 7 | changes_file="$repo/CHANGES.md" 8 | 9 | main() { 10 | local version 11 | 12 | assert-clean-working-dir 13 | assert-changelog-has-additions 14 | assert-changelog-has-changes-for-next 15 | 16 | version="${1:-$(next-version)}" 17 | echo "New version will be $version." 18 | 19 | update-version "$version" 20 | update-changelog "$version" 21 | commit-and-tag "$version" 22 | unreleased-version "$version+git" 23 | remind-to-push "$version" 24 | } 25 | 26 | assert-clean-working-dir() { 27 | local status 28 | 29 | status="$(git status --porcelain --untracked-files=no | grep -vwF "$(basename "$changes_file")" || true)" 30 | 31 | if [[ -n $status ]]; then 32 | echo "Please commit all changes before releasing:" >&2 33 | echo >&2 34 | echo "$status" >&2 35 | echo >&2 36 | echo "Only $(basename "$changes_file") is allowed to have uncommitted changes." >&2 37 | exit 1 38 | fi 39 | } 40 | 41 | assert-changelog-has-additions() { 42 | local current_version numstat 43 | 44 | current_version="$("$devel"/read-version)" 45 | numstat="$(git diff --numstat "${current_version%%+*}" -- "$changes_file")" 46 | 47 | local insertions deletions rest 48 | 49 | if [[ -z $numstat ]]; then 50 | insertions=0 51 | deletions=0 52 | else 53 | read -r insertions deletions rest <<<"$numstat" 54 | fi 55 | 56 | local net_changed=$((insertions - deletions)) 57 | 58 | if [[ $net_changed -lt 1 ]]; then 59 | echo "It doesn't look like $(basename "$changes_file") was updated; only $insertions - $deletions = $net_changed line(s) were changed." >&2 60 | exit 1 61 | fi 62 | } 63 | 64 | assert-changelog-has-changes-for-next() { 65 | if [[ -z "$("$devel"/changes __NEXT__)" ]]; then 66 | echo "It doesn't look like $(basename "$changes_file") has entries for __NEXT__." >&2 67 | exit 1 68 | fi 69 | } 70 | 71 | next-version() { 72 | local current_version 73 | current_version="$("$devel"/read-version)" 74 | current_version="${current_version%%+*}" 75 | 76 | read -r -e -p "Current version is $current_version."$'\n'"New version? " -i "$current_version" new_version 77 | 78 | if [[ -z $new_version || $new_version == "$current_version" ]]; then 79 | echo "You must provide a new version!" >&2 80 | exit 1 81 | fi 82 | 83 | echo "$new_version" 84 | } 85 | 86 | update-version() { 87 | local version="$1" 88 | 89 | "$devel"/update-version "$version" 90 | git add "$version_file" 91 | } 92 | 93 | update-changelog() { 94 | local new_version="$1" 95 | local today 96 | today="$(date +"%d %B %Y")" 97 | 98 | # Remove leading zero from day if present 99 | today="${today#0}" 100 | 101 | # Add the new version heading immediately after the __NEXT__ heading, 102 | # preserving the __NEXT__ heading itself. 103 | perl -pi -e "s/(?<=^# __NEXT__$)/\n\n\n# $new_version ($today)/" "$changes_file" 104 | 105 | # Replace any occurrences of __NEXT__ under the new version heading, e.g. 106 | # for use in doc URLs that should point to the released version. 107 | perl -pi -e "s/__NEXT__/$new_version/g if /^# \\Q$new_version/ ... /^# /" "$changes_file" 108 | 109 | # Remove the __NEXT__ heading and the sentence «The "__NEXT__" heading 110 | # below…» for the next commit, but leave the working tree untouched. 111 | perl -0p -e ' 112 | s/^# __NEXT__\n\n\n//m; 113 | s/\s*The "__NEXT__" heading below .+?\.//s; 114 | ' "$changes_file" | git-add-stdin-as "$changes_file" 115 | } 116 | 117 | git-add-stdin-as() { 118 | local path="$1" 119 | local repo_path mode object 120 | 121 | # Convert filesystem $path to a canonicalized path from the root of the 122 | # repo. This is required for the commands below. 123 | repo_path="$(git ls-files --full-name --error-unmatch "$path")" 124 | 125 | # Use existing mode (e.g. 100644) 126 | mode="$(git ls-tree --format "%(objectmode)" HEAD :/"$repo_path")" 127 | 128 | # Create new object in git's object database from the contents on stdin. 129 | # Using --path ensures that any filters (e.g. eol textconv or otherwise) 130 | # that would apply to $path are applied to the contents on stdin too. 131 | object="$(git hash-object -w --stdin --path "$repo_path")" 132 | 133 | # Stage the new object as an update to $path (as if with `git add` after 134 | # actually modifying $path). 135 | git update-index --cacheinfo "$mode,$object,$repo_path" 136 | } 137 | 138 | commit-and-tag() { 139 | local version="$1" 140 | 141 | # Staged changes to commit are added to the index by update-version and 142 | # update-changelog above. 143 | git commit -m "version $version" 144 | git tag -sm "version $version" "$version" 145 | } 146 | 147 | unreleased-version() { 148 | local unreleased_version="$1" 149 | 150 | # Add +git local part to mark any further development 151 | "$devel"/update-version "$unreleased_version" 152 | 153 | git add "$version_file" "$changes_file" 154 | git commit -m "dev: Bump version to $unreleased_version" 155 | } 156 | 157 | remind-to-push() { 158 | local version="$1" 159 | 160 | echo 161 | echo 162 | echo "Version updated, committed, and tagged!" 163 | echo 164 | echo "Please remember to push, including tags:" 165 | echo 166 | echo " git push origin master tag $version" 167 | echo 168 | echo "CI will build dists for the release tag and publish them after tests pass." 169 | echo 170 | } 171 | 172 | main "$@" 173 | -------------------------------------------------------------------------------- /devel/rtd-pre-build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | devel="$(dirname "$0")" 5 | 6 | set -x 7 | 8 | # Regenerate doc/changes.md for the "latest" version and PR previews to fix 9 | # __NEXT__ links. 10 | # 11 | # https://docs.readthedocs.com/platform/stable/reference/environment-variables.html#envvar-READTHEDOCS_VERSION 12 | # https://docs.readthedocs.com/platform/stable/reference/environment-variables.html#envvar-READTHEDOCS_VERSION_TYPE 13 | if [[ ${READTHEDOCS_VERSION:-} == latest || ${READTHEDOCS_VERSION_TYPE:-} == external ]]; then 14 | "$devel"/generate-changes-doc --diff 15 | fi 16 | -------------------------------------------------------------------------------- /devel/setup-venv: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | base="$(cd "$(dirname "$0")/.."; pwd)" 5 | venv="$base/.venv" 6 | 7 | set -x 8 | rm -rf "$venv" 9 | python3 -m venv "$venv" 10 | "$venv"/bin/pip install --upgrade pip setuptools wheel pip-tools 11 | "$venv"/bin/pip install -e '.[dev]' 12 | -------------------------------------------------------------------------------- /devel/update-version: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | devel="$(dirname "$0")" 5 | repo="$devel/.." 6 | version_file="$repo/nextstrain/cli/__version__.py" 7 | 8 | main() { 9 | local new_version="${1:?version is required}" 10 | 11 | perl -pi -e "s/(?<=^__version__ = ')(.*)(?='$)/$new_version/" "$version_file" 12 | 13 | if [[ $new_version != $("$devel"/read-version) ]]; then 14 | echo "Failed to update $version_file!" >&2 15 | exit 1 16 | fi 17 | } 18 | 19 | main "$@" 20 | -------------------------------------------------------------------------------- /devel/venv-run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # shellcheck disable=SC1091 3 | set -eou pipefail 4 | 5 | base="$(cd "$(dirname "$0")/.."; pwd)" 6 | 7 | source "$base/.venv/bin/activate" 8 | exec "$@" 9 | -------------------------------------------------------------------------------- /devel/within-container: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # usage: devel/within-container [ [ [...]]] 3 | set -euo pipefail 4 | 5 | exec docker run \ 6 | --rm \ 7 | --env HOME=/tmp \ 8 | --volume "$PWD:$PWD" \ 9 | --workdir "$PWD" \ 10 | --user "$(id -u):$(id -g)" \ 11 | "$@" 12 | -------------------------------------------------------------------------------- /doc/.gitignore: -------------------------------------------------------------------------------- 1 | /_build/ 2 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | SHELL := bash -euo pipefail 4 | 5 | # You can set these variables from the command line, and also 6 | # from the environment for the first three. 7 | SPHINXOPTS ?= 8 | SPHINXBUILD ?= sphinx-build 9 | BUILDDIR ?= _build 10 | SOURCEDIR = . 11 | 12 | # Require stricter builds with 13 | # -n: warn on missing references 14 | # -W: error on warnings 15 | # --keep-going: find all warnings 16 | # https://www.sphinx-doc.org/en/master/man/sphinx-build.html 17 | STRICT = -n -W --keep-going 18 | LOOSE = -n 19 | 20 | GENERATE = ../devel/generate-doc 21 | 22 | # Put it first so that "make" without argument is like "make help". 23 | help: 24 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 25 | 26 | .PHONY: help Makefile livehtml 27 | 28 | # Catch-all target: route all unknown targets to Sphinx using the new 29 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 30 | %: Makefile 31 | if [[ $@ != clean ]]; then $(GENERATE); fi 32 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(STRICT) $(SPHINXOPTS) $(O) 33 | 34 | HOST ?= 127.0.0.1 35 | PORT ?= 8000 36 | 37 | serve: dirhtml 38 | cd "$(BUILDDIR)/dirhtml" && python3 -m http.server --bind "$(HOST)" "$(PORT)" 39 | 40 | livehtml: 41 | sphinx-autobuild -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)/dirhtml" --host "$(HOST)" --port "$(PORT)" --watch ../nextstrain/cli --pre-build "$(GENERATE)" $(LOOSE) $(SPHINXOPTS) $(O) 42 | -------------------------------------------------------------------------------- /doc/commands/authorization.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain authorization 6 | 7 | .. _nextstrain authorization: 8 | 9 | ======================== 10 | nextstrain authorization 11 | ======================== 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain authorization [-h] [] 16 | 17 | 18 | Produce an Authorization header appropriate for the web API of nextstrain.org 19 | (and other remotes). 20 | 21 | This is a development tool unnecessary for normal usage. It's useful for 22 | directly making API requests to nextstrain.org (and other remotes) with `curl` 23 | or similar commands. For example:: 24 | 25 | curl -si https://nextstrain.org/whoami \ 26 | --header "Accept: application/json" \ 27 | --header @<(nextstrain authorization) 28 | 29 | Exits with an error if no one is logged in. 30 | 31 | positional arguments 32 | ==================== 33 | 34 | 35 | 36 | .. option:: 37 | 38 | Remote URL for which to produce an Authorization header. Expects 39 | URLs like the remote source/destination URLs used by the 40 | `nextstrain remote` family of commands. Only the domain name 41 | (technically, the origin) of the URL is required/used, but a full 42 | URL may be specified. 43 | 44 | options 45 | ======= 46 | 47 | 48 | 49 | .. option:: -h, --help 50 | 51 | show this help message and exit 52 | 53 | -------------------------------------------------------------------------------- /doc/commands/check-setup.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain check-setup 6 | 7 | .. _nextstrain check-setup: 8 | 9 | ====================== 10 | nextstrain check-setup 11 | ====================== 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain check-setup [--set-default] [ [ ...]] 16 | nextstrain check-setup --help 17 | 18 | 19 | Checks for supported runtimes. 20 | 21 | Five runtimes are tested by default: 22 | 23 | • Our Docker image is the preferred runtime. Docker itself must 24 | be installed and configured on your computer first, but once it is, the 25 | runtime is robust and reproducible. 26 | 27 | • Our Conda runtime will be tested for existence and appearance of 28 | completeness. This runtime is more isolated and reproducible than your 29 | ambient runtime, but is less isolated and robust than the Docker 30 | runtime. 31 | 32 | • Our Singularity runtime uses the same container image as our Docker 33 | runtime. Singularity must be installed and configured on your computer 34 | first, although it is often already present on HPC systems. This runtime 35 | is more isolated and reproducible than the Conda runtime, but potentially 36 | less so than the Docker runtime. 37 | 38 | • Your ambient setup will be tested for snakemake, augur, and auspice. 39 | Their presence implies a working runtime, but does not guarantee 40 | it. 41 | 42 | • Remote jobs on AWS Batch. Your AWS account, if credentials are available 43 | in your environment or via aws-cli configuration, will be tested for the 44 | presence of appropriate resources. Their presence implies a working AWS 45 | Batch runtime, but does not guarantee it. 46 | 47 | Provide one or more runtime names as arguments to test just those instead. 48 | 49 | Exits with an error code if the default runtime (docker) is not 50 | supported or, when the default runtime is omitted from checks, if none of the 51 | checked runtimes are supported. 52 | 53 | positional arguments 54 | ==================== 55 | 56 | 57 | 58 | .. option:: 59 | 60 | The Nextstrain runtimes to check. (default: docker, conda, singularity, ambient, aws-batch) 61 | 62 | options 63 | ======= 64 | 65 | 66 | 67 | .. option:: -h, --help 68 | 69 | show this help message and exit 70 | 71 | .. option:: --set-default 72 | 73 | Set the default runtime to the first which passes check-setup. Checks run in the order given, if any, otherwise in the default order: docker, conda, singularity, ambient, aws-batch. 74 | 75 | -------------------------------------------------------------------------------- /doc/commands/debugger.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. default-role:: literal 4 | 5 | .. role:: command-reference(ref) 6 | 7 | .. program:: nextstrain debugger 8 | 9 | .. _nextstrain debugger: 10 | 11 | =================== 12 | nextstrain debugger 13 | =================== 14 | 15 | .. code-block:: none 16 | 17 | usage: nextstrain debugger [-h] 18 | 19 | 20 | Launch pdb from within the Nextstrain CLI process. 21 | 22 | This is a development and troubleshooting tool unnecessary for normal usage. 23 | 24 | options 25 | ======= 26 | 27 | 28 | 29 | .. option:: -h, --help 30 | 31 | show this help message and exit 32 | 33 | -------------------------------------------------------------------------------- /doc/commands/deploy.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain deploy 6 | 7 | .. _nextstrain deploy: 8 | 9 | ================= 10 | nextstrain deploy 11 | ================= 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain deploy [-h] [--dry-run] [ […]] [ [ […]] ...] 16 | 17 | 18 | Upload dataset and narratives files to a remote destination. 19 | 20 | 21 | The `nextstrain deploy` command is an alias for `nextstrain remote upload`. 22 | 23 | 24 | A remote destination URL specifies where to upload, e.g. to upload the dataset 25 | files:: 26 | 27 | auspice/ncov_local.json 28 | auspice/ncov_local_root-sequence.json 29 | auspice/ncov_local_tip-frequencies.json 30 | 31 | so they're visible at `https://nextstrain.org/groups/example/ncov`:: 32 | 33 | nextstrain remote upload nextstrain.org/groups/example/ncov auspice/ncov_local*.json 34 | 35 | If uploading multiple datasets or narratives, uploading to the top-level of a 36 | Nextstrain Group, or uploading to an S3 remote, then the local filenames are 37 | used in combination with any path prefix in the remote source URL. 38 | 39 | See :command-reference:`nextstrain remote` for more information on remote sources. 40 | 41 | positional arguments 42 | ==================== 43 | 44 | 45 | 46 | .. option:: 47 | 48 | Remote destination URL for a dataset or narrative. A path prefix if the files to upload comprise more than one dataset or narrative or the remote is S3. 49 | 50 | .. option:: [ […]] 51 | 52 | Files to upload. Typically dataset and sidecar files (Auspice JSON files) and/or narrative files (Markdown files). 53 | 54 | options 55 | ======= 56 | 57 | 58 | 59 | .. option:: -h, --help 60 | 61 | show this help message and exit 62 | 63 | .. option:: --dry-run 64 | 65 | Don't actually upload anything, just show what would be uploaded 66 | 67 | -------------------------------------------------------------------------------- /doc/commands/index.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. default-role:: literal 4 | 5 | .. role:: command-reference(ref) 6 | 7 | .. program:: nextstrain 8 | 9 | .. _nextstrain: 10 | 11 | ========== 12 | nextstrain 13 | ========== 14 | 15 | .. code-block:: none 16 | 17 | usage: nextstrain [-h] {run,build,view,deploy,remote,shell,update,setup,check-setup,login,logout,whoami,version,init-shell,authorization,debugger} ... 18 | 19 | 20 | Nextstrain command-line interface (CLI) 21 | 22 | The `nextstrain` program and its subcommands aim to provide a consistent way to 23 | run and visualize pathogen builds and access Nextstrain components like Augur 24 | and Auspice across computing platforms such as Docker, Conda, Singularity, and 25 | AWS Batch. 26 | 27 | Run `nextstrain --help` for usage information about each command. 28 | See <:doc:`/index`> for more documentation. 29 | 30 | options 31 | ======= 32 | 33 | 34 | 35 | .. option:: -h, --help 36 | 37 | show this help message and exit 38 | 39 | commands 40 | ======== 41 | 42 | 43 | 44 | .. option:: run 45 | 46 | Run pathogen workflow. See :doc:`/commands/run`. 47 | 48 | .. option:: build 49 | 50 | Run pathogen build. See :doc:`/commands/build`. 51 | 52 | .. option:: view 53 | 54 | View pathogen builds and narratives. See :doc:`/commands/view`. 55 | 56 | .. option:: deploy 57 | 58 | Deploy pathogen build. See :doc:`/commands/deploy`. 59 | 60 | .. option:: remote 61 | 62 | Upload, download, and manage remote datasets and narratives.. See :doc:`/commands/remote/index`. 63 | 64 | .. option:: shell 65 | 66 | Start a new shell in a runtime. See :doc:`/commands/shell`. 67 | 68 | .. option:: update 69 | 70 | Update a pathogen or runtime. See :doc:`/commands/update`. 71 | 72 | .. option:: setup 73 | 74 | Set up a pathogen or runtime. See :doc:`/commands/setup`. 75 | 76 | .. option:: check-setup 77 | 78 | Check runtime setups. See :doc:`/commands/check-setup`. 79 | 80 | .. option:: login 81 | 82 | Log into Nextstrain.org (and other remotes). See :doc:`/commands/login`. 83 | 84 | .. option:: logout 85 | 86 | Log out of Nextstrain.org (and other remotes). See :doc:`/commands/logout`. 87 | 88 | .. option:: whoami 89 | 90 | Show information about the logged-in user. See :doc:`/commands/whoami`. 91 | 92 | .. option:: version 93 | 94 | Show version information. See :doc:`/commands/version`. 95 | 96 | .. option:: init-shell 97 | 98 | Print shell init script. See :doc:`/commands/init-shell`. 99 | 100 | .. option:: authorization 101 | 102 | Print an HTTP Authorization header. See :doc:`/commands/authorization`. 103 | 104 | .. option:: debugger 105 | 106 | Start a debugger. See :doc:`/commands/debugger`. 107 | 108 | -------------------------------------------------------------------------------- /doc/commands/init-shell.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. default-role:: literal 4 | 5 | .. role:: command-reference(ref) 6 | 7 | .. program:: nextstrain init-shell 8 | 9 | .. _nextstrain init-shell: 10 | 11 | ===================== 12 | nextstrain init-shell 13 | ===================== 14 | 15 | .. code-block:: none 16 | 17 | usage: nextstrain init-shell [-h] [shell] 18 | 19 | 20 | Prints the shell init script for a Nextstrain CLI standalone installation. 21 | 22 | If PATH does not contain the expected installation path, emits an appropriate 23 | ``export PATH=…`` statement. Otherwise, emits only a comment. 24 | 25 | Use this command in your shell config with a line like the following:: 26 | 27 | eval "$(…/path/to/nextstrain init-shell)" 28 | 29 | Exits with error if run in an non-standalone installation. 30 | 31 | positional arguments 32 | ==================== 33 | 34 | 35 | 36 | .. option:: shell 37 | 38 | Shell that's being initialized (e.g. bash, zsh, etc.); currently we always emit POSIX shell syntax but this may change in the future. 39 | 40 | options 41 | ======= 42 | 43 | 44 | 45 | .. option:: -h, --help 46 | 47 | show this help message and exit 48 | 49 | -------------------------------------------------------------------------------- /doc/commands/login.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain login 6 | 7 | .. _nextstrain login: 8 | 9 | ================ 10 | nextstrain login 11 | ================ 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain login [-h] [--username ] [--no-prompt] [--renew] [] 16 | 17 | 18 | Log into Nextstrain.org (and other remotes) and save credentials for later use. 19 | 20 | The first time you log in to a remote you'll be prompted to authenticate via 21 | your web browser or, if you provide a username (e.g. with --username), for your 22 | Nextstrain.org password. After that, locally-saved authentication tokens will 23 | be used and automatically renewed as needed when you run other `nextstrain` 24 | commands requiring log in. You can also re-run this `nextstrain login` command 25 | to force renewal if you want. You'll only be prompted to reauthenticate (via 26 | your web browser or username/password) if the locally-saved tokens are unable 27 | to be renewed or missing entirely. 28 | 29 | If you log out of Nextstrain.org (or other remotes) on other devices/clients 30 | (like your web browser), you may be prompted to reauthenticate by this command 31 | sooner than usual. 32 | 33 | Your username and password themselves are never saved locally. 34 | 35 | positional arguments 36 | ==================== 37 | 38 | 39 | 40 | .. option:: 41 | 42 | Remote URL to log in to, like the remote source/destination URLs 43 | used by the `nextstrain remote` family of commands. Only the 44 | domain name (technically, the origin) of the URL is required/used, 45 | but a full URL may be specified. 46 | 47 | options 48 | ======= 49 | 50 | 51 | 52 | .. option:: -h, --help 53 | 54 | show this help message and exit 55 | 56 | .. option:: --username , -u 57 | 58 | The username to log in as. If not provided, the :envvar:`NEXTSTRAIN_USERNAME` environment variable will be used if available, otherwise you'll be prompted to enter your username. 59 | 60 | .. option:: --no-prompt 61 | 62 | Never prompt for authentication (via web browser or username/password); succeed only if there are login credentials in the environment or existing valid/renewable tokens saved locally, otherwise error. Useful for scripting. 63 | 64 | .. option:: --renew 65 | 66 | Renew existing tokens, if possible. Useful to refresh group membership information (for example) sooner than the tokens would normally be renewed. 67 | 68 | For automation purposes, you may opt to provide environment variables instead 69 | of interactive input and/or command-line options: 70 | 71 | .. envvar:: NEXTSTRAIN_USERNAME 72 | 73 | Username on nextstrain.org. Ignored if :option:`--username` is also 74 | provided. 75 | 76 | .. envvar:: NEXTSTRAIN_PASSWORD 77 | 78 | Password for nextstrain.org user. Required if :option:`--no-prompt` is 79 | used without existing valid/renewable tokens. 80 | 81 | If you want to suppress ever opening a web browser automatically, you 82 | may set the environment variable ``NOBROWSER=1``. -------------------------------------------------------------------------------- /doc/commands/logout.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain logout 6 | 7 | .. _nextstrain logout: 8 | 9 | ================= 10 | nextstrain logout 11 | ================= 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain logout [] 16 | nextstrain logout --all 17 | nextstrain logout --help 18 | 19 | 20 | Log out of Nextstrain.org (and other remotes) by deleting locally-saved 21 | credentials. 22 | 23 | The authentication tokens are removed but not invalidated, so if you used them 24 | outside of the `nextstrain` command, they will remain valid until they expire. 25 | 26 | Other devices/clients (like your web browser) are not logged out of 27 | Nextstrain.org (or other remotes). 28 | 29 | positional arguments 30 | ==================== 31 | 32 | 33 | 34 | .. option:: 35 | 36 | Remote URL to log out of, like the remote source/destination URLs 37 | used by the `nextstrain remote` family of commands. Only the 38 | domain name (technically, the origin) of the URL is required/used, 39 | but a full URL may be specified. 40 | 41 | options 42 | ======= 43 | 44 | 45 | 46 | .. option:: -h, --help 47 | 48 | show this help message and exit 49 | 50 | .. option:: --all 51 | 52 | Log out of all remotes for which there are locally-saved credentials 53 | 54 | -------------------------------------------------------------------------------- /doc/commands/remote/delete.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain remote delete 6 | 7 | .. _nextstrain remote delete: 8 | 9 | ======================== 10 | nextstrain remote delete 11 | ======================== 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain remote delete [--recursively] 16 | nextstrain remote delete --help 17 | 18 | 19 | Delete datasets and narratives on a remote source. 20 | 21 | A remote source URL specifies what to delete, e.g. to delete the "beta-cov" 22 | dataset in the Nextstrain Group "blab":: 23 | 24 | nextstrain remote delete nextstrain.org/groups/blab/beta-cov 25 | 26 | The :option:`--recursively` option allows for deleting multiple datasets or narratives 27 | at once, e.g. to delete all the "ncov/wa/…" datasets in the "blab" group:: 28 | 29 | nextstrain remote delete --recursively nextstrain.org/groups/blab/ncov/wa 30 | 31 | See :command-reference:`nextstrain remote` for more information on remote sources. 32 | 33 | positional arguments 34 | ==================== 35 | 36 | 37 | 38 | .. option:: 39 | 40 | Remote source URL for a dataset or narrative. A path prefix to scope/filter by if using :option:`--recursively`. 41 | 42 | options 43 | ======= 44 | 45 | 46 | 47 | .. option:: -h, --help 48 | 49 | show this help message and exit 50 | 51 | .. option:: --recursively, -r 52 | 53 | Delete everything under the given remote URL path prefix 54 | 55 | .. option:: --dry-run 56 | 57 | Don't actually delete anything, just show what would be deleted 58 | 59 | -------------------------------------------------------------------------------- /doc/commands/remote/download.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain remote download 6 | 7 | .. _nextstrain remote download: 8 | 9 | ========================== 10 | nextstrain remote download 11 | ========================== 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain remote download [] 16 | nextstrain remote download --recursively [] 17 | nextstrain remote download --help 18 | 19 | 20 | Download datasets and narratives from a remote source. 21 | 22 | A remote source URL specifies what to download, e.g. to download one of the 23 | seasonal influenza datasets:: 24 | 25 | nextstrain remote download nextstrain.org/flu/seasonal/h3n2/ha/2y 26 | 27 | which creates three files in the current directory:: 28 | 29 | flu_seasonal_h3n2_ha_2y.json 30 | flu_seasonal_h3n2_ha_2y_root-sequence.json 31 | flu_seasonal_h3n2_ha_2y_tip-frequencies.json 32 | 33 | The --recursively option allows for downloading multiple datasets or narratives 34 | at once, e.g. to download all the datasets under "ncov/open/…" into an existing 35 | directory named "sars-cov-2":: 36 | 37 | nextstrain remote download --recursively nextstrain.org/ncov/open sars-cov-2/ 38 | 39 | which creates files for each dataset:: 40 | 41 | sars-cov-2/ncov_open_global.json 42 | sars-cov-2/ncov_open_global_root-sequence.json 43 | sars-cov-2/ncov_open_global_tip-frequencies.json 44 | sars-cov-2/ncov_open_africa.json 45 | sars-cov-2/ncov_open_africa_root-sequence.json 46 | sars-cov-2/ncov_open_africa_tip-frequencies.json 47 | … 48 | 49 | See :command-reference:`nextstrain remote` for more information on remote sources. 50 | 51 | positional arguments 52 | ==================== 53 | 54 | 55 | 56 | .. option:: 57 | 58 | Remote source URL for a dataset or narrative. A path prefix to scope/filter by if using :option:`--recursively`. 59 | 60 | .. option:: 61 | 62 | Local directory to save files in. May be a local filename to use if not using :option:`--recursively`. Defaults to current directory ("."). 63 | 64 | options 65 | ======= 66 | 67 | 68 | 69 | .. option:: -h, --help 70 | 71 | show this help message and exit 72 | 73 | .. option:: --recursively, -r 74 | 75 | Download everything under the given remote URL path prefix 76 | 77 | .. option:: --dry-run 78 | 79 | Don't actually download anything, just show what would be downloaded 80 | 81 | -------------------------------------------------------------------------------- /doc/commands/remote/index.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain remote 6 | 7 | .. _nextstrain remote: 8 | 9 | ================= 10 | nextstrain remote 11 | ================= 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain remote [-h] {upload,download,list,ls,delete,rm} ... 16 | 17 | 18 | Upload, download, and manage remote datasets and narratives. 19 | 20 | nextstrain.org is the primary remote source/destination for most users, but 21 | Amazon S3 buckets are also supported for some internal use cases. 22 | 23 | Remote sources/destinations are specified using URLs starting with 24 | ``https://nextstrain.org/`` and ``s3:///``. nextstrain.org remote 25 | URLs represent datasets and narratives each as a whole, where datasets may 26 | consist of multiple files (the main JSON file + optional sidecar files) when 27 | uploading/downloading. Amazon S3 remote URLs represent dataset and narrative 28 | files individually. 29 | 30 | For more details on using each remote, see their respective documentation 31 | pages: 32 | 33 | * :doc:`/remotes/nextstrain.org` 34 | * :doc:`/remotes/s3` 35 | 36 | For more information on dataset (Auspice JSON) and narrative (Markdown) files, 37 | see :doc:`docs:reference/data-formats`. 38 | 39 | options 40 | ======= 41 | 42 | 43 | 44 | .. option:: -h, --help 45 | 46 | show this help message and exit 47 | 48 | commands 49 | ======== 50 | 51 | 52 | 53 | .. option:: upload 54 | 55 | Upload dataset and narrative files. See :doc:`/commands/remote/upload`. 56 | 57 | .. option:: download 58 | 59 | Download dataset and narrative files. See :doc:`/commands/remote/download`. 60 | 61 | .. option:: list 62 | 63 | List datasets and narratives. See :doc:`/commands/remote/list`. 64 | 65 | .. option:: delete 66 | 67 | Delete dataset and narratives. See :doc:`/commands/remote/delete`. 68 | 69 | -------------------------------------------------------------------------------- /doc/commands/remote/list.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain remote list 6 | 7 | .. _nextstrain remote list: 8 | 9 | ====================== 10 | nextstrain remote list 11 | ====================== 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain remote list 16 | nextstrain remote list --help 17 | 18 | 19 | List datasets and narratives on a remote source. 20 | 21 | A remote source URL specifies what to list, e.g. to list what's in the 22 | Nextstrain Group named "Blab":: 23 | 24 | nextstrain remote list nextstrain.org/groups/blab 25 | 26 | or list the core seasonal influenza datasets:: 27 | 28 | nextstrain remote list nextstrain.org/flu/seasonal 29 | 30 | See :command-reference:`nextstrain remote` for more information on remote sources. 31 | 32 | positional arguments 33 | ==================== 34 | 35 | 36 | 37 | .. option:: 38 | 39 | Remote source URL, with optional path prefix to scope/filter the results 40 | 41 | options 42 | ======= 43 | 44 | 45 | 46 | .. option:: -h, --help 47 | 48 | show this help message and exit 49 | 50 | -------------------------------------------------------------------------------- /doc/commands/remote/upload.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain remote upload 6 | 7 | .. _nextstrain remote upload: 8 | 9 | ======================== 10 | nextstrain remote upload 11 | ======================== 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain remote upload [ […]] 16 | nextstrain remote upload --help 17 | 18 | 19 | Upload dataset and narratives files to a remote destination. 20 | 21 | A remote destination URL specifies where to upload, e.g. to upload the dataset 22 | files:: 23 | 24 | auspice/ncov_local.json 25 | auspice/ncov_local_root-sequence.json 26 | auspice/ncov_local_tip-frequencies.json 27 | 28 | so they're visible at `https://nextstrain.org/groups/example/ncov`:: 29 | 30 | nextstrain remote upload nextstrain.org/groups/example/ncov auspice/ncov_local*.json 31 | 32 | If uploading multiple datasets or narratives, uploading to the top-level of a 33 | Nextstrain Group, or uploading to an S3 remote, then the local filenames are 34 | used in combination with any path prefix in the remote source URL. 35 | 36 | See :command-reference:`nextstrain remote` for more information on remote sources. 37 | 38 | positional arguments 39 | ==================== 40 | 41 | 42 | 43 | .. option:: 44 | 45 | Remote destination URL for a dataset or narrative. A path prefix if the files to upload comprise more than one dataset or narrative or the remote is S3. 46 | 47 | .. option:: [ […]] 48 | 49 | Files to upload. Typically dataset and sidecar files (Auspice JSON files) and/or narrative files (Markdown files). 50 | 51 | options 52 | ======= 53 | 54 | 55 | 56 | .. option:: -h, --help 57 | 58 | show this help message and exit 59 | 60 | .. option:: --dry-run 61 | 62 | Don't actually upload anything, just show what would be uploaded 63 | 64 | -------------------------------------------------------------------------------- /doc/commands/setup.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain setup 6 | 7 | .. _nextstrain setup: 8 | 9 | ================ 10 | nextstrain setup 11 | ================ 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain setup [--dry-run] [--force] [--set-default] [@[=]] 16 | nextstrain setup [--dry-run] [--force] [--set-default] 17 | nextstrain setup --help 18 | 19 | 20 | Sets up a Nextstrain pathogen for use with `nextstrain run` or a Nextstrain 21 | runtime for use with `nextstrain run`, `nextstrain build`, `nextstrain view`, 22 | etc. 23 | 24 | For pathogens, set up involves downloading a specific version of the pathogen's 25 | Nextstrain workflows. By convention, this download is from Nextstrain's 26 | repositories. More than one version of the same pathogen may be set up and 27 | used independently. This can be useful for comparing analyses across workflow 28 | versions. A default version can be set. 29 | 30 | For runtimes, only the Conda runtime currently supports fully-automated set up, 31 | but this command may still be used with other runtimes to check an existing 32 | (manual) setup and set the runtime as the default on success. 33 | 34 | Exits with an error code if automated set up fails or if setup checks fail. 35 | 36 | positional arguments 37 | ==================== 38 | 39 | 40 | 41 | .. option:: | 42 | 43 | The Nextstrain pathogen or runtime to set up. 44 | 45 | A pathogen is usually the plain name of a Nextstrain-maintained 46 | pathogen (e.g. ``measles``), optionally with an ``@`` 47 | specifier (e.g. ``measles@v42``). If ```` is specified in 48 | this case, it must be a tag name (i.e. a release name), development 49 | branch name, or a development commit id. 50 | 51 | A pathogen may also be fully-specified as ``@=`` 52 | where ```` and ```` in this case are (mostly) 53 | arbitrary and ```` points to a ZIP file containing the 54 | pathogen repository contents (e.g. 55 | ``https://github.com/nextstrain/measles/zipball/83b446d67fc03de2ce1c72bb1345b4c4eace7231``). 56 | 57 | A runtime is one of {docker, conda, singularity, ambient, aws-batch}. 58 | 59 | 60 | options 61 | ======= 62 | 63 | 64 | 65 | .. option:: -h, --help 66 | 67 | show this help message and exit 68 | 69 | .. option:: --dry-run 70 | 71 | Don't actually set up anything, just show what would happen. 72 | 73 | .. option:: --force 74 | 75 | Ignore existing setup, if any, and always start fresh. 76 | 77 | .. option:: --set-default 78 | 79 | Use this pathogen version or runtime as the default if set up is successful. 80 | 81 | -------------------------------------------------------------------------------- /doc/commands/shell.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain shell 6 | 7 | .. _nextstrain shell: 8 | 9 | ================ 10 | nextstrain shell 11 | ================ 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain shell [options] [...] 16 | nextstrain shell --help 17 | 18 | 19 | Start a new shell inside a Nextstrain runtime to run ad-hoc 20 | commands and perform debugging. 21 | 22 | positional arguments 23 | ==================== 24 | 25 | 26 | 27 | .. option:: 28 | 29 | Path to pathogen build directory 30 | 31 | .. option:: ... 32 | 33 | Additional arguments to pass to the executed program 34 | 35 | options 36 | ======= 37 | 38 | 39 | 40 | .. option:: --help, -h 41 | 42 | Show a brief help message of common options and exit 43 | 44 | .. option:: --help-all 45 | 46 | Show a full help message of all options and exit 47 | 48 | runtime selection options 49 | ========================= 50 | 51 | Select the Nextstrain runtime to use, if the 52 | default is not suitable. 53 | 54 | .. option:: --docker 55 | 56 | Run commands inside a container image using Docker. (default) 57 | 58 | .. option:: --conda 59 | 60 | Run commands with access to a fully-managed Conda environment. 61 | 62 | .. option:: --singularity 63 | 64 | Run commands inside a container image using Singularity. 65 | 66 | runtime options 67 | =============== 68 | 69 | Options shared by all runtimes. 70 | 71 | .. option:: --env [=] 72 | 73 | Set the environment variable ```` to the value in the current environment (i.e. pass it thru) or to the given ````. May be specified more than once. Overrides any variables of the same name set via :option:`--envdir`. When this option or :option:`--envdir` is given, the default behaviour of automatically passing thru several "well-known" variables is disabled. The "well-known" variables are ``AUGUR_RECURSION_LIMIT``, ``AUGUR_MINIFY_JSON``, ``AWS_ACCESS_KEY_ID``, ``AWS_SECRET_ACCESS_KEY``, ``AWS_SESSION_TOKEN``, ``ID3C_URL``, ``ID3C_USERNAME``, ``ID3C_PASSWORD``, ``RETHINK_HOST``, and ``RETHINK_AUTH_KEY``. Pass those variables explicitly via :option:`--env` or :option:`--envdir` if you need them in combination with other variables. 74 | 75 | .. option:: --envdir 76 | 77 | Set environment variables from the envdir at ````. May be specified more than once. An envdir is a directory containing files describing environment variables. Each filename is used as the variable name. The first line of the contents of each file is used as the variable value. When this option or :option:`--env` is given, the default behaviour of automatically passing thru several "well-known" variables is disabled. Envdirs may also be specified by setting ``NEXTSTRAIN_RUNTIME_ENVDIRS`` in the environment to a ``:``-separated list of paths. See the description of :option:`--env` for more details. 78 | 79 | development options 80 | =================== 81 | 82 | These should generally be unnecessary unless you're developing Nextstrain. 83 | 84 | .. option:: --image 85 | 86 | Container image name to use for the Nextstrain runtime (default: nextstrain/base for Docker and AWS Batch, docker://nextstrain/base for Singularity) 87 | 88 | .. option:: --augur 89 | 90 | Replace the image's copy of augur with a local copy 91 | 92 | .. option:: --auspice 93 | 94 | Replace the image's copy of auspice with a local copy 95 | 96 | .. option:: --fauna 97 | 98 | Replace the image's copy of fauna with a local copy 99 | 100 | .. option:: --exec 101 | 102 | Program to run inside the runtime 103 | 104 | development options for --docker 105 | ================================ 106 | 107 | 108 | 109 | .. option:: --docker-arg ... 110 | 111 | Additional arguments to pass to `docker run` 112 | 113 | -------------------------------------------------------------------------------- /doc/commands/update.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain update 6 | 7 | .. _nextstrain update: 8 | 9 | ================= 10 | nextstrain update 11 | ================= 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain update [[@] | […]] 16 | nextstrain update 17 | nextstrain update --help 18 | 19 | 20 | Updates Nextstrain pathogens and runtimes to the latest available versions, if any. 21 | 22 | When this command is run without arguments, the default version for each set up 23 | pathogen (none) and the default runtime (docker) 24 | are updated. Provide one or more pathogens and/or runtimes as arguments to 25 | update a select list instead. 26 | 27 | Three runtimes currently support updates: Docker, Conda, and Singularity. 28 | Updates may take several minutes as new software versions are downloaded. 29 | 30 | This command also checks for newer versions of the Nextstrain CLI (the 31 | `nextstrain` program) itself and will suggest upgrade instructions if an 32 | upgrade is available. 33 | 34 | positional arguments 35 | ==================== 36 | 37 | 38 | 39 | .. option:: | 40 | 41 | The Nextstrain pathogens and/or runtimes to update. 42 | 43 | A pathogen is the name (and optionally, version) of a previously 44 | set up pathogen. See :command-reference:`nextstrain setup`. If no 45 | version is specified, then the default version will be updated to 46 | the latest available version. 47 | 48 | A runtime is one of {docker, conda, singularity, ambient, aws-batch}. 49 | 50 | 51 | 52 | 53 | options 54 | ======= 55 | 56 | 57 | 58 | .. option:: -h, --help 59 | 60 | show this help message and exit 61 | 62 | -------------------------------------------------------------------------------- /doc/commands/version.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain version 6 | 7 | .. _nextstrain version: 8 | 9 | ================== 10 | nextstrain version 11 | ================== 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain version [-h] [--verbose] [--pathogens] [--runtimes] 16 | 17 | 18 | Prints the version of the Nextstrain CLI. 19 | 20 | options 21 | ======= 22 | 23 | 24 | 25 | .. option:: -h, --help 26 | 27 | show this help message and exit 28 | 29 | .. option:: --verbose 30 | 31 | Show versions of each runtime, plus select individual Nextstrain components within, and versions of each pathogen, including URLs 32 | 33 | .. option:: --pathogens 34 | 35 | Show pathogen versions; implied by --verbose 36 | 37 | .. option:: --runtimes 38 | 39 | Show runtime versions; implied by --verbose 40 | 41 | -------------------------------------------------------------------------------- /doc/commands/whoami.rst: -------------------------------------------------------------------------------- 1 | .. default-role:: literal 2 | 3 | .. role:: command-reference(ref) 4 | 5 | .. program:: nextstrain whoami 6 | 7 | .. _nextstrain whoami: 8 | 9 | ================= 10 | nextstrain whoami 11 | ================= 12 | 13 | .. code-block:: none 14 | 15 | usage: nextstrain whoami [-h] [] 16 | 17 | 18 | Show information about the logged-in user for Nextstrain.org (and other 19 | remotes). 20 | 21 | The username, email address (if available), and Nextstrain Groups memberships 22 | of the currently logged-in user are shown. 23 | 24 | Exits with an error if no one is logged in. 25 | 26 | positional arguments 27 | ==================== 28 | 29 | 30 | 31 | .. option:: 32 | 33 | Remote URL for which to show the logged-in user. Expects URLs like 34 | the remote source/destination URLs used by the `nextstrain remote` 35 | family of commands. Only the domain name (technically, the origin) 36 | of the URL is required/used, but a full URL may be specified. 37 | 38 | options 39 | ======= 40 | 41 | 42 | 43 | .. option:: -h, --help 44 | 45 | show this help message and exit 46 | 47 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | import re 18 | 19 | 20 | # -- Project information ----------------------------------------------------- 21 | 22 | from datetime import date 23 | from nextstrain.cli import __version__ as cli_version 24 | 25 | project = 'Nextstrain CLI' 26 | version = cli_version 27 | release = version 28 | copyright = '2018–%d, Trevor Bedford and Richard Neher' % (date.today().year) 29 | author = 'Thomas Sibley and the rest of the Nextstrain team' 30 | 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 36 | # ones. 37 | extensions = [ 38 | 'myst_parser', 39 | 'sphinx.ext.autodoc', 40 | 'sphinx.ext.intersphinx', 41 | 'sphinx_markdown_tables', 42 | 'nextstrain.sphinx.theme', 43 | ] 44 | 45 | # Add any paths that contain templates here, relative to this directory. 46 | templates_path = ['_templates'] 47 | 48 | # List of patterns, relative to source directory, that match files and 49 | # directories to ignore when looking for source files. 50 | # This pattern also affects html_static_path and html_extra_path. 51 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 52 | 'development.md', 53 | ] 54 | 55 | 56 | # -- Options for MyST-Parser ------------------------------------------------- 57 | 58 | # Generate Markdown-only link targets for Markdown headings up to
. These 59 | # are NOT used for rST cross-references or implicit section targets. 60 | # 61 | # You can see the effect of changing this by running, e.g.: 62 | # 63 | # myst-anchors --level 1 CHANGES.md 64 | # 65 | # See also . 66 | myst_heading_anchors = 5 67 | 68 | 69 | # -- Options for HTML output ------------------------------------------------- 70 | 71 | # The theme to use for HTML and HTML Help pages. See the documentation for 72 | # a list of builtin themes. 73 | # 74 | html_theme = 'nextstrain-sphinx-theme' 75 | 76 | # Add any paths that contain custom static files (such as style sheets) here, 77 | # relative to this directory. They are copied after the builtin static files, 78 | # so a file named "default.css" will overwrite the builtin "default.css". 79 | html_static_path = ['_static'] 80 | 81 | 82 | # -- Cross-project references ------------------------------------------------ 83 | 84 | intersphinx_mapping = { 85 | 'augur': ('https://docs.nextstrain.org/projects/augur/en/stable', None), 86 | 'auspice': ('https://docs.nextstrain.org/projects/auspice/en/stable', None), 87 | 'docs': ('https://docs.nextstrain.org/en/latest/', None), 88 | } 89 | 90 | 91 | # -- Linkchecking ------------------------------------------------------------ 92 | 93 | ## NOTE: for both sets of regular expressions that follow, the 94 | ## underlying linkchecker code uses `re.match()` to apply them to URLs 95 | ## — so there's already an implicit "only at the beginning of a 96 | ## string" matching happening, and something like a plain `r'google'` 97 | ## regular expression will _NOT_ match all google.com URLs. 98 | linkcheck_ignore = [ 99 | # Fixed-string prefixes 100 | *map(re.escape, [ 101 | # we have links to localhost for explanatory purposes; obviously 102 | # they will never work in the linkchecker 103 | 'http://127.0.0.1:', 104 | 'http://localhost:', 105 | 106 | # Cloudflare "protection" gets in the way with a 403 107 | 'https://conda.anaconda.org', 108 | 109 | # Can't easily check __NEXT__ links that might not exist yet (at least 110 | # outside of the PR preview build). 111 | # 112 | # XXX TODO: An improvement in the future would be munging them to the 113 | # PR preview build. This would helpfully catch mistakes! But we'd 114 | # need to coordinate CI jobs and figure out how/where to do the 115 | # munging. We could also munge them to a local doc build in the 116 | # linkcheck job, I guess. In any case, ENOTIME right now. 117 | # -trs, 29 May 2025 118 | 'https://docs.nextstrain.org/projects/cli/en/__NEXT__/', 119 | ]), 120 | ] 121 | linkcheck_anchors_ignore_for_url = [ 122 | # Fixed-string prefixes 123 | *map(re.escape, [ 124 | # Github uses anchor-looking links for highlighting lines but 125 | # handles the actual resolution with Javascript, so skip anchor 126 | # checks for Github URLs: 127 | 'https://github.com', 128 | 'https://console.aws.amazon.com/batch/home', 129 | 'https://console.aws.amazon.com/ec2/v2/home', 130 | ]), 131 | ] 132 | -------------------------------------------------------------------------------- /doc/config/file.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Config file 3 | =========== 4 | 5 | Nextstrain CLI uses an INI-style configuration file to store information about 6 | the runtimes that are set up. For example: 7 | 8 | .. code-block:: ini 9 | 10 | [core] 11 | runner = docker 12 | 13 | [docker] 14 | image = nextstrain/base:build-20230623T174208Z 15 | 16 | The default configuration file is :file:`~/.nextstrain/config`. This path may 17 | be overridden entirely by the :envvar:`NEXTSTRAIN_CONFIG` environment variable. 18 | Alternatively, the path of the containing directory (i.e. 19 | :file:`~/.nextstrain/`) may be overridden by the :envvar:`NEXTSTRAIN_HOME` 20 | environment variable. 21 | 22 | 23 | Sections 24 | ======== 25 | 26 | - `Core variables`_ 27 | - :ref:`Docker runtime variables ` 28 | - :ref:`Singularity runtime variables ` 29 | - :ref:`AWS Batch runtime variables ` 30 | 31 | 32 | Core variables 33 | ============== 34 | 35 | .. glossary:: 36 | 37 | :index:`core.runner ` 38 | Short name of the default :term:`runtime`. Typically set by running 39 | one of: 40 | 41 | .. code-block:: none 42 | 43 | nextstrain setup --set-default 44 | nextstrain check-setup --set-default 45 | 46 | If not set, the :doc:`/runtimes/docker` (``docker``) is used. 47 | -------------------------------------------------------------------------------- /doc/config/paths.rst: -------------------------------------------------------------------------------- 1 | =================== 2 | Paths configuration 3 | =================== 4 | 5 | Nextstrain CLI uses various local filesystem paths for config and runtime data. 6 | If necessary, the defaults can be overridden by environment variables. 7 | 8 | .. envvar:: NEXTSTRAIN_HOME 9 | 10 | Directory for config and other application data. Used as the basis for all 11 | other paths. 12 | 13 | Default is :file:`~/.nextstrain/`, assuming a home directory is 14 | discernable. If not, :file:`.nextstrain` (i.e. in the current directory) 15 | is used as a last resort. 16 | 17 | .. envvar:: NEXTSTRAIN_CONFIG 18 | 19 | File for :doc:`configuration `. 20 | 21 | Default is :file:`{${NEXTSTRAIN_HOME}}/config`. 22 | 23 | .. envvar:: NEXTSTRAIN_SECRETS 24 | 25 | File for secrets (e.g. nextstrain.org tokens) managed by 26 | :doc:`/commands/login` and :doc:`/commands/logout`. 27 | 28 | Default is :file:`{${NEXTSTRAIN_HOME}}/secrets`. 29 | 30 | .. envvar:: NEXTSTRAIN_LOCK 31 | 32 | File for serializing access to other config files to prevent corruption and 33 | other bugs. 34 | 35 | Default is :file:`{${NEXTSTRAIN_HOME}}/lock`. 36 | 37 | .. envvar:: NEXTSTRAIN_PATHOGENS 38 | 39 | Directory for pathogen workflow data managed by :doc:`/commands/setup`, 40 | e.g. local copies of pathogen repos like `nextstrain/measles 41 | `__. 42 | 43 | Default is :file:`{${NEXTSTRAIN_HOME}}/pathogens/`. 44 | 45 | .. envvar:: NEXTSTRAIN_RUNTIMES 46 | 47 | Directory for runtime-specific data, e.g. Singularity images or a Conda 48 | environment. Each runtime uses a subdirectory within here. 49 | 50 | Default is :file:`{${NEXTSTRAIN_HOME}}/runtimes/`. 51 | 52 | .. envvar:: NEXTSTRAIN_SHELL_HISTORY 53 | 54 | File for preserving command history across :doc:`/commands/shell` invocations. 55 | 56 | Default is :file:`{${NEXTSTRAIN_HOME}}/shell-history`. 57 | -------------------------------------------------------------------------------- /doc/glossary.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Glossary 3 | ======== 4 | 5 | Terms used by Nextstrain CLI. Primarily intended for developers new to the 6 | codebase, though some terms are also used in user-facing messages and 7 | documentation. 8 | 9 | .. glossary:: 10 | 11 | computing environment 12 | A general term for any given set of software, configuration, and other 13 | resources available for running programs. Computing environments are 14 | often nested, with inner environments inheriting to varying degrees 15 | from the outer environments. The isolation and reproducibility of 16 | different computing environments varies widely. 17 | 18 | :term:`Nextstrain runtimes ` are examples of specific 19 | computing environments. 20 | 21 | computing platform 22 | The foundation of a :term:`computing environment` (or part of it), such 23 | as Docker, Conda, Singularity, AWS Batch, etc. 24 | 25 | runner 26 | The code (i.e. Python module, e.g. :file:`nextstrain/cli/runner/docker.py`) 27 | which arranges to execute things inside a :term:`runtime`. 28 | 29 | Used by commands like ``nextstrain build`` and ``nextstrain view``, for 30 | example, to execute ``snakemake`` or ``auspice`` (respectively). 31 | 32 | Runners have a 1:1 mapping to runtimes. 33 | 34 | runtime 35 | A specific :term:`computing environment` (e.g. container image or Conda 36 | environment) in which Nextstrain CLI expects to find and execute other 37 | Nextstrain programs. 38 | 39 | See the :doc:`runtimes overview `. 40 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | Nextstrain CLI 3 | ============== 4 | 5 | .. hint:: 6 | This is reference documentation for the Nextstrain CLI (command-line 7 | interface). If you're just getting started with Nextstrain, please see 8 | :doc:`our general documentation ` instead. 9 | 10 | The Nextstrain CLI is a program called ``nextstrain``. It aims to provide a 11 | consistent way to run and visualize pathogen builds and access Nextstrain 12 | components like :doc:`Augur ` and :doc:`Auspice ` 13 | computing platforms, such as 14 | :doc:`/runtimes/docker`, 15 | :doc:`/runtimes/conda`, 16 | :doc:`/runtimes/singularity`, and 17 | :doc:`/runtimes/aws-batch`. 18 | 19 | .. note:: 20 | If you're unfamiliar with Nextstrain builds, you may want to follow our 21 | :doc:`docs:tutorials/running-a-phylogenetic-workflow` tutorial first and 22 | then come back here. 23 | 24 | 25 | Table of Contents 26 | ================= 27 | 28 | .. toctree:: 29 | :titlesonly: 30 | 31 | Introduction 32 | installation 33 | upgrading 34 | changes 35 | AWS Batch 36 | 37 | .. toctree:: 38 | :caption: Commands 39 | :name: commands 40 | :titlesonly: 41 | :maxdepth: 3 42 | 43 | commands/run 44 | commands/build 45 | commands/view 46 | commands/deploy 47 | commands/remote/index 48 | commands/remote/list 49 | commands/remote/download 50 | commands/remote/upload 51 | commands/remote/delete 52 | commands/shell 53 | commands/update 54 | commands/setup 55 | commands/check-setup 56 | commands/login 57 | commands/logout 58 | commands/whoami 59 | commands/authorization 60 | commands/version 61 | 62 | .. toctree:: 63 | :caption: Remotes 64 | :titlesonly: 65 | 66 | nextstrain.org 67 | Amazon S3 68 | 69 | .. toctree:: 70 | :caption: Runtimes 71 | :titlesonly: 72 | 73 | Overview 74 | Docker 75 | Conda 76 | Singularity 77 | AWS Batch 78 | Ambient 79 | 80 | .. toctree:: 81 | :caption: Configuration 82 | :titlesonly: 83 | 84 | Config file 85 | Paths 86 | 87 | .. toctree:: 88 | :caption: Development 89 | :titlesonly: 90 | 91 | glossary 92 | 93 | 94 | Big Picture 95 | =========== 96 | 97 | .. XXX TODO: Move this into our explanatory doc pages when they're written. 98 | 99 | The Nextstrain CLI glues together many different components. Below is a brief 100 | overview of the `big picture <_static/big-picture.svg>`__: 101 | 102 | .. image:: _static/big-picture.svg 103 | :target: _static/big-picture.svg 104 | 105 | 106 | Indices and Tables 107 | ================== 108 | 109 | * :ref:`genindex` 110 | * :ref:`search` 111 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | if "%BUILDDIR%" == "" ( 11 | set BUILDDIR=_build 12 | ) 13 | set SOURCEDIR=. 14 | 15 | if "%1" == "" goto help 16 | 17 | %SPHINXBUILD% >NUL 2>NUL 18 | if errorlevel 9009 ( 19 | echo. 20 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 21 | echo.installed, then set the SPHINXBUILD environment variable to point 22 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 23 | echo.may add the Sphinx directory to PATH. 24 | echo. 25 | echo.If you don't have Sphinx installed, grab it from 26 | echo.http://sphinx-doc.org/ 27 | exit /b 1 28 | ) 29 | 30 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 31 | goto end 32 | 33 | :help 34 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 35 | 36 | :end 37 | popd 38 | -------------------------------------------------------------------------------- /doc/redirects.yaml: -------------------------------------------------------------------------------- 1 | # Authoritative list of redirects we have configured in RTD for this project. 2 | --- 3 | - type: page 4 | from_url: /development/ 5 | to_url: https://docs.nextstrain.org/en/latest/guides/contribute/cli-develop.html 6 | 7 | - type: page 8 | from_url: /development/index.html 9 | to_url: https://docs.nextstrain.org/en/latest/guides/contribute/cli-develop.html 10 | 11 | - type: page 12 | from_url: /commands/ 13 | to_url: /#commands 14 | 15 | - type: page 16 | from_url: /commands/index.html 17 | to_url: /index.html#commands 18 | -------------------------------------------------------------------------------- /doc/remotes/nextstrain.org.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | nextstrain.org remote 3 | ===================== 4 | 5 | .. automodule:: nextstrain.cli.remote.nextstrain_dot_org 6 | :noindex: 7 | -------------------------------------------------------------------------------- /doc/remotes/s3.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Amazon S3 remote 3 | ================ 4 | 5 | .. automodule:: nextstrain.cli.remote.s3 6 | :noindex: 7 | -------------------------------------------------------------------------------- /doc/runtimes/ambient.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Ambient runtime 3 | =============== 4 | 5 | .. automodule:: nextstrain.cli.runner.ambient 6 | :noindex: 7 | -------------------------------------------------------------------------------- /doc/runtimes/aws-batch.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | AWS Batch runtime 3 | ================= 4 | 5 | .. automodule:: nextstrain.cli.runner.aws_batch 6 | :noindex: 7 | -------------------------------------------------------------------------------- /doc/runtimes/comparison.csv: -------------------------------------------------------------------------------- 1 | ,Isolation level,Containerized?,Locality,External dependencies 2 | :doc:`/runtimes/docker`,great (3),yes,local or remote,``docker`` command 3 | :doc:`/runtimes/conda`,some (1),no,local,none 4 | :doc:`/runtimes/singularity`,good (2),yes,local,``singularity`` command 5 | :doc:`/runtimes/ambient`,none (0),no,local,many 6 | :doc:`/runtimes/aws-batch`,great (3),yes,remote,AWS account with Batch set up 7 | -------------------------------------------------------------------------------- /doc/runtimes/conda.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Conda runtime 3 | ============= 4 | 5 | .. automodule:: nextstrain.cli.runner.conda 6 | :noindex: 7 | -------------------------------------------------------------------------------- /doc/runtimes/docker.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | Docker runtime 3 | ============== 4 | 5 | .. automodule:: nextstrain.cli.runner.docker 6 | :noindex: 7 | -------------------------------------------------------------------------------- /doc/runtimes/index.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Runtimes 3 | ======== 4 | 5 | Nextstrain's runtimes are specific :term:`computing environments ` in which Nextstrain CLI expects to find and run other Nextstrain 7 | programs, like :doc:`Augur ` and :doc:`Auspice `. 8 | In turn, Nextstrain CLI provides a consistent set of commands to run and 9 | visualize Nextstrain pathogen builds regardless of the underlying runtime in 10 | use. Together, this allows Nextstrain to be used across many different 11 | computing platforms and operating systems. 12 | 13 | The :doc:`/commands/build`, :doc:`/commands/view`, and :doc:`/commands/shell` 14 | commands all require a runtime, as they require access to other Nextstrain 15 | software. 16 | 17 | The :doc:`/commands/setup`, :doc:`/commands/check-setup`, and 18 | :doc:`/commands/update` commands manage the runtimes available for use by the 19 | commands above. The :doc:`/commands/version` command's :option:`--verbose 20 | ` option reports detailed version information 21 | about all available runtimes. 22 | 23 | Other Nextstrain CLI commands, such as the :doc:`/commands/remote/index` family 24 | of commands and the related :doc:`/commands/login` and :doc:`/commands/logout` 25 | commands, do not require a runtime. They may be used without any prior set up 26 | of a runtime. 27 | 28 | The runtimes currently available are the: 29 | 30 | - :doc:`/runtimes/docker` 31 | - :doc:`/runtimes/singularity` 32 | - :doc:`/runtimes/conda` 33 | - :doc:`/runtimes/ambient` 34 | - :doc:`/runtimes/aws-batch` 35 | 36 | Runtimes are managed (maintained, tested, versioned, released) by the 37 | Nextstrain team, except for the ambient runtime. The ambient runtime is 38 | special in that it's whatever computing environment in which Nextstrain CLI 39 | itself is running (that is, it's managed by the user). 40 | 41 | You can set up and use multiple runtimes on the same computer, for example if 42 | you want to use them in different contexts. Runtime-using commands let you 43 | select a different runtime than your default with command-line options (e.g. 44 | ``--docker``, ``--conda``, and so on). For example, you might use the Docker 45 | runtime to run small builds locally and then use the AWS Batch runtime to run 46 | large scale builds with more computing power. 47 | 48 | If you pick one runtime and later realize you want to switch, you can go back 49 | and set up the other. 50 | 51 | 52 | Comparison 53 | ========== 54 | 55 | .. csv-table:: 56 | :file: comparison.csv 57 | :header-rows: 1 58 | :stub-columns: 1 59 | 60 | Isolation level 61 | A relative ranking from least isolated (*none*, 0) to most isolated 62 | (*great*, 3) from the underlying computer system. 63 | 64 | Containerized? 65 | A containerized :term:`computing platform` provides a higher degree of 66 | isolation, which in turn usually means a higher degree of portabililty and 67 | reproducibility across different computers and users. 68 | 69 | Locality 70 | *Local* means "on the same computer where ``nextstrain`` is being run". 71 | *Remote* means "on a different computer". 72 | 73 | Docker is most often used to run containers locally, but can also be used 74 | to run them remotely. 75 | 76 | External dependencies 77 | Third-party programs or configuration which are required to use a runtime 78 | but not managed by :doc:`/commands/setup` and :doc:`/commands/update`. 79 | 80 | 81 | Compatibility 82 | ============= 83 | 84 | Switching runtimes or updating the version of a runtime may result in different 85 | versions of Nextstrain components like Augur and Auspice as well as other 86 | programs, and thus different behaviour. Use the :doc:`/commands/version` 87 | command's :option:`--verbose ` option to report 88 | detailed version information about all available runtimes. 89 | 90 | Exact behavioural compatibility is not guaranteed between different runtimes 91 | (e.g. between the Docker vs. Conda runtimes) or between versions of the same 92 | runtime (e.g. between Docker runtime images 93 | ``nextstrain/base:build-20230714T205715Z`` and 94 | ``nextstrain/base:build-20230720T001758Z``). However, the containerized 95 | runtimes (Docker, Singularity, AWS Batch; see comparison_ above) will usually 96 | have identical behaviour given the same runtime version (e.g. ``build-…``) as 97 | they are all based the same runtime image (``nextstrain/base``). Any variance 98 | is typically due to use of external resources (intentional or otherwise) from 99 | outside the container. 100 | -------------------------------------------------------------------------------- /doc/runtimes/singularity.rst: -------------------------------------------------------------------------------- 1 | =================== 2 | Singularity runtime 3 | =================== 4 | 5 | .. automodule:: nextstrain.cli.runner.singularity 6 | :noindex: 7 | -------------------------------------------------------------------------------- /doc/upgrading.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Upgrading 3 | ========= 4 | 5 | This page describes how to upgrade the Nextstrain CLI--the ``nextstrain`` 6 | command--itself, without also upgrading the entire Nextstrain :term:`runtime`. 7 | (To upgrade your Nextstrain runtime, use the :doc:`/commands/update` command.) 8 | 9 | The way to upgrade depends on what kind of Nextstrain CLI installation you have 10 | (i.e. how it was first installed), so running ``nextstrain check-setup``: 11 | 12 | .. code-block:: console 13 | :emphasize-lines: 10 14 | 15 | $ nextstrain check-setup 16 | A new version of nextstrain-cli, X.Y.Z, is available! You're running A.B.C. 17 | 18 | See what's new in the changelog: 19 | 20 | https://github.com/nextstrain/cli/blob/X.Y.Z/CHANGES.md#readme 21 | 22 | Upgrade by running: 23 | 24 | [UPGRADE COMMAND] 25 | 26 | Testing your setup… 27 | … 28 | 29 | will suggest a command to run to upgrade ``nextstrain``, if there are any 30 | upgrades available. Run the suggested command to perform the upgrade. 31 | 32 | Afterwards, you can check that the new version was installed by running: 33 | 34 | .. code-block:: console 35 | 36 | $ nextstrain version 37 | nextstrain.cli X.Y.Z 38 | -------------------------------------------------------------------------------- /nextstrain/cli/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Nextstrain command-line interface (CLI) 3 | 4 | The `nextstrain` program and its subcommands aim to provide a consistent way to 5 | run and visualize pathogen builds and access Nextstrain components like Augur 6 | and Auspice across computing platforms such as Docker, Conda, Singularity, and 7 | AWS Batch. 8 | 9 | Run `nextstrain --help` for usage information about each command. 10 | See <:doc:`/index`> for more documentation. 11 | """ 12 | 13 | 14 | import sys 15 | import traceback 16 | from argparse import ArgumentParser, Action, SUPPRESS 17 | from textwrap import dedent 18 | 19 | from .argparse import HelpFormatter, register_commands, register_default_command 20 | from .command import all_commands 21 | from .debug import DEBUGGING 22 | from .errors import NextstrainCliError, UsageError 23 | from .util import warn 24 | from .__version__ import __version__ # noqa: F401 (for re-export) 25 | 26 | 27 | def run(args): 28 | """ 29 | Command-line entrypoint to the nextstrain-cli package, called by the 30 | `nextstrain` program. 31 | """ 32 | parser = make_parser() 33 | opts = parser.parse_args(args) 34 | 35 | try: 36 | return opts.__command__.run(opts) 37 | 38 | except NextstrainCliError as error: 39 | exit_status = 1 40 | 41 | if DEBUGGING: 42 | traceback.print_exc() 43 | else: 44 | if isinstance(error, UsageError): 45 | warn(opts.__parser__.format_usage()) 46 | exit_status = 2 47 | 48 | warn(error) 49 | 50 | return exit_status 51 | 52 | except AssertionError: 53 | traceback.print_exc() 54 | warn("\n") 55 | warn(dedent("""\ 56 | An error occurred (see above) that likely indicates a bug in the 57 | Nextstrain CLI. 58 | 59 | To report this, please open a new issue and include the error above: 60 | 61 | """)) 62 | return 1 63 | 64 | 65 | def make_parser(): 66 | parser = ArgumentParser( 67 | prog = "nextstrain", 68 | description = __doc__, 69 | formatter_class = HelpFormatter, 70 | ) 71 | 72 | register_default_command(parser) 73 | register_commands(parser, all_commands) 74 | register_version_alias(parser) 75 | 76 | return parser 77 | 78 | 79 | def register_version_alias(parser): 80 | """ 81 | Add --version as a (hidden) alias for the version command. 82 | 83 | It's not uncommon to blindly run a command with --version as the sole 84 | argument, so its useful to make that Just Work. 85 | """ 86 | 87 | class run_version_command(Action): 88 | def __call__(self, *args, **kwargs): 89 | # Go thru parse_args() rather than creating an opts Namespace 90 | # ourselves and passing it directly to version.run() so that the 91 | # version command's options pick up their normal defaults. 92 | opts = parser.parse_args(["version"]) 93 | sys.exit( opts.__command__.run(opts) ) 94 | 95 | parser.add_argument( 96 | "--version", 97 | nargs = 0, 98 | help = SUPPRESS, 99 | action = run_version_command) 100 | -------------------------------------------------------------------------------- /nextstrain/cli/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Stub function and module used as a setuptools entry point. 3 | """ 4 | 5 | import sys 6 | from io import TextIOWrapper 7 | from sys import argv, exit 8 | from nextstrain import cli 9 | 10 | # Entry point for setuptools-installed script. 11 | def main(): 12 | # Explicitly configure our stdio output streams to be as assumed in the 13 | # rest of the codebase. Avoids needing to instruct folks to set 14 | # PYTHONIOENCODING=UTF-8 or use Python's UTF-8 mode (-X utf8 or 15 | # PYTHONUTF8=1). 16 | reconfigure_stdio(sys.stdout) # pyright: ignore[reportArgumentType] 17 | reconfigure_stdio(sys.stderr) # pyright: ignore[reportArgumentType] 18 | 19 | return cli.run( argv[1:] ) 20 | 21 | 22 | def reconfigure_stdio(stdio: TextIOWrapper): 23 | """ 24 | Reconfigure *stdio* to match the assumptions of this codebase. 25 | 26 | Suitable only for output streams (e.g. stdout, stderr), as reconfiguring an 27 | input stream is more complicated. 28 | """ 29 | 30 | # Configure new text stream on the same underlying buffered byte stream. 31 | stdio.reconfigure( 32 | # Always use UTF-8 and be more lenient on stderr so even mangled error 33 | # messages make it out. 34 | encoding = "UTF-8", 35 | errors = "backslashreplace" if stdio is sys.stderr else "strict", 36 | 37 | # Explicitly enable universal newlines mode so we do the right thing. 38 | newline = None) 39 | 40 | 41 | # Run when called as `python -m nextstrain.cli`, here for good measure. 42 | if __name__ == "__main__": 43 | exit( main() ) 44 | -------------------------------------------------------------------------------- /nextstrain/cli/__version__.py: -------------------------------------------------------------------------------- 1 | __version__ = '10.2.0+git' 2 | -------------------------------------------------------------------------------- /nextstrain/cli/authn/configuration.py: -------------------------------------------------------------------------------- 1 | """ 2 | Authentication configuration. 3 | """ 4 | from functools import lru_cache 5 | from .. import requests 6 | from ..errors import UserError 7 | from ..url import Origin 8 | 9 | 10 | @lru_cache(maxsize = None) 11 | def openid_configuration(origin: Origin): 12 | """ 13 | Fetch the OpenID provider configuration/metadata for *origin*. 14 | 15 | While this information is typically served by an OP (OpenID provider), aka 16 | IdP, here we expect *origin* to be a nextstrain.org-like RP (relying 17 | party), aka SP (service provider), which is passing along its own IdP/OP's 18 | configuration for us to discover. 19 | """ 20 | assert origin 21 | 22 | with requests.Session() as http: 23 | try: 24 | response = http.get(origin.rstrip("/") + "/.well-known/openid-configuration") 25 | response.raise_for_status() 26 | return response.json() 27 | 28 | except requests.exceptions.ConnectionError as err: 29 | raise UserError(f""" 30 | Could not connect to {origin} to retrieve 31 | authentication metadata: 32 | 33 | {type(err).__name__}: {err} 34 | 35 | That remote may be invalid or you may be experiencing network 36 | connectivity issues. 37 | """) from err 38 | 39 | except (requests.exceptions.HTTPError, requests.exceptions.JSONDecodeError) as err: 40 | raise UserError(f""" 41 | Failed to retrieve authentication metadata for {origin}: 42 | 43 | {type(err).__name__}: {err} 44 | 45 | That remote seems unlikely to be an alternate nextstrain.org 46 | instance or an internal Nextstrain Groups Server instance. 47 | """) from err 48 | 49 | 50 | def client_configuration(origin: Origin): 51 | """ 52 | OpenID client configuration/metadata for *origin*. 53 | 54 | The OpenID provider configuration of a nextstrain.org-like remote includes 55 | client configuration for Nextstrain CLI. This details the OpenID client 56 | that's registered with the corresponding provider for Nextstrain CLI's use. 57 | """ 58 | assert origin 59 | 60 | config = openid_configuration(origin) 61 | 62 | if "nextstrain_cli_client_configuration" not in config: 63 | raise UserError(f""" 64 | Authentication metadata for {origin} 65 | does not contain required client information for Nextstrain CLI. 66 | 67 | That remote seems unlikely to be an alternate nextstrain.org 68 | instance or an internal Nextstrain Groups Server instance. 69 | """) 70 | 71 | return config["nextstrain_cli_client_configuration"] 72 | -------------------------------------------------------------------------------- /nextstrain/cli/authn/errors.py: -------------------------------------------------------------------------------- 1 | """ 2 | Authentication errors. 3 | """ 4 | from ..aws.cognito.srp import NewPasswordRequiredError # noqa: F401 (NewPasswordRequiredError is for re-export) 5 | 6 | class IdPError(Exception): 7 | """Error from IdP during authentication.""" 8 | pass 9 | 10 | class NotAuthorizedError(IdPError): 11 | """Not Authorized response during authentication.""" 12 | pass 13 | 14 | class TokenError(Exception): 15 | """Error when verifying tokens.""" 16 | pass 17 | 18 | class MissingTokenError(TokenError): 19 | """ 20 | No token provided but one is required. 21 | 22 | Context is the kind of token ("use") that was missing. 23 | """ 24 | pass 25 | 26 | class ExpiredTokenError(TokenError): 27 | """ 28 | Token is expired. 29 | 30 | Context is the kind of token ("use") that was missing. 31 | """ 32 | pass 33 | 34 | class InvalidUseError(TokenError): 35 | """ 36 | The "use" of the token does not match the expected value. 37 | 38 | May indicate an accidental token swap. 39 | """ 40 | pass 41 | -------------------------------------------------------------------------------- /nextstrain/cli/aws/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | boto3 client handling for AWS services. 3 | """ 4 | 5 | import boto3 6 | from botocore.exceptions import NoRegionError 7 | 8 | 9 | DEFAULT_REGION = "us-east-1" 10 | 11 | 12 | def client_with_default_region(service, default_region = DEFAULT_REGION): 13 | """ 14 | Return a boto3 client for the named *service* in the *default_region* if no 15 | region is specified in the default session (via the environment, AWS 16 | config, or ``boto3.setup_default_session``). 17 | """ 18 | try: 19 | return boto3.client(service) 20 | except NoRegionError: 21 | return boto3.client(service, region_name = default_region) 22 | -------------------------------------------------------------------------------- /nextstrain/cli/aws/cognito/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nextstrain/cli/b5af91bbe581a54730d8d4873db8d817633c4f89/nextstrain/cli/aws/cognito/__init__.py -------------------------------------------------------------------------------- /nextstrain/cli/browser.py: -------------------------------------------------------------------------------- 1 | """ 2 | Web browser interaction. 3 | 4 | .. envvar:: BROWSER 5 | 6 | A ``PATH``-like list of web browsers to try in preference order, before 7 | falling back to a set of default browsers. May be program names, e.g. 8 | ``firefox``, or absolute paths to specific executables, e.g. 9 | ``/usr/bin/firefox``. 10 | 11 | .. envvar:: NOBROWSER 12 | 13 | If set to a truthy value (e.g. 1) then no web browser will be considered 14 | available. This can be useful to prevent opening of a browser when there 15 | are not other means of doing so. 16 | """ 17 | import webbrowser 18 | from threading import Thread, ThreadError 19 | from os import environ 20 | from typing import Union 21 | from .url import URL 22 | from .util import warn 23 | 24 | 25 | if environ.get("NOBROWSER"): 26 | BROWSER = None 27 | else: 28 | # Avoid text-mode browsers 29 | TERM = environ.pop("TERM", None) 30 | try: 31 | BROWSER = webbrowser.get() 32 | except: 33 | BROWSER = None 34 | finally: 35 | if TERM is not None: 36 | environ["TERM"] = TERM 37 | 38 | 39 | def open_browser(url: Union[str, URL], new_thread: bool = True): 40 | """ 41 | Opens *url* in a web browser. 42 | 43 | Opens in a new tab, if possible, and raises the window to the top, if 44 | possible. 45 | 46 | Launches the browser from a separate thread by default so waiting on the 47 | browser child process doesn't block the main (or calling) thread. Set 48 | *new_thread* to False to launch from the same thread as the caller (e.g. if 49 | you've already spawned a dedicated thread or process for the browser). 50 | Note that some registered browsers launch in the background themselves, but 51 | not all do, so this feature makes launch behaviour consistent across 52 | browsers. 53 | 54 | Prints a warning to stderr if a browser can't be found or can't be 55 | launched, as automatically opening a browser is considered a 56 | nice-but-not-necessary feature. 57 | """ 58 | if not BROWSER: 59 | warn(f"Couldn't open <{url}> in browser: no browser found") 60 | return 61 | 62 | try: 63 | if new_thread: 64 | Thread(target = open_browser, args = (str(url), False), daemon = True).start() 65 | else: 66 | # new = 2 means new tab, if possible 67 | BROWSER.open(str(url), new = 2, autoraise = True) 68 | except (ThreadError, webbrowser.Error) as err: 69 | warn(f"Couldn't open <{url}> in browser: {err!r}") 70 | -------------------------------------------------------------------------------- /nextstrain/cli/command/__init__.py: -------------------------------------------------------------------------------- 1 | from . import ( 2 | run, 3 | build, 4 | view, 5 | deploy, 6 | remote, 7 | shell, 8 | update, 9 | setup, 10 | check_setup, 11 | login, 12 | logout, 13 | whoami, 14 | version, 15 | init_shell, 16 | authorization, 17 | debugger, 18 | ) 19 | 20 | # Maintain this list manually for now while its relatively static. If we need 21 | # to support pluggable commands or command discovery, we can switch to using 22 | # the "entry points" system: 23 | # https://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins 24 | # 25 | # The order of this list is important and intentional: it determines the order 26 | # in various user interfaces, e.g. `nextstrain --help`. 27 | # 28 | all_commands = [ 29 | run, 30 | build, 31 | view, 32 | deploy, 33 | remote, 34 | shell, 35 | update, 36 | setup, 37 | check_setup, 38 | login, 39 | logout, 40 | whoami, 41 | version, 42 | init_shell, 43 | authorization, 44 | debugger, 45 | ] 46 | -------------------------------------------------------------------------------- /nextstrain/cli/command/authorization.py: -------------------------------------------------------------------------------- 1 | """ 2 | Produce an Authorization header appropriate for the web API of nextstrain.org 3 | (and other remotes). 4 | 5 | This is a development tool unnecessary for normal usage. It's useful for 6 | directly making API requests to nextstrain.org (and other remotes) with `curl` 7 | or similar commands. For example:: 8 | 9 | curl -si https://nextstrain.org/whoami \\ 10 | --header "Accept: application/json" \\ 11 | --header @<(nextstrain authorization) 12 | 13 | Exits with an error if no one is logged in. 14 | """ 15 | from inspect import cleandoc 16 | from ..errors import UserError 17 | from ..remote import parse_remote_path 18 | 19 | 20 | def register_parser(subparser): 21 | parser = subparser.add_parser("authorization", help = "Print an HTTP Authorization header") 22 | 23 | parser.add_argument( 24 | "remote", 25 | help = cleandoc(""" 26 | Remote URL for which to produce an Authorization header. Expects 27 | URLs like the remote source/destination URLs used by the 28 | `nextstrain remote` family of commands. Only the domain name 29 | (technically, the origin) of the URL is required/used, but a full 30 | URL may be specified. 31 | """), 32 | metavar = "", 33 | nargs = "?", 34 | default = "nextstrain.org") 35 | 36 | return parser 37 | 38 | 39 | def run(opts): 40 | remote, url = parse_remote_path(opts.remote) 41 | assert url.origin 42 | 43 | user = remote.current_user(url.origin) 44 | 45 | if not user: 46 | raise UserError(f"Not logged in to {url.origin}.") 47 | 48 | print(f"Authorization: {user.http_authorization}") 49 | -------------------------------------------------------------------------------- /nextstrain/cli/command/check_setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Checks for supported runtimes. 3 | 4 | Five runtimes are tested by default: 5 | 6 | • Our Docker image is the preferred runtime. Docker itself must 7 | be installed and configured on your computer first, but once it is, the 8 | runtime is robust and reproducible. 9 | 10 | • Our Conda runtime will be tested for existence and appearance of 11 | completeness. This runtime is more isolated and reproducible than your 12 | ambient runtime, but is less isolated and robust than the Docker 13 | runtime. 14 | 15 | • Our Singularity runtime uses the same container image as our Docker 16 | runtime. Singularity must be installed and configured on your computer 17 | first, although it is often already present on HPC systems. This runtime 18 | is more isolated and reproducible than the Conda runtime, but potentially 19 | less so than the Docker runtime. 20 | 21 | • Your ambient setup will be tested for snakemake, augur, and auspice. 22 | Their presence implies a working runtime, but does not guarantee 23 | it. 24 | 25 | • Remote jobs on AWS Batch. Your AWS account, if credentials are available 26 | in your environment or via aws-cli configuration, will be tested for the 27 | presence of appropriate resources. Their presence implies a working AWS 28 | Batch runtime, but does not guarantee it. 29 | 30 | Provide one or more runtime names as arguments to test just those instead. 31 | 32 | Exits with an error code if the default runtime ({default_runner_name}) is not 33 | supported or, when the default runtime is omitted from checks, if none of the 34 | checked runtimes are supported. 35 | """ 36 | 37 | from functools import partial 38 | from ..argparse import SKIP_AUTO_DEFAULT_IN_HELP, runner_module_argument 39 | from ..types import Options 40 | from ..util import colored, check_for_new_version, runner_name, print_and_check_setup_tests 41 | from ..runner import all_runners, all_runners_by_name, default_runner # noqa: F401 (it's wrong; we use it in run()) 42 | 43 | 44 | __doc__ = (__doc__ or "").format(default_runner_name = runner_name(default_runner)) 45 | 46 | 47 | # XXX TODO: Add support for checking pathogen setups too? Not sure this makes 48 | # much sense. 49 | # -trs, 3 March 2025 50 | 51 | 52 | def register_parser(subparser): 53 | """ 54 | %(prog)s [--set-default] [ [ ...]] 55 | %(prog)s --help 56 | """ 57 | parser = subparser.add_parser("check-setup", help = "Check runtime setups") 58 | 59 | parser.add_argument( 60 | "runners", 61 | help = "The Nextstrain runtimes to check. " 62 | f"(default: {', '.join(all_runners_by_name)})" 63 | f"{SKIP_AUTO_DEFAULT_IN_HELP}", 64 | metavar = "", 65 | nargs = "*", 66 | type = runner_module_argument, 67 | default = all_runners) 68 | 69 | parser.add_argument( 70 | "--set-default", 71 | help = "Set the default runtime to the first which passes check-setup. " 72 | "Checks run in the order given, if any, " 73 | "otherwise in the default order: %s." % (", ".join(all_runners_by_name),), 74 | action = "store_true") 75 | 76 | return parser 77 | 78 | 79 | def run(opts: Options) -> int: 80 | global default_runner 81 | 82 | success = partial(colored, "green") 83 | failure = partial(colored, "red") 84 | 85 | # Check our own version for updates 86 | check_for_new_version() 87 | 88 | # Run and collect our runners' self-tests 89 | print("Testing your setup…") 90 | 91 | runner_tests = ( 92 | (runner, runner.test_setup()) 93 | for runner in opts.runners 94 | ) 95 | 96 | supported_runners = [] 97 | 98 | # Print test results. The first print() separates results from the 99 | # previous header or stderr output, making it easier to read. 100 | print() 101 | 102 | for runner, tests in runner_tests: 103 | print(colored("blue", "#"), "Checking %s…" % (runner_name(runner))) 104 | 105 | ok = print_and_check_setup_tests(tests) 106 | 107 | if ok: 108 | supported = success("supported") 109 | supported_runners.append(runner) 110 | else: 111 | supported = failure("not supported") 112 | 113 | print(colored("blue", "#"), "%s is %s" % (runner_name(runner), supported)) 114 | print() 115 | 116 | # Print overall status. 117 | if supported_runners: 118 | print("Supported Nextstrain runtimes:", ", ".join(success(runner_name(r)) for r in supported_runners)) 119 | 120 | if opts.set_default: 121 | default_runner = supported_runners[0] 122 | print() 123 | print("Setting default runtime to %s." % runner_name(default_runner)) 124 | default_runner.set_default_config() 125 | else: 126 | if set(opts.runners) == set(all_runners): 127 | print(failure("No support for any Nextstrain runtime.")) 128 | else: 129 | print(failure("No support for selected Nextstrain runtimes.")) 130 | 131 | print() 132 | if default_runner in supported_runners: 133 | print(f"All good! Default runtime ({runner_name(default_runner)}) is supported.") 134 | else: 135 | if default_runner in opts.runners: 136 | print(failure(f"No good. Default runtime ({runner_name(default_runner)}) is not supported.")) 137 | else: 138 | print(f"Warning: Support for the default runtime ({runner_name(default_runner)}) was not checked.") 139 | 140 | if supported_runners and not opts.set_default: 141 | print() 142 | print("Try running check-setup again with the --set-default option to pick a supported runtime above.") 143 | 144 | # Return a 1 or 0 exit code 145 | if default_runner in opts.runners: 146 | return 0 if default_runner in supported_runners else 1 147 | else: 148 | return 0 if supported_runners else 1 149 | -------------------------------------------------------------------------------- /nextstrain/cli/command/debugger.py: -------------------------------------------------------------------------------- 1 | """ 2 | Launch pdb from within the Nextstrain CLI process. 3 | 4 | This is a development and troubleshooting tool unnecessary for normal usage. 5 | """ 6 | import pdb 7 | 8 | 9 | def register_parser(subparser): 10 | parser = subparser.add_parser("debugger", help = "Start a debugger") 11 | return parser 12 | 13 | 14 | def run(opts): 15 | pdb.set_trace() 16 | -------------------------------------------------------------------------------- /nextstrain/cli/command/deploy.py: -------------------------------------------------------------------------------- 1 | # This command, `nextstrain deploy`, is now an alias for `nextstrain remote 2 | # upload`. 3 | # 4 | # Registering our own parser lets us preserve the original short description 5 | # and avoids introducing "upload" as a top-level command. 6 | 7 | from textwrap import dedent 8 | from .remote.upload import register_arguments, run, __doc__ # noqa: F401 (these are for re-export) 9 | 10 | def register_parser(subparser): 11 | parser = subparser.add_parser("deploy", help = "Deploy pathogen build") 12 | register_arguments(parser) 13 | return parser 14 | 15 | def insert_paragraph(text, index, insert): 16 | paras = text.split("\n\n") 17 | paras.insert(index, dedent(insert)) 18 | return "\n\n".join(paras) 19 | 20 | __doc__ = insert_paragraph( 21 | __doc__, 1, """ 22 | The `nextstrain deploy` command is an alias for `nextstrain remote upload`. 23 | """) 24 | -------------------------------------------------------------------------------- /nextstrain/cli/command/init_shell.py: -------------------------------------------------------------------------------- 1 | """ 2 | Prints the shell init script for a Nextstrain CLI standalone installation. 3 | 4 | If PATH does not contain the expected installation path, emits an appropriate 5 | ``export PATH=…`` statement. Otherwise, emits only a comment. 6 | 7 | Use this command in your shell config with a line like the following:: 8 | 9 | eval "$({INSTALLATION_PATH}/nextstrain init-shell)" 10 | 11 | Exits with error if run in an non-standalone installation. 12 | """ 13 | 14 | import os 15 | import shutil 16 | from pathlib import Path 17 | from shlex import quote as shquote 18 | from textwrap import dedent 19 | from typing import Optional 20 | from ..errors import UserError 21 | from ..util import standalone_installation_path 22 | 23 | 24 | try: 25 | INSTALLATION_PATH = standalone_installation_path() 26 | except: 27 | INSTALLATION_PATH = None 28 | 29 | # Guard against __doc__ being None to appease the type checkers. 30 | __doc__ = (__doc__ or "").format( 31 | INSTALLATION_PATH = ( 32 | shquote(str(INSTALLATION_PATH)) 33 | if INSTALLATION_PATH 34 | else "…/path/to" 35 | ) 36 | ) 37 | 38 | 39 | def register_parser(subparser): 40 | parser = subparser.add_parser("init-shell", help = "Print shell init script") 41 | 42 | # We don't currently need to know this as we always emit POSIX shell. 43 | # Hedge against needing it in the future, though, by accepting it now so 44 | # people won't have to update their shell rcs later. 45 | parser.add_argument( 46 | "shell", 47 | help = "Shell that's being initialized (e.g. bash, zsh, etc.); " 48 | "currently we always emit POSIX shell syntax but this may change in the future.", 49 | nargs = "?", 50 | default = "sh") 51 | 52 | return parser 53 | 54 | 55 | def run(opts): 56 | # The output of this command must remain less than the limits on command 57 | # line length. These vary by OS but also system to system. 58 | # 59 | # execve() is typically limited on the total argv + environ size to 60 | # `getconf ARG_MAX`, but there are also limits on the size of a single 61 | # argument within that total. This single argument limit applies to the 62 | # `eval "$(…)"` pattern we recommend. On Linux, for example, the limit is 63 | # hardcoded to 131071 bytes (MAX_ARG_STRLEN constant). 64 | # 65 | # The `source <(…)` pattern doesn't suffer this limitation but has other 66 | # issues on the old Bash version found on macOS. 67 | # -trs, 7 March 2023 68 | 69 | if not INSTALLATION_PATH: 70 | raise UserError("No shell init required because this is not a standalone installation.") 71 | 72 | nextstrain = which("nextstrain") 73 | 74 | if not nextstrain or nextstrain.parent != INSTALLATION_PATH: 75 | if nextstrain: 76 | print(f"# This will mask {nextstrain}") 77 | print() 78 | 79 | # This risks duplication if INSTALLATION_PATH is already in PATH but 80 | # `nextstrain` is found on an earlier PATH entry. A duplication-free 81 | # solution would be to detect that condition and *move* the existing 82 | # INSTALLATION_PATH entry in PATH to the front. This is easy on the 83 | # face of it, so I started drafting an implementation before realizing 84 | # it's full of traps and adds lots of complexity, such as: 85 | # 86 | # - Proper handling of original paths vs. resolved paths 87 | # (e.g. symlinks, etc). We'd need to match INSTALLATION_PATH on 88 | # resolved paths but remove the original paths. 89 | # 90 | # - Shell syntax for robustly filtering PATH. Alternately, we could 91 | # filter in Python and emit the static value instead of using $PATH. 92 | # Both are unpleasant for the reason above. 93 | # 94 | # These are possible, but since this would all be for little gain in 95 | # what's already a edge case, let's not sweat it after all. 96 | # -trs, 14 Sept 2022 97 | init = """ 98 | # Add %(INSTALLATION_PATH)s to front of PATH 99 | export PATH=%(INSTALLATION_PATH)s"${PATH:+%(pathsep)s}$PATH" 100 | """ 101 | else: 102 | init = """ 103 | # PATH already finds this nextstrain at %(INSTALLATION_PATH)s 104 | """ 105 | 106 | print(dedent(init.lstrip("\n")) % { 107 | "INSTALLATION_PATH": shquote(str(INSTALLATION_PATH)), 108 | "pathsep": os.pathsep 109 | }) 110 | 111 | return 0 112 | 113 | 114 | def which(cmd = "nextstrain") -> Optional[Path]: 115 | path = shutil.which(cmd) 116 | return Path(path).resolve(strict = True) if path else None 117 | -------------------------------------------------------------------------------- /nextstrain/cli/command/logout.py: -------------------------------------------------------------------------------- 1 | """ 2 | Log out of Nextstrain.org (and other remotes) by deleting locally-saved 3 | credentials. 4 | 5 | The authentication tokens are removed but not invalidated, so if you used them 6 | outside of the `nextstrain` command, they will remain valid until they expire. 7 | 8 | Other devices/clients (like your web browser) are not logged out of 9 | Nextstrain.org (or other remotes). 10 | """ 11 | from inspect import cleandoc 12 | from .. import authn 13 | from ..remote import parse_remote_path 14 | 15 | 16 | def register_parser(subparser): 17 | """ 18 | %(prog)s [] 19 | %(prog)s --all 20 | %(prog)s --help 21 | """ 22 | parser = subparser.add_parser("logout", help = "Log out of Nextstrain.org (and other remotes)") 23 | 24 | parser.add_argument( 25 | "remote", 26 | help = cleandoc(""" 27 | Remote URL to log out of, like the remote source/destination URLs 28 | used by the `nextstrain remote` family of commands. Only the 29 | domain name (technically, the origin) of the URL is required/used, 30 | but a full URL may be specified. 31 | """), 32 | metavar = "", 33 | nargs = "?", 34 | default = "nextstrain.org") 35 | 36 | parser.add_argument( 37 | "--all", 38 | help = "Log out of all remotes for which there are locally-saved credentials", 39 | action = "store_true") 40 | 41 | return parser 42 | 43 | 44 | def run(opts): 45 | if opts.all: 46 | authn.logout_all() 47 | 48 | else: 49 | remote, url = parse_remote_path(opts.remote) 50 | assert url.origin 51 | 52 | remote.logout(url.origin) 53 | -------------------------------------------------------------------------------- /nextstrain/cli/command/remote/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Upload, download, and manage remote datasets and narratives. 3 | 4 | nextstrain.org is the primary remote source/destination for most users, but 5 | Amazon S3 buckets are also supported for some internal use cases. 6 | 7 | Remote sources/destinations are specified using URLs starting with 8 | ``https://nextstrain.org/`` and ``s3:///``. nextstrain.org remote 9 | URLs represent datasets and narratives each as a whole, where datasets may 10 | consist of multiple files (the main JSON file + optional sidecar files) when 11 | uploading/downloading. Amazon S3 remote URLs represent dataset and narrative 12 | files individually. 13 | 14 | For more details on using each remote, see their respective documentation 15 | pages: 16 | 17 | * :doc:`/remotes/nextstrain.org` 18 | * :doc:`/remotes/s3` 19 | 20 | For more information on dataset (Auspice JSON) and narrative (Markdown) files, 21 | see :doc:`docs:reference/data-formats`. 22 | """ 23 | 24 | # Guard against __doc__ being None to appease the type checkers. 25 | __shortdoc__ = (__doc__ or "").strip().splitlines()[0] 26 | 27 | 28 | from . import upload, download, ls, delete 29 | 30 | 31 | def register_parser(subparser): 32 | parser = subparser.add_parser("remote", help = __shortdoc__) 33 | 34 | parser.subcommands = [ 35 | upload, 36 | download, 37 | ls, 38 | delete, 39 | ] 40 | 41 | return parser 42 | -------------------------------------------------------------------------------- /nextstrain/cli/command/remote/delete.py: -------------------------------------------------------------------------------- 1 | """ 2 | Delete datasets and narratives on a remote source. 3 | 4 | A remote source URL specifies what to delete, e.g. to delete the "beta-cov" 5 | dataset in the Nextstrain Group "blab":: 6 | 7 | nextstrain remote delete nextstrain.org/groups/blab/beta-cov 8 | 9 | The :option:`--recursively` option allows for deleting multiple datasets or narratives 10 | at once, e.g. to delete all the "ncov/wa/…" datasets in the "blab" group:: 11 | 12 | nextstrain remote delete --recursively nextstrain.org/groups/blab/ncov/wa 13 | 14 | See :command-reference:`nextstrain remote` for more information on remote sources. 15 | """ 16 | 17 | from ... import console 18 | from ...remote import parse_remote_path 19 | from ...util import warn 20 | 21 | 22 | def register_parser(subparser): 23 | """ 24 | %(prog)s [--recursively] 25 | %(prog)s --help 26 | """ 27 | parser = subparser.add_parser( 28 | "delete", 29 | aliases = ["rm"], 30 | help = "Delete dataset and narratives") 31 | 32 | parser.add_argument( 33 | "remote_path", 34 | help = "Remote source URL for a dataset or narrative. " 35 | "A path prefix to scope/filter by if using :option:`--recursively`.", 36 | metavar = "") 37 | 38 | parser.add_argument( 39 | "--recursively", "-r", 40 | help = "Delete everything under the given remote URL path prefix", 41 | action = "store_true") 42 | 43 | parser.add_argument( 44 | "--dry-run", 45 | help = "Don't actually delete anything, just show what would be deleted", 46 | action = "store_true") 47 | 48 | return parser 49 | 50 | 51 | @console.auto_dry_run_indicator() 52 | def run(opts): 53 | remote, url = parse_remote_path(opts.remote_path) 54 | 55 | deletions = remote.delete(url, recursively = opts.recursively, dry_run = opts.dry_run) 56 | deleted_count = 0 57 | 58 | for file in deletions: 59 | print("Deleting", file) 60 | deleted_count += 1 61 | 62 | if deleted_count: 63 | return 0 64 | else: 65 | warn("Nothing deleted!") 66 | return 1 67 | -------------------------------------------------------------------------------- /nextstrain/cli/command/remote/download.py: -------------------------------------------------------------------------------- 1 | """ 2 | Download datasets and narratives from a remote source. 3 | 4 | A remote source URL specifies what to download, e.g. to download one of the 5 | seasonal influenza datasets:: 6 | 7 | nextstrain remote download nextstrain.org/flu/seasonal/h3n2/ha/2y 8 | 9 | which creates three files in the current directory:: 10 | 11 | flu_seasonal_h3n2_ha_2y.json 12 | flu_seasonal_h3n2_ha_2y_root-sequence.json 13 | flu_seasonal_h3n2_ha_2y_tip-frequencies.json 14 | 15 | The --recursively option allows for downloading multiple datasets or narratives 16 | at once, e.g. to download all the datasets under "ncov/open/…" into an existing 17 | directory named "sars-cov-2":: 18 | 19 | nextstrain remote download --recursively nextstrain.org/ncov/open sars-cov-2/ 20 | 21 | which creates files for each dataset:: 22 | 23 | sars-cov-2/ncov_open_global.json 24 | sars-cov-2/ncov_open_global_root-sequence.json 25 | sars-cov-2/ncov_open_global_tip-frequencies.json 26 | sars-cov-2/ncov_open_africa.json 27 | sars-cov-2/ncov_open_africa_root-sequence.json 28 | sars-cov-2/ncov_open_africa_tip-frequencies.json 29 | … 30 | 31 | See :command-reference:`nextstrain remote` for more information on remote sources. 32 | """ 33 | 34 | import shlex 35 | from pathlib import Path 36 | from ... import console 37 | from ...remote import parse_remote_path 38 | from ...errors import UserError 39 | 40 | 41 | def register_parser(subparser): 42 | """ 43 | %(prog)s [] 44 | %(prog)s --recursively [] 45 | %(prog)s --help 46 | """ 47 | parser = subparser.add_parser("download", help = "Download dataset and narrative files") 48 | 49 | parser.add_argument( 50 | "remote_path", 51 | help = "Remote source URL for a dataset or narrative. " 52 | "A path prefix to scope/filter by if using :option:`--recursively`.", 53 | metavar = "") 54 | 55 | parser.add_argument( 56 | "local_path", 57 | help = "Local directory to save files in. " 58 | "May be a local filename to use if not using :option:`--recursively`. " 59 | 'Defaults to current directory ("%(default)s").', 60 | metavar = "", 61 | type = Path, 62 | nargs = "?", 63 | default = ".") 64 | 65 | parser.add_argument( 66 | "--recursively", "-r", 67 | help = "Download everything under the given remote URL path prefix", 68 | action = "store_true") 69 | 70 | parser.add_argument( 71 | "--dry-run", 72 | help = "Don't actually download anything, just show what would be downloaded", 73 | action = "store_true") 74 | 75 | return parser 76 | 77 | 78 | @console.auto_dry_run_indicator() 79 | def run(opts): 80 | remote, url = parse_remote_path(opts.remote_path) 81 | 82 | if opts.recursively and not opts.local_path.is_dir(): 83 | if opts.local_path.exists(): 84 | raise UserError(f"Local path must be a directory when using --recursively, but «{opts.local_path}» is not.") 85 | else: 86 | raise UserError(f""" 87 | Local path must be a directory when using --recursively, but «{opts.local_path}» doesn't exist. 88 | 89 | If the name is correct, you must create the directory before downloading, e.g.: 90 | 91 | mkdir -p {shlex.quote(str(opts.local_path))} 92 | """) 93 | 94 | downloads = remote.download(url, opts.local_path, recursively = opts.recursively, dry_run = opts.dry_run) 95 | 96 | for remote_file, local_file in downloads: 97 | print("Downloading", remote_file, "as", local_file) 98 | 99 | return 0 100 | -------------------------------------------------------------------------------- /nextstrain/cli/command/remote/ls.py: -------------------------------------------------------------------------------- 1 | """ 2 | List datasets and narratives on a remote source. 3 | 4 | A remote source URL specifies what to list, e.g. to list what's in the 5 | Nextstrain Group named "Blab":: 6 | 7 | nextstrain remote list nextstrain.org/groups/blab 8 | 9 | or list the core seasonal influenza datasets:: 10 | 11 | nextstrain remote list nextstrain.org/flu/seasonal 12 | 13 | See :command-reference:`nextstrain remote` for more information on remote sources. 14 | """ 15 | 16 | from ...remote import parse_remote_path 17 | 18 | 19 | def register_parser(subparser): 20 | """ 21 | %(prog)s 22 | %(prog)s --help 23 | """ 24 | parser = subparser.add_parser( 25 | "list", 26 | aliases = ["ls"], 27 | help = "List datasets and narratives") 28 | 29 | parser.add_argument( 30 | "remote_path", 31 | help = "Remote source URL, with optional path prefix to scope/filter the results", 32 | metavar = "") 33 | 34 | return parser 35 | 36 | 37 | def run(opts): 38 | remote, url = parse_remote_path(opts.remote_path) 39 | 40 | files = remote.ls(url) 41 | 42 | for file in files: 43 | print(file) 44 | 45 | return 0 46 | -------------------------------------------------------------------------------- /nextstrain/cli/command/remote/upload.py: -------------------------------------------------------------------------------- 1 | """ 2 | Upload dataset and narratives files to a remote destination. 3 | 4 | A remote destination URL specifies where to upload, e.g. to upload the dataset 5 | files:: 6 | 7 | auspice/ncov_local.json 8 | auspice/ncov_local_root-sequence.json 9 | auspice/ncov_local_tip-frequencies.json 10 | 11 | so they're visible at `https://nextstrain.org/groups/example/ncov`:: 12 | 13 | nextstrain remote upload nextstrain.org/groups/example/ncov auspice/ncov_local*.json 14 | 15 | If uploading multiple datasets or narratives, uploading to the top-level of a 16 | Nextstrain Group, or uploading to an S3 remote, then the local filenames are 17 | used in combination with any path prefix in the remote source URL. 18 | 19 | See :command-reference:`nextstrain remote` for more information on remote sources. 20 | """ 21 | 22 | from pathlib import Path 23 | from ... import console 24 | from ...remote import parse_remote_path 25 | 26 | 27 | def register_parser(subparser): 28 | """ 29 | %(prog)s [ […]] 30 | %(prog)s --help 31 | """ 32 | parser = subparser.add_parser("upload", help = "Upload dataset and narrative files") 33 | 34 | register_arguments(parser) 35 | 36 | return parser 37 | 38 | 39 | def register_arguments(parser): 40 | # Destination 41 | parser.add_argument( 42 | "destination", 43 | help = "Remote destination URL for a dataset or narrative. " 44 | "A path prefix if the files to upload comprise more than " 45 | "one dataset or narrative or the remote is S3.", 46 | metavar = "") 47 | 48 | # Files to upload 49 | parser.add_argument( 50 | "files", 51 | help = "Files to upload. " 52 | "Typically dataset and sidecar files (Auspice JSON files) " 53 | "and/or narrative files (Markdown files).", 54 | metavar = " [ […]]", 55 | nargs = "+") 56 | 57 | parser.add_argument( 58 | "--dry-run", 59 | help = "Don't actually upload anything, just show what would be uploaded", 60 | action = "store_true") 61 | 62 | 63 | @console.auto_dry_run_indicator() 64 | def run(opts): 65 | remote, url = parse_remote_path(opts.destination) 66 | 67 | files = [Path(f) for f in opts.files] 68 | 69 | uploads = remote.upload(url, files, dry_run = opts.dry_run) 70 | 71 | for local_file, remote_file in uploads: 72 | print("Uploading", local_file, "as", remote_file) 73 | 74 | return 0 75 | -------------------------------------------------------------------------------- /nextstrain/cli/command/shell.py: -------------------------------------------------------------------------------- 1 | """ 2 | Start a new shell inside a Nextstrain runtime to run ad-hoc 3 | commands and perform debugging. 4 | """ 5 | 6 | from pathlib import Path 7 | from typing import Tuple 8 | from .. import resources 9 | from .. import runner 10 | from ..argparse import add_extended_help_flags 11 | from ..paths import SHELL_HISTORY 12 | from ..runner import docker, conda, singularity 13 | from ..util import colored, remove_prefix, runner_name 14 | from ..volume import NamedVolume 15 | from .build import assert_overlay_volumes_support, pathogen_volumes 16 | 17 | 18 | def register_parser(subparser): 19 | """ 20 | %(prog)s [options] [...] 21 | %(prog)s --help 22 | """ 23 | 24 | parser = subparser.add_parser("shell", help = "Start a new shell in a runtime", add_help = False) 25 | 26 | # Support --help and --help-all 27 | add_extended_help_flags(parser) 28 | 29 | # Positional parameters 30 | parser.add_argument( 31 | "directory", 32 | help = "Path to pathogen build directory", 33 | metavar = "", 34 | type = Path) 35 | 36 | # Register runner flags and arguments; excludes ambient and AWS Batch 37 | # runners since those don't make any sense here. 38 | runner.register_runners( 39 | parser, 40 | exec = ["bash", ...], 41 | runners = [docker, conda, singularity]) 42 | 43 | return parser 44 | 45 | 46 | def run(opts): 47 | assert_overlay_volumes_support(opts) 48 | 49 | # Interpret the given directory 50 | build_volume, working_volume = pathogen_volumes(opts.directory) 51 | 52 | opts.volumes.append(build_volume) # for Docker and Singularity 53 | 54 | print(colored("bold", f"Entering the Nextstrain runtime ({runner_name(opts.__runner__)})")) 55 | print() 56 | 57 | if opts.volumes and opts.__runner__ in {docker, singularity}: 58 | print(colored("bold", "Mapped volumes:")) 59 | 60 | # This is more tightly coupled to the Docker/Singularity runners than 61 | # I'd like (i.e. assuming /nextstrain/…), but the number of runtimes 62 | # will always be small so some special-casing seems ok. 63 | # -trs, 5 Jan 2023 (updated from 25 Sept 2018) 64 | for volume in opts.volumes: 65 | print(" %s is from %s" % (docker.mount_point(volume), volume.src.resolve(strict = True))) 66 | 67 | print() 68 | 69 | print(colored("bold", 'Run the command "exit" to leave the runtime.')) 70 | print() 71 | 72 | with resources.as_file("bashrc") as bashrc: 73 | # Ensure the history file exists to pass checks the Docker/Singularity 74 | # runners perform for mounted volumes. This also makes sure that the 75 | # file is writable by the Conda runtime too by ensuring the parent 76 | # directory exists. 77 | # 78 | # Don't use strict=True because it's ok if it doesn't exist yet! 79 | history_file = SHELL_HISTORY.resolve() 80 | history_file.parent.mkdir(parents = True, exist_ok = True) 81 | 82 | try: 83 | # Don't use exist_ok=True so we don't touch the mtime unnecessarily 84 | history_file.touch() 85 | except FileExistsError: 86 | pass 87 | 88 | if opts.__runner__ is conda: 89 | opts.default_exec_args = [ 90 | *opts.default_exec_args, 91 | "--rcfile", str(bashrc), 92 | ] 93 | 94 | elif opts.__runner__ in {docker, singularity}: 95 | opts.volumes.append(NamedVolume("bashrc", bashrc, dir = False, writable = False)) 96 | 97 | history_volume = NamedVolume("bash_history", history_file, dir = False) 98 | history_file = docker.mount_point(history_volume) 99 | opts.volumes.append(history_volume) 100 | 101 | extra_env = { 102 | "NEXTSTRAIN_PS1": ps1(), 103 | "NEXTSTRAIN_HISTFILE": str(history_file), 104 | } 105 | 106 | return runner.run(opts, working_volume = working_volume, extra_env = extra_env) 107 | 108 | 109 | def ps1() -> str: 110 | # ESC[ 38;2;⟨r⟩;⟨g⟩;⟨b⟩ m — Select RGB foreground color 111 | # ESC[ 48;2;⟨r⟩;⟨g⟩;⟨b⟩ m — Select RGB background color 112 | def fg(color: str) -> str: return r'\[\e[38;2;{};{};{}m\]'.format(*rgb(color)) 113 | def bg(color: str) -> str: return r'\[\e[48;2;{};{};{}m\]'.format(*rgb(color)) 114 | 115 | def rgb(color: str) -> Tuple[int, int, int]: 116 | color = remove_prefix("#", color) 117 | r,g,b = (int(c, 16) for c in (color[0:2], color[2:4], color[4:6])) 118 | return r,g,b 119 | 120 | wordmark = ( 121 | (' ', '#4377cd'), 122 | ('N', '#4377cd'), 123 | ('e', '#5097ba'), 124 | ('x', '#63ac9a'), 125 | ('t', '#7cb879'), 126 | ('s', '#9abe5c'), 127 | ('t', '#b9bc4a'), 128 | ('r', '#d4b13f'), 129 | ('a', '#e49938'), 130 | ('i', '#e67030'), 131 | ('n', '#de3c26'), 132 | (' ', '#de3c26')) 133 | 134 | # Bold, bright white text (fg)… 135 | PS1 = r'\[\e[1;97m\]' 136 | 137 | # …on a colored background 138 | for letter, color in wordmark: 139 | PS1 += bg(color) + letter 140 | 141 | # Add working dir and traditional prompt char (in magenta) 142 | PS1 += r'\[\e[0m\] \w' + fg('#ff00ff') + r' \$ ' 143 | 144 | # Reset 145 | PS1 += r'\[\e[0m\]' 146 | 147 | return PS1 148 | -------------------------------------------------------------------------------- /nextstrain/cli/command/version.py: -------------------------------------------------------------------------------- 1 | """ 2 | Prints the version of the Nextstrain CLI. 3 | """ 4 | 5 | import sys 6 | from textwrap import indent 7 | from ..__version__ import __version__ 8 | from ..pathogens import all_pathogen_versions_by_name, every_pathogen_default_by_name 9 | from ..runner import all_runners, default_runner 10 | from ..util import runner_name, standalone_installation 11 | 12 | def register_parser(subparser): 13 | parser = subparser.add_parser("version", help = "Show version information") 14 | 15 | parser.add_argument( 16 | "--verbose", 17 | help = "Show versions of each runtime, plus select individual Nextstrain components within, and versions of each pathogen, including URLs", 18 | action = "store_true") 19 | 20 | parser.add_argument( 21 | "--pathogens", 22 | help = "Show pathogen versions; implied by --verbose", 23 | action = "store_true") 24 | 25 | parser.add_argument( 26 | "--runtimes", 27 | help = "Show runtime versions; implied by --verbose", 28 | action = "store_true") 29 | 30 | return parser 31 | 32 | 33 | def run(opts): 34 | print("Nextstrain CLI", __version__, "(standalone)" if standalone_installation() else "") 35 | 36 | if opts.verbose: 37 | print() 38 | print("Python") 39 | print(" " + sys.executable) 40 | print(indent(sys.version, " ")) 41 | 42 | if opts.runtimes or (opts.verbose and not opts.pathogens): 43 | print() 44 | print("Runtimes") 45 | for i, runner in enumerate(all_runners): 46 | if i != 0: 47 | print() 48 | print(" " + runner_name(runner), "(default)" if runner is default_runner else "") 49 | versions = iter(runner.versions()) 50 | version = next(versions, None) 51 | if version: 52 | print(" " + version) 53 | else: 54 | print(" unknown") 55 | if opts.verbose: 56 | for version in versions: 57 | print(" " + version) 58 | 59 | if opts.pathogens or (opts.verbose and not opts.runtimes): 60 | print() 61 | print("Pathogens") 62 | if pathogens := all_pathogen_versions_by_name(): 63 | defaults = every_pathogen_default_by_name(pathogens) 64 | 65 | for i, (name, versions) in enumerate(pathogens.items()): 66 | if i != 0: 67 | print() 68 | print(" " + name) 69 | for version in versions.values(): 70 | is_default = version == defaults.get(name) 71 | print(" " + str(version) + (f"={version.url or ''}" if opts.verbose else ""), "(default)" if is_default else "") 72 | if opts.verbose: 73 | print(" " + str(version.path)) 74 | else: 75 | print(" (none)") 76 | -------------------------------------------------------------------------------- /nextstrain/cli/command/whoami.py: -------------------------------------------------------------------------------- 1 | """ 2 | Show information about the logged-in user for Nextstrain.org (and other 3 | remotes). 4 | 5 | The username, email address (if available), and Nextstrain Groups memberships 6 | of the currently logged-in user are shown. 7 | 8 | Exits with an error if no one is logged in. 9 | """ 10 | from inspect import cleandoc 11 | from ..errors import UserError 12 | from ..remote import parse_remote_path 13 | 14 | 15 | def register_parser(subparser): 16 | parser = subparser.add_parser("whoami", help = "Show information about the logged-in user") 17 | 18 | parser.add_argument( 19 | "remote", 20 | help = cleandoc(""" 21 | Remote URL for which to show the logged-in user. Expects URLs like 22 | the remote source/destination URLs used by the `nextstrain remote` 23 | family of commands. Only the domain name (technically, the origin) 24 | of the URL is required/used, but a full URL may be specified. 25 | """), 26 | metavar = "", 27 | nargs = "?", 28 | default = "nextstrain.org") 29 | 30 | # XXX TODO: Supporting `nextstrain whoami --all` would be nice. 31 | # -trs, 15 Nov 2023 32 | 33 | return parser 34 | 35 | 36 | def run(opts): 37 | remote, url = parse_remote_path(opts.remote) 38 | assert url.origin 39 | 40 | user = remote.current_user(url.origin) 41 | 42 | if not user: 43 | raise UserError(f"Not logged in to {url.origin}.") 44 | 45 | print(f"# Logged into {user.origin} as…") 46 | print(f"username: {user.username}") 47 | if user.email: 48 | print(f"email: {user.email}") 49 | print(f"groups: {format_groups(user)}") 50 | 51 | 52 | def format_groups(user): 53 | if user.groups: 54 | return "".join(f"\n - {g}" for g in sorted(user.groups)) 55 | else: 56 | return "\n # (none)" 57 | -------------------------------------------------------------------------------- /nextstrain/cli/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Configuration file handling. 3 | """ 4 | 5 | import stat 6 | from configparser import ConfigParser 7 | from contextlib import contextmanager 8 | from fasteners import InterProcessReaderWriterLock 9 | from pathlib import Path 10 | from typing import Optional 11 | from .paths import CONFIG, SECRETS, LOCK 12 | 13 | 14 | # Permissions to use for the secrets file if we have to create it. 15 | SECRETS_PERMS = \ 16 | ( stat.S_IRUSR # u+r 17 | | stat.S_IWUSR # u+w 18 | ) # u=rw,go= 19 | 20 | 21 | def load(path: Path = CONFIG) -> ConfigParser: 22 | """ 23 | Load the config file at *path* and return a ConfigParser object. 24 | 25 | If *path* does not exist, no error is raised, but an empty ConfigParser 26 | object is returned. This is the default behaviour of ConfigParser and 27 | intended so that a missing config file isn't fatal. 28 | """ 29 | config = ConfigParser() 30 | config.read(str(path), encoding = "utf-8") 31 | return config 32 | 33 | 34 | def save(config, path: Path = CONFIG): 35 | """ 36 | Write the *config* object to *path*. 37 | 38 | If the immediate parent directory of the file named by *path* is 39 | ``.nextstrain``, then that directory will be created if it does not already 40 | exist. 41 | """ 42 | secrets = path is SECRETS 43 | 44 | path = path.resolve(strict = False) 45 | 46 | if path.parent.name == ".nextstrain": 47 | path.parent.mkdir(exist_ok = True) 48 | 49 | if secrets: 50 | if path.exists(): 51 | path.chmod(SECRETS_PERMS) 52 | else: 53 | path.touch(SECRETS_PERMS) 54 | 55 | with path.open(mode = "w", encoding = "utf-8") as file: 56 | config.write(file) 57 | 58 | 59 | def get(section: str, field: str, fallback: str = None, path: Path = CONFIG) -> Optional[str]: 60 | """ 61 | Return *field* from *section* in the config file at the given *path*. 62 | 63 | If *section* or *field* does not exist, returns *fallback* (which defaults 64 | to None). 65 | """ 66 | with read_lock(): 67 | config = load(path) 68 | 69 | if section in config: 70 | return config[section].get(field, fallback) 71 | else: 72 | return fallback 73 | 74 | 75 | def set(section: str, field: str, value: str, path: Path = CONFIG): 76 | """ 77 | Set *field* in *section* to *value* in the config file at the given *path*. 78 | 79 | If *section* does not exist, it is automatically created. 80 | """ 81 | with write_lock(): 82 | config = load(path) 83 | 84 | if section not in config: 85 | config.add_section(section) 86 | 87 | config[section][field] = value 88 | 89 | save(config, path) 90 | 91 | 92 | def setdefault(section: str, field: str, value: str, path: Path = CONFIG): 93 | """ 94 | Set *field* in *section* to *value* in the config file at the given *path*, 95 | if *field* doesn't already exist. 96 | 97 | If *section* does not exist, it is automatically created. 98 | """ 99 | with write_lock(): 100 | config = load(path) 101 | 102 | if section not in config: 103 | config.add_section(section) 104 | 105 | config[section].setdefault(field, value) 106 | 107 | save(config, path) 108 | 109 | 110 | def remove(section: str, path: Path) -> bool: 111 | """ 112 | Remove the *section* in the config file at the given *path*. 113 | 114 | Returns ``True`` when *section* is removed. Returns ``False`` if *section* 115 | or *path* does not exist. 116 | """ 117 | if not path.exists(): 118 | return False 119 | 120 | with write_lock(): 121 | config = load(path) 122 | 123 | if section not in config: 124 | return False 125 | 126 | del config[section] 127 | 128 | save(config, path) 129 | 130 | return True 131 | 132 | 133 | @contextmanager 134 | def read_lock(): 135 | """ 136 | Lock for reading across processes (but not within). 137 | 138 | Uses a global advisory/cooperative lock. 139 | """ 140 | with InterProcessReaderWriterLock(LOCK).read_lock(): 141 | yield 142 | 143 | 144 | @contextmanager 145 | def write_lock(): 146 | """ 147 | Lock for writing across processes (but not within). 148 | 149 | Uses a global advisory/cooperative lock. 150 | """ 151 | with InterProcessReaderWriterLock(LOCK).write_lock(): 152 | yield 153 | -------------------------------------------------------------------------------- /nextstrain/cli/console.py: -------------------------------------------------------------------------------- 1 | """ 2 | Console interface. 3 | """ 4 | import re 5 | import sys 6 | from contextlib import contextmanager, ExitStack, redirect_stdout, redirect_stderr 7 | from functools import wraps 8 | from typing import Callable, TextIO 9 | from wrapt import ObjectProxy 10 | 11 | 12 | def auto_dry_run_indicator(getter: Callable[..., bool] = lambda opts, *args, **kwargs: opts.dry_run): 13 | """ 14 | Automatically wraps a function in a :py:func:`dry_run_indicator` context 15 | based on the function's arguments. 16 | 17 | *getter* is a callable which accepts any arguments and returns a boolean 18 | indicating if a dry run mode is active or not. 19 | 20 | The default *getter* is intended for the typical ``run(opts)`` functions of 21 | our command modules that use an ``opts.dry_run`` parameter (i.e. set by the 22 | ``--dry-run`` command-line option). Provide a custom *getter* if that's 23 | not the case. 24 | 25 | The primary usefulness of this decorator is avoiding additional near-global 26 | levels of indentation. 27 | """ 28 | def decorator(f): 29 | @wraps(f) 30 | def decorated(*args, **kwargs): 31 | # Get dry run status from function args 32 | dry_run = getter(*args, **kwargs) 33 | 34 | # Run under an indicator context 35 | with dry_run_indicator(dry_run): 36 | return f(*args, **kwargs) 37 | return decorated 38 | return decorator 39 | 40 | 41 | @contextmanager 42 | def dry_run_indicator(dry_run: bool = False): 43 | """ 44 | Context manager to add an indication to :py:attr:`sys.stdout` and 45 | :py:attr:`sys.stderr` output that a "dry run" is taking place. 46 | 47 | Prefixes each line with ``DRY RUN │ `` if *dry_run* is true. 48 | 49 | Does nothing if *dry_run* is not true. 50 | 51 | When entered, returns *dry_run* for the target of the ``with`` statement, 52 | if any. 53 | 54 | >>> from io import StringIO 55 | >>> with redirect_stdout(StringIO()) as out, redirect_stderr(out) as stderr: 56 | ... with dry_run_indicator(True) as dry_run: 57 | ... print("stdout") 58 | ... print("stderr", file = sys.stderr) 59 | 60 | >>> print(out.getvalue(), end = "") 61 | DRY RUN │ stdout 62 | DRY RUN │ stderr 63 | 64 | >>> dry_run 65 | True 66 | 67 | >>> with redirect_stdout(StringIO()) as out, redirect_stderr(out) as stderr: 68 | ... with dry_run_indicator(False) as dry_run: 69 | ... print("stdout") 70 | ... print("stderr", file = sys.stderr) 71 | 72 | >>> print(out.getvalue(), end = "") 73 | stdout 74 | stderr 75 | 76 | >>> dry_run 77 | False 78 | """ 79 | with ExitStack() as stack: 80 | if dry_run: 81 | stack.enter_context(redirect_stdout(LinePrefixer(sys.stdout, "DRY RUN │ "))) 82 | stack.enter_context(redirect_stderr(LinePrefixer(sys.stderr, "DRY RUN │ "))) 83 | yield dry_run 84 | 85 | 86 | class LinePrefixer(ObjectProxy): # pyright: ignore[reportUntypedBaseClass] 87 | """ 88 | Add *prefix* to every line written to *file*. 89 | 90 | >>> import sys 91 | >>> def output(file): 92 | ... print("Swizzling the sporks…", file = file) 93 | ... print("Reticulating splines…", file = file, end = "") 94 | ... print("\\n done!", file = file) 95 | ... print("Gimbling the wabe (this may take a while)\\n\\n\\n", file = file) 96 | ... print("Gyre away!", file = file) 97 | 98 | >>> output(sys.stdout) 99 | Swizzling the sporks… 100 | Reticulating splines… 101 | done! 102 | Gimbling the wabe (this may take a while) 103 | 104 | 105 | 106 | Gyre away! 107 | 108 | >>> output(LinePrefixer(sys.stdout, "DRY RUN: ")) 109 | DRY RUN: Swizzling the sporks… 110 | DRY RUN: Reticulating splines… 111 | DRY RUN: done! 112 | DRY RUN: Gimbling the wabe (this may take a while) 113 | DRY RUN: 114 | DRY RUN: 115 | DRY RUN: 116 | DRY RUN: Gyre away! 117 | 118 | Attributes of *file* are passed through the :py:cls:`LinePrefixer` object: 119 | 120 | >>> p = LinePrefixer(sys.__stdout__, " ") 121 | >>> p.fileno() 122 | 1 123 | """ 124 | # Declaring these here keeps them local to this proxy object instead of 125 | # being written through to the wrapped file object. 126 | __prefix: str = "" 127 | __next_write_needs_prefix: bool = True 128 | 129 | def __init__(self, file: TextIO, prefix: str): 130 | super().__init__(file) 131 | self.__prefix = prefix 132 | self.__next_write_needs_prefix = True 133 | 134 | def write(self, s) -> int: 135 | s_ = "" 136 | 137 | if self.__next_write_needs_prefix: 138 | s_ += self.__prefix 139 | 140 | # Wait to write a prefix after a trailing newline until the next call. 141 | # This avoids a dangling prefix if we're never called again. 142 | self.__next_write_needs_prefix = s.endswith("\n") 143 | 144 | # Insert prefix after every newline except for an end-of-string 145 | # newline, which we'll account for as above on the next, if any, call 146 | # to us. 147 | s_ += re.sub(r'(?<=\n)(?!\Z)', self.__prefix, s) 148 | 149 | return self.__wrapped__.write(s_) 150 | -------------------------------------------------------------------------------- /nextstrain/cli/debug.py: -------------------------------------------------------------------------------- 1 | """ 2 | Debug flags and utilities. 3 | 4 | .. envvar:: NEXTSTRAIN_DEBUG 5 | 6 | Set to a truthy value (e.g. 1) to print more information about (handled) 7 | errors. For example, when this is not set or falsey, stack traces and 8 | parent exceptions in an exception chain are omitted from handled errors. 9 | """ 10 | from os import environ 11 | from sys import stderr 12 | 13 | DEBUGGING = bool(environ.get("NEXTSTRAIN_DEBUG")) 14 | 15 | if DEBUGGING: 16 | def debug(*args): 17 | print("DEBUG:", *args, file = stderr) 18 | else: 19 | def debug(*args): 20 | pass 21 | -------------------------------------------------------------------------------- /nextstrain/cli/errors.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exception classes for internal use. 3 | """ 4 | from textwrap import dedent 5 | 6 | 7 | class NextstrainCliError(Exception): 8 | """Exception base class for all custom :mod:`nextstrain.cli` exceptions.""" 9 | pass 10 | 11 | class InternalError(NextstrainCliError): 12 | pass 13 | 14 | class UserError(NextstrainCliError): 15 | """ 16 | Error intended for display to the user, e.g. an error aiming to be clear, 17 | friendly, and, if possible, actionable. 18 | """ 19 | def __init__(self, message, *args, **kwargs): 20 | # Remove leading newlines, trailing whitespace, and then indentation 21 | # to better support nicely-formatted """multi-line strings""". 22 | formatted_message = dedent(message.lstrip("\n").rstrip()).format(*args, **kwargs) 23 | 24 | super().__init__("Error: " + formatted_message) 25 | 26 | class UsageError(UserError): 27 | """ 28 | Prints brief command usage before the error message. 29 | """ 30 | pass 31 | -------------------------------------------------------------------------------- /nextstrain/cli/gzip.py: -------------------------------------------------------------------------------- 1 | """ 2 | Gzip stream utilities. 3 | """ 4 | import zlib 5 | from io import BufferedIOBase 6 | from typing import BinaryIO 7 | from .util import byte_quantity, warn 8 | 9 | 10 | DEFAULT_ITER_SIZE = byte_quantity("8 MiB") 11 | 12 | 13 | class GzipCompressingReader(BufferedIOBase): 14 | """ 15 | Compress a data stream as it is being read. 16 | 17 | The constructor takes an existing, readable byte *stream*. Calls to this 18 | class's :meth:`.read` method will read data from the source *stream* and 19 | return a compressed copy. 20 | """ 21 | def __init__(self, stream: BinaryIO, iter_size: int = DEFAULT_ITER_SIZE): 22 | if not stream.readable(): 23 | raise ValueError('"stream" argument must be readable.') 24 | 25 | if type(iter_size) is not int: 26 | raise TypeError('"iter_size" argument must be an int') 27 | 28 | self.stream = stream 29 | self.iter_size = iter_size 30 | self.__buffer = b'' 31 | self.__gzip = zlib.compressobj( 32 | wbits = 16 + zlib.MAX_WBITS, # Offset of 16 is gzip encapsulation 33 | memLevel = 9, # Memory is ~cheap; use it for better compression 34 | ) 35 | 36 | def readable(self): 37 | return True 38 | 39 | def read(self, size = None): 40 | assert size != 0 41 | assert self.stream 42 | 43 | if size is None: 44 | size = -1 45 | 46 | # Keep reading chunks until we have a bit of compressed data to return, 47 | # since returning an empty byte string would be interpreted as EOF. 48 | while not self.__buffer and self.__gzip: 49 | chunk = self.stream.read(size) 50 | 51 | self.__buffer += self.__gzip.compress(chunk) 52 | 53 | if not chunk or size < 0: 54 | # Read to EOF on underlying stream, so flush any remaining 55 | # compressed data and return whatever we have. We'll return an 56 | # empty byte string as EOF ourselves on any subsequent calls. 57 | self.__buffer += self.__gzip.flush(zlib.Z_FINISH) 58 | self.__gzip = None 59 | 60 | if size > 0 and len(self.__buffer) > size: 61 | # This should be pretty rare since we're reading N bytes and then 62 | # *compressing* to fewer bytes. It could happen in the rare case 63 | # of lots of data still stuck in the buffer from a previous call. 64 | compressed = self.__buffer[0:size] 65 | self.__buffer = self.__buffer[size:] 66 | else: 67 | compressed = self.__buffer 68 | self.__buffer = b'' 69 | 70 | return compressed 71 | 72 | def close(self): 73 | if self.stream: 74 | try: 75 | self.stream.close() 76 | finally: 77 | self.stream = None 78 | 79 | def __next__(self): 80 | """ 81 | Iterate in :attr:`.iter_size` chunks. 82 | 83 | Overrides default line-wise iteration from :cls:`io.IOBase`. Line-wise 84 | iteration has no reasonable semantics for binary IO streams like this 85 | one. It only serves to slow down the stream by using many short reads 86 | instead of fewer longer reads. 87 | """ 88 | chunk = self.read(self.iter_size) 89 | 90 | if not len(chunk): 91 | raise StopIteration 92 | 93 | return chunk 94 | 95 | 96 | class GzipDecompressingWriter(BufferedIOBase): 97 | """ 98 | Decompress a gzip data stream as it is being written. 99 | 100 | The constructor takes an existing, writable byte *stream*. Data written to 101 | this class's :meth:`.write` will be decompressed and then passed along to 102 | the destination *stream*. 103 | """ 104 | # Offset of 32 means we will accept a zlib or gzip encapsulation, per 105 | # . Seems no 106 | # downside to applying Postel's Law here. 107 | # 108 | def __init__(self, stream: BinaryIO): 109 | if not stream.writable(): 110 | raise ValueError('"stream" argument must be writable.') 111 | 112 | self.stream = stream 113 | self.__gunzip = zlib.decompressobj(32 + zlib.MAX_WBITS) 114 | 115 | def writable(self): 116 | return True 117 | 118 | def write(self, data: bytes): # pyright: ignore[reportIncompatibleMethodOverride] 119 | assert self.stream and self.__gunzip 120 | return self.stream.write(self.__gunzip.decompress(data)) 121 | 122 | def flush(self): 123 | assert self.stream and self.__gunzip 124 | super().flush() 125 | self.stream.flush() 126 | 127 | def close(self): 128 | if self.stream: 129 | assert self.__gunzip 130 | try: 131 | self.stream.write(self.__gunzip.flush()) 132 | self.stream.close() 133 | finally: 134 | self.stream = None 135 | self.__gunzip = None 136 | 137 | 138 | def ContentDecodingWriter(encoding, stream): 139 | """ 140 | Wrap a writeable *stream* in a layer which decodes *encoding*. 141 | 142 | *encoding* is expected to be a ``Content-Encoding`` HTTP header value. 143 | ``gzip`` and ``deflate`` are supported. Unsupported values will issue a 144 | warning and return the *stream* unwrapped. An *encoding* of ``None`` will 145 | also return the stream unwrapped, but without a warning. 146 | """ 147 | if encoding is not None and encoding.lower() in {"gzip", "deflate"}: 148 | return GzipDecompressingWriter(stream) 149 | else: 150 | if encoding is not None: 151 | warn("Ignoring unknown content encoding «%s»" % encoding) 152 | return stream 153 | -------------------------------------------------------------------------------- /nextstrain/cli/hostenv.py: -------------------------------------------------------------------------------- 1 | """ 2 | Translation of local host environment to the runtime environments. 3 | """ 4 | 5 | import os 6 | from typing import List, Tuple 7 | 8 | 9 | # Host environment variables to pass through (forward) into the Nextstrain 10 | # build/runner environments (e.g. the Docker container or AWS Batch job). This 11 | # is intended to be a central, authoritative list. 12 | 13 | # XXX TODO: Remove build-specific variables below (which don't belong in this 14 | # generic CLI tool) in favor of another mechanism for consistently passing 15 | # environment variables into the containers. 16 | # -trs, 13 Dec 2019 17 | 18 | forwarded_names = [ 19 | # Augur 20 | "AUGUR_RECURSION_LIMIT", 21 | "AUGUR_MINIFY_JSON", 22 | 23 | # AWS 24 | "AWS_ACCESS_KEY_ID", 25 | "AWS_SECRET_ACCESS_KEY", 26 | "AWS_SESSION_TOKEN", 27 | 28 | # ID3C 29 | "ID3C_URL", 30 | "ID3C_USERNAME", 31 | "ID3C_PASSWORD", 32 | 33 | # RethinkDB credentials 34 | "RETHINK_HOST", 35 | "RETHINK_AUTH_KEY", 36 | ] 37 | 38 | 39 | def forwarded_values() -> List[Tuple[str, str]]: 40 | """ 41 | Return a list of (name, value) tuples for the ``hostenv.forwarded_names`` 42 | defined in the current ``os.environ``. 43 | 44 | Values may be sensitive credentials, so if at all possible, values returned 45 | from this should generally be omitted from command-line invocations and 46 | similar widely-visible contexts. 47 | """ 48 | 49 | return [ 50 | (name, os.environ.get(name, "")) 51 | for name in forwarded_names 52 | if name in os.environ 53 | ] 54 | -------------------------------------------------------------------------------- /nextstrain/cli/net.py: -------------------------------------------------------------------------------- 1 | """ 2 | Network handling. 3 | """ 4 | from ipaddress import ip_address, IPv4Address, IPv6Address 5 | from typing import Optional, Set, Union 6 | from socket import getaddrinfo, getnameinfo, gaierror, AddressFamily, AF_UNSPEC, NI_NUMERICHOST 7 | 8 | 9 | def is_loopback(host: Optional[str]) -> Optional[bool]: 10 | """ 11 | Check if *host* points to only loopback IPs. 12 | 13 | Returns True if so, otherwise False. Returns None if *host* is None or on 14 | host resolution error (e.g. DNS error). 15 | """ 16 | if host is None: 17 | return None 18 | 19 | try: 20 | ips = resolve_host(host) 21 | except gaierror: 22 | return None 23 | 24 | if not ips: 25 | return None 26 | 27 | return all(ip.is_loopback for ip in ips) 28 | 29 | 30 | def resolve_host(host: str, family: AddressFamily = AF_UNSPEC) -> Set[Union[IPv4Address, IPv6Address]]: 31 | """ 32 | Resolves a named or numeric *host* to a set of IP addresses. 33 | 34 | By default, all IPv4 and IPv6 addresses are resolved, as applicable 35 | depending on the local IP stack and DNS records for the named host. A 36 | specific address family can be chosen by providing *family*. 37 | """ 38 | # The isinstance() checks filter out IPv6 addresses when Python lacks 39 | # support.¹ 40 | # -trs, 11 Feb 2025 41 | # 42 | # ¹ See 43 | # and . 44 | return { 45 | ip_address(getnameinfo(sockaddr, NI_NUMERICHOST)[0]) 46 | for _, _, _, _, sockaddr 47 | in getaddrinfo(host, None, family = family) 48 | if isinstance(sockaddr[0], str) 49 | and isinstance(sockaddr[1], int) } 50 | -------------------------------------------------------------------------------- /nextstrain/cli/paths.py: -------------------------------------------------------------------------------- 1 | """ 2 | Filesystem paths. 3 | 4 | Documented manually in :file:`doc/config/paths.rst` because the autodoc 5 | directives make it too difficult to produce reasonable output for this module. 6 | (Granted, it's a bit of an off-label use.) 7 | """ 8 | import os 9 | from pathlib import Path 10 | from typing import Union 11 | 12 | 13 | def from_env(name: str, default: Union[str, Path]) -> Path: 14 | """ 15 | Wrap a :py:cls:`Path` around the value of the environment variable *name*, 16 | if any, otherwise *default*. 17 | 18 | Environment variables which are set but empty will be treated as unset 19 | (i.e. *default* will be used). 20 | """ 21 | return Path(os.environ.get(name) or default) 22 | 23 | 24 | # Not finding a homedir is unlikely, but possible. Fallback to the current 25 | # directory. 26 | try: 27 | HOME = Path.home() 28 | except: 29 | HOME = Path(".") 30 | 31 | 32 | # Path to our config/app data dir 33 | NEXTSTRAIN_HOME = from_env("NEXTSTRAIN_HOME", HOME / ".nextstrain/") 34 | 35 | # Path to runtime data dirs 36 | RUNTIMES = from_env("NEXTSTRAIN_RUNTIMES", NEXTSTRAIN_HOME / "runtimes/") 37 | 38 | # Path to pathogen workflow data dirs 39 | PATHOGENS = from_env("NEXTSTRAIN_PATHOGENS", NEXTSTRAIN_HOME / "pathogens/") 40 | 41 | # Path to our config file 42 | CONFIG = from_env("NEXTSTRAIN_CONFIG", NEXTSTRAIN_HOME / "config") 43 | 44 | # Path to our secrets file 45 | SECRETS = from_env("NEXTSTRAIN_SECRETS", NEXTSTRAIN_HOME / "secrets") 46 | 47 | # Path to our global lock file 48 | LOCK = from_env("NEXTSTRAIN_LOCK", NEXTSTRAIN_HOME / "lock") 49 | 50 | # Path to our shell history file 51 | SHELL_HISTORY = from_env("NEXTSTRAIN_SHELL_HISTORY", NEXTSTRAIN_HOME / "shell-history") 52 | -------------------------------------------------------------------------------- /nextstrain/cli/requests.py: -------------------------------------------------------------------------------- 1 | """ 2 | HTTP requests and responses with consistent defaults for us. 3 | 4 | .. envvar:: NEXTSTRAIN_CLI_USER_AGENT_MINIMAL 5 | 6 | Set to a truthy value (e.g. 1) to send only a minimal `User-Agent header 7 | `__ 8 | in HTTP requests. 9 | 10 | The minimal User-Agent header includes just the Nextstrain CLI version, 11 | e.g.:: 12 | 13 | Nextstrain-CLI/9.0.0 (https://nextstrain.org/cli) 14 | 15 | The full User-Agent header normally sent with requests includes basic 16 | information on several important software components, e.g.:: 17 | 18 | Nextstrain-CLI/9.0.0 (https://nextstrain.org/cli) Python/3.10.9 python-requests/2.32.3 platform/Linux-x86_64 installer/standalone tty/yes 19 | 20 | This information is non-identifying and useful for our troubleshooting and 21 | aggregate usage metrics, so we do not recommend omitting it unless 22 | necessary. 23 | """ 24 | import certifi 25 | import os 26 | import platform 27 | import requests 28 | import sys 29 | from functools import lru_cache 30 | from typing import Tuple 31 | 32 | # Import these for re-export for better drop-in compatibility 33 | # with existing callers. 34 | import requests.auth as auth # noqa: F401 35 | import requests.exceptions as exceptions # noqa: F401 36 | import requests.utils as utils # noqa: F401 37 | from requests import PreparedRequest, RequestException, Response # noqa: F401 38 | 39 | from .__version__ import __version__ 40 | 41 | 42 | USER_AGENT_MINIMAL = bool(os.environ.get("NEXTSTRAIN_CLI_USER_AGENT_MINIMAL")) 43 | 44 | CA_BUNDLE = os.environ.get("REQUESTS_CA_BUNDLE") \ 45 | or os.environ.get("CURL_CA_BUNDLE") \ 46 | or certifi.where() 47 | 48 | 49 | class Session(requests.Session): 50 | def __init__(self): 51 | super().__init__() 52 | 53 | # Add our own user agent with useful information 54 | self.headers["User-Agent"] = default_user_agent() 55 | 56 | 57 | def get(*args, **kwargs) -> Response: 58 | with Session() as session: 59 | return session.get(*args, **kwargs) 60 | 61 | def post(*args, **kwargs) -> Response: 62 | with Session() as session: 63 | return session.post(*args, **kwargs) 64 | 65 | 66 | @lru_cache(maxsize = None) 67 | def default_user_agent(minimal: bool = USER_AGENT_MINIMAL) -> str: 68 | """ 69 | Returns an informative user-agent for ourselves. 70 | 71 | If *minimal*, only our own version is included. Otherwise, useful 72 | information on several components is also included. 73 | 74 | Format complies with `RFC 9110 75 | `__. 76 | """ 77 | if minimal: 78 | return f"Nextstrain-CLI/{__version__} (https://nextstrain.org/cli)" 79 | 80 | py_version = version_info_to_str(sys.version_info) 81 | 82 | from .util import distribution_installer # import here to avoid import cycle 83 | installer = distribution_installer() or "unknown" 84 | 85 | system = platform.system() 86 | machine = platform.machine() 87 | 88 | tty = "yes" if any(os.isatty(fd) for fd in [0, 1, 2]) else "no" 89 | 90 | return f"Nextstrain-CLI/{__version__} (https://nextstrain.org/cli) Python/{py_version} python-requests/{requests.__version__} platform/{system}-{machine} installer/{installer} tty/{tty}" 91 | 92 | 93 | def version_info_to_str(version_info: Tuple[int, int, int, str, int]) -> str: 94 | """ 95 | Convert a :attr:`sys.version_info` tuple (or lookalike) to its canonical 96 | string representation. 97 | 98 | >>> version_info_to_str((1, 2, 3, "final", 0)) 99 | '1.2.3' 100 | >>> version_info_to_str((1, 2, 3, "alpha", 0)) 101 | '1.2.3a0' 102 | >>> version_info_to_str((1, 2, 3, "beta", 1)) 103 | '1.2.3b1' 104 | >>> version_info_to_str((1, 2, 3, "candidate", 2)) 105 | '1.2.3rc2' 106 | >>> version_info_to_str((1, 2, 3, "bogus", 3)) 107 | '1.2.3bogus3' 108 | """ 109 | major, minor, micro, releaselevel, serial = version_info 110 | 111 | if pre := {"alpha":"a", "beta":"b", "candidate":"rc", "final":""}.get(releaselevel, releaselevel): 112 | pre += str(serial) 113 | 114 | return f"{major}.{minor}.{micro}{pre}" 115 | -------------------------------------------------------------------------------- /nextstrain/cli/resources/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Resource files. 3 | """ 4 | # We gate usage of the stdlib implementation on 3.11 because that's the first 5 | # version with a full adapter for making the new files() / Traversable API 6 | # backwards compatible with importers only providing the original path() / 7 | # ResourceReader API. The PyPI backport, on the other hand, contains the full 8 | # adapter since 5.3.0, which we declare as our minimum version in setup.py, so 9 | # we use that even on 3.9 and 3.10. 10 | # 11 | # We're using the new API at all because the original one is being deprecated 12 | # and we want to avoid warnings both from the stdlib implementation on 3.11 and 13 | # from the PyPI backport implementation on older Python versions. 14 | # -trs, 13 Sept 2022 15 | import sys 16 | 17 | if sys.version_info >= (3, 11): 18 | from importlib.resources import files as _files, as_file as _as_file 19 | else: 20 | from importlib_resources import files as _files, as_file as _as_file 21 | 22 | from pathlib import Path 23 | from typing import ContextManager 24 | 25 | 26 | def as_file(path: str) -> ContextManager[Path]: 27 | return _as_file(_files(__name__) / path) 28 | -------------------------------------------------------------------------------- /nextstrain/cli/resources/bashrc: -------------------------------------------------------------------------------- 1 | # Don't run if we've already run once for whatever reason, or if PS1 isn't set 2 | # as that implies a non-interactive session. 3 | if [[ -n "${__NEXTSTRAIN_BASHRC:-}" || -z "${PS1:-}" ]]; then 4 | return 0 5 | fi 6 | 7 | __NEXTSTRAIN_BASHRC=1 8 | 9 | # Override PS1, this file's reason for being. 10 | OLDPS1="${PS1:-}" 11 | 12 | if [[ -n "${NEXTSTRAIN_PS1:-}" ]]; then 13 | PS1="$NEXTSTRAIN_PS1" 14 | fi 15 | 16 | # Remember things… 17 | if [[ -n "${NEXTSTRAIN_HISTFILE:-}" ]]; then 18 | HISTFILE="$NEXTSTRAIN_HISTFILE" 19 | HISTCONTROL=ignoredups 20 | HISTSIZE=-1 21 | HISTTIMEFORMAT="%F %H:%M:%S %z" 22 | fi 23 | -------------------------------------------------------------------------------- /nextstrain/cli/runner/ambient.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run commands in the ambient environment, outside of any container image or managed environment. 3 | 4 | The "ambient" runtime allows you to use the Nextstrain CLI with your own ambient 5 | setup, for when you cannot or do not want to have Nextstrain CLI manage its own 6 | runtime. 7 | 8 | .. versionadded:: 1.5.0 9 | .. versionchanged:: 5.0.0 10 | Renamed from "native" to "ambient". 11 | 12 | 13 | .. _ambient-setup: 14 | 15 | Setup 16 | ===== 17 | 18 | You will need to make sure all of the Nextstrain software dependencies 19 | are available locally or "ambiently" on your computer. 20 | 21 | A common way to do this is by manually using `Conda 22 | `__ to manage your own 23 | environment that includes the required software, however you're responsible for 24 | making sure the correct software is installed and kept up-to-date. Our 25 | :doc:`general Nextstrain installation page ` describes more 26 | comprehensively how to do this. 27 | 28 | It is also possible to install the required Nextstrain software `Augur 29 | `__ and `Auspice 30 | `__ and their dependencies manually, 31 | although this is not recommended. 32 | 33 | Once you've installed dependencies, proceed with ``nextstrain setup ambient``. 34 | """ 35 | 36 | import os 37 | import shutil 38 | import sys 39 | from subprocess import CalledProcessError 40 | from typing import Iterable, cast 41 | from .. import config 42 | from ..types import Env, RunnerModule, SetupStatus, SetupTestResults, UpdateStatus 43 | from ..util import capture_output, exec_or_return, runner_name 44 | 45 | 46 | def register_arguments(parser) -> None: 47 | """ 48 | No-op. No arguments necessary. 49 | """ 50 | pass 51 | 52 | 53 | def run(opts, argv, working_volume = None, extra_env: Env = {}, cpus: int = None, memory: int = None) -> int: 54 | if working_volume: 55 | os.chdir(str(working_volume.src)) 56 | 57 | # XXX TODO: In the future we might want to set rlimits based on cpus and 58 | # memory, at least on POSIX systems. 59 | # -trs, 21 May 2020 60 | 61 | return exec_or_return(argv, extra_env) 62 | 63 | 64 | def setup(dry_run: bool = False, force: bool = False) -> SetupStatus: 65 | """ 66 | Not supported. 67 | """ 68 | return None 69 | 70 | 71 | def test_setup() -> SetupTestResults: 72 | def runnable(*argv) -> bool: 73 | try: 74 | capture_output(argv) 75 | return True 76 | except (OSError, CalledProcessError): 77 | return False 78 | 79 | 80 | yield ('snakemake is installed and runnable', 81 | shutil.which("snakemake") is not None and runnable("snakemake", "--version")) 82 | 83 | yield ('augur is installed and runnable', 84 | shutil.which("augur") is not None and runnable("augur", "--version")) 85 | 86 | yield ('auspice is installed and runnable', 87 | shutil.which("auspice") is not None and runnable("auspice", "--version")) 88 | 89 | 90 | def set_default_config() -> None: 91 | """ 92 | Sets ``core.runner`` to this runner's name (``ambient``). 93 | """ 94 | config.set("core", "runner", runner_name(cast(RunnerModule, sys.modules[__name__]))) 95 | 96 | 97 | def update() -> UpdateStatus: 98 | """ 99 | Not supported. Updating the ambient environment isn't reasonably possible. 100 | """ 101 | return None 102 | 103 | 104 | def versions() -> Iterable[str]: 105 | try: 106 | yield capture_output(["augur", "--version"])[0] 107 | except (OSError, CalledProcessError): 108 | pass 109 | 110 | try: 111 | yield "auspice " + capture_output(["auspice", "--version"])[0] 112 | except (OSError, CalledProcessError): 113 | pass 114 | -------------------------------------------------------------------------------- /nextstrain/cli/runner/aws_batch/logs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Log handling for AWS Batch jobs. 3 | """ 4 | 5 | import threading 6 | from botocore.exceptions import ClientError, ConnectionError as BotocoreConnectionError 7 | from typing import Any, Callable, Dict, Generator, MutableSet # noqa: F401 (it's wrong; we use it in a type comment) 8 | from ... import aws 9 | 10 | 11 | LOG_GROUP = "/aws/batch/job" 12 | MAX_FAILURES = 10 13 | 14 | 15 | def fetch_stream(stream: str, start_time: int = None) -> Generator[dict, None, None]: 16 | """ 17 | Fetch all log entries from the named AWS Batch job *stream*. Returns a 18 | generator. 19 | 20 | If the *start_time* argument is given, only entries with timestamps on or 21 | after the given value are fetched. 22 | """ 23 | 24 | client = aws.client_with_default_region("logs") 25 | 26 | log_events = client.get_paginator("filter_log_events") 27 | 28 | query: Dict[str, Any] = { 29 | "logGroupName": LOG_GROUP, 30 | "logStreamNames": [ stream ], 31 | } 32 | 33 | if start_time: 34 | query["startTime"] = start_time 35 | 36 | for page in log_events.paginate(**query): 37 | yield from page.get("events", []) 38 | 39 | 40 | class LogWatcher(threading.Thread): 41 | """ 42 | Monitor an AWS Batch job log stream and call a supplied function (the 43 | *consumer*) with each log entry. 44 | 45 | This is a Thread. Call start() to begin monitoring the log stream and 46 | stop() (and then join()) to stop. 47 | """ 48 | 49 | def __init__(self, stream: str, consumer: Callable[[dict], None]) -> None: 50 | super().__init__(name = "log-watcher", daemon = True) 51 | self.stream = stream 52 | self.consumer = consumer 53 | self.stopped = threading.Event() 54 | 55 | def stop(self) -> None: 56 | """ 57 | Tell the log watcher to cease watching for new logs. 58 | 59 | This method merely signals to the thread that it should stop, so you 60 | must call the thread's join() method afterwards to wait for the thread 61 | to exit. It is an error to call stop() on a thread which isn't alive 62 | (running). 63 | """ 64 | assert self.is_alive(), "Thread not alive" 65 | self.stopped.set() 66 | 67 | def run(self) -> None: 68 | """ 69 | Watch for new logs and pass each log entry to the "consumer" function. 70 | """ 71 | 72 | # Track the last timestamp we see. When we fetch_stream() again on the 73 | # next iteration, we'll start from that timestamp onwards to avoid 74 | # fetching every single page again. The last event or two will be 75 | # still be in the response, but our de-duping will ignore those. 76 | last_timestamp = None 77 | 78 | # Keep track of what log entries we've consumed so that we suppress 79 | # duplicates. Duplicates will arise in our stream due to the way we 80 | # watch for new entries. 81 | consumed = set() # type: MutableSet 82 | 83 | # How many successful vs failed fetch_stream calls. If we consistently see 84 | # failures but we never see a successful attempt, we should raise an exception 85 | # and stop. 86 | success_count = 0 87 | failure_count = 0 88 | 89 | while not self.stopped.wait(0.2): 90 | try: 91 | for entry in fetch_stream(self.stream, start_time = last_timestamp): 92 | if entry["eventId"] not in consumed: 93 | consumed.add(entry["eventId"]) 94 | 95 | last_timestamp = entry["timestamp"] 96 | 97 | self.consumer(entry) 98 | except (ClientError, BotocoreConnectionError): 99 | failure_count += 1 100 | if failure_count > MAX_FAILURES and not success_count: 101 | raise 102 | else: 103 | success_count += 1 104 | -------------------------------------------------------------------------------- /nextstrain/cli/types.py: -------------------------------------------------------------------------------- 1 | """ 2 | Type definitions for internal use. 3 | """ 4 | 5 | import argparse 6 | import builtins 7 | import sys 8 | from pathlib import Path 9 | from typing import Any, Callable, Iterable, List, Mapping, Optional, Protocol, Tuple, Union, TYPE_CHECKING, runtime_checkable 10 | # TODO: Use typing.TypeAlias once Python 3.10 is the minimum supported version. 11 | from typing_extensions import TypeAlias 12 | 13 | # Import concrete types from our other modules only during type checking to 14 | # avoid import cycles during runtime. 15 | if TYPE_CHECKING: 16 | from .authn import User 17 | from .volume import NamedVolume 18 | from .url import URL, Origin 19 | 20 | # Re-export EllipsisType so we can paper over its absence from older Pythons 21 | if sys.version_info >= (3, 10): 22 | from types import EllipsisType 23 | else: 24 | EllipsisType: TypeAlias = 'builtins.ellipsis' 25 | 26 | """ 27 | An immutable mapping of (*name*, *value*) pairs representing a set of 28 | additional environment variables to overlay on the current environment (e.g. 29 | when executing a subprocess). 30 | 31 | Each (*name*, *value*) pair represents a single environment variable. 32 | 33 | A *value* of ``None`` indicates the positive absence of *name* (e.g. it is to 34 | be removed if present). 35 | """ 36 | Env = Mapping['EnvName', 'EnvValue'] 37 | EnvItem = Tuple['EnvName', 'EnvValue'] 38 | EnvName = str 39 | EnvValue = Union[str, None] 40 | 41 | Options = argparse.Namespace 42 | 43 | SetupStatus = Optional[bool] 44 | 45 | SetupTestResults = Iterable['SetupTestResult'] 46 | SetupTestResult = Tuple[str, 'SetupTestResultStatus'] 47 | SetupTestResultStatus: TypeAlias = Union[bool, None, EllipsisType] 48 | 49 | UpdateStatus = Optional[bool] 50 | 51 | # Cleaner-reading type annotations for boto3 S3 objects, which maybe can be 52 | # improved later. The actual types are generated at runtime in 53 | # boto3.resources.factory, which means we can't use them here easily. :( 54 | S3Bucket = Any 55 | S3Object = Any 56 | 57 | 58 | @runtime_checkable 59 | class RunnerModule(Protocol): 60 | @staticmethod 61 | def register_arguments(parser: argparse.ArgumentParser) -> None: ... 62 | 63 | @staticmethod 64 | def run(opts: Options, 65 | argv: List[str], 66 | working_volume: Optional['NamedVolume'], 67 | extra_env: Env, 68 | cpus: Optional[int], 69 | memory: Optional[int]) -> int: 70 | ... 71 | 72 | @staticmethod 73 | def setup(dry_run: bool = False, force: bool = False) -> SetupStatus: ... 74 | 75 | @staticmethod 76 | def test_setup() -> SetupTestResults: ... 77 | 78 | @staticmethod 79 | def set_default_config() -> None: ... 80 | 81 | @staticmethod 82 | def update() -> UpdateStatus: ... 83 | 84 | @staticmethod 85 | def versions() -> Iterable[str]: ... 86 | 87 | 88 | class RemoteModule(Protocol): 89 | @staticmethod 90 | def upload(url: 'URL', local_files: List[Path], dry_run: bool = False) -> Iterable[Tuple[Path, str]]: ... 91 | 92 | @staticmethod 93 | def download(url: 'URL', local_path: Path, recursively: bool = False, dry_run: bool = False) -> Iterable[Tuple[str, Path]]: ... 94 | 95 | @staticmethod 96 | def ls(url: 'URL') -> Iterable[str]: ... 97 | 98 | @staticmethod 99 | def delete(url: 'URL', recursively: bool = False, dry_run: bool = False) -> Iterable[str]: ... 100 | 101 | @staticmethod 102 | def current_user(origin: 'Origin') -> Optional['User']: ... 103 | 104 | @staticmethod 105 | def login(origin: 'Origin', credentials: Optional[Callable[[], Tuple[str, str]]] = None) -> 'User': ... 106 | 107 | @staticmethod 108 | def renew(origin: 'Origin') -> Optional['User']: ... 109 | 110 | @staticmethod 111 | def logout(origin: 'Origin'): ... 112 | -------------------------------------------------------------------------------- /nextstrain/cli/url.py: -------------------------------------------------------------------------------- 1 | """ 2 | URL handling. 3 | 4 | Extended forms of :mod:`urllib.parse` with an API that fits a little better. 5 | """ 6 | import os 7 | from typing import List, Mapping, NewType, Optional, Union 8 | from urllib.parse import urlsplit, SplitResult, parse_qs as parse_query, urlencode as construct_query, urljoin 9 | 10 | 11 | Origin = NewType("Origin", str) 12 | 13 | 14 | class URL(SplitResult): 15 | """ 16 | A parsed URL. 17 | 18 | Combines the parsing functionality of :func:`urllib.parse.urlsplit` with 19 | its resulting data tuple of :cls:`urllib.parse.SplitResult`. 20 | 21 | May be constructed only from a single *url* string. 22 | 23 | >>> u = URL("https://example.com:123/abc/def?x=y#z") 24 | >>> u 25 | URL(scheme='https', netloc='example.com:123', path='/abc/def', query='x=y', fragment='z') 26 | 27 | >>> u.hostname 28 | 'example.com' 29 | 30 | >>> u.origin 31 | 'https://example.com:123' 32 | 33 | Standard named tuple methods are available. 34 | 35 | >>> u = u._replace(netloc="example.org") 36 | >>> u 37 | URL(scheme='https', netloc='example.org', path='/abc/def', query='x=y', fragment='z') 38 | 39 | >>> u.origin 40 | 'https://example.org' 41 | 42 | An optional *base* URL string or :cls:`URL` is accepted and combined with 43 | *url* using :func:`urllib.parse.urljoin` before parsing. 44 | 45 | >>> URL("def?x=y#z", "https://example.com:123/abc/") 46 | URL(scheme='https', netloc='example.com:123', path='/abc/def', query='x=y', fragment='z') 47 | 48 | >>> URL("https://example.com/abc/def?x=y#z", "https://example.com:123/xyz?n=42#abc") 49 | URL(scheme='https', netloc='example.com', path='/abc/def', query='x=y', fragment='z') 50 | """ 51 | __slots__ = () 52 | 53 | def __new__(cls, url: str, base: Union[str, 'URL'] = None) -> 'URL': 54 | return super().__new__(cls, *urlsplit(urljoin(str(base), url) if base else url)) 55 | 56 | # This is for the type checkers, which otherwise consider URL.__init__ to 57 | # have a signature based on SplitResult.__new__ instead of our own __new__. 58 | # It's not clear to me *why* they do that, but this hint helps sort it out. 59 | # -trs, 16 Nov 2023 60 | def __init__(self, url: str, base: Union[str, 'URL'] = None) -> None: ... 61 | 62 | def __str__(self) -> str: 63 | return self.geturl() 64 | 65 | @property 66 | def origin(self) -> Optional[Origin]: 67 | """ 68 | The URL's origin, in the `web sense 69 | `__. 70 | 71 | >>> u = URL("https://example.com:123/abc/def?x=y#z") 72 | >>> u 73 | URL(scheme='https', netloc='example.com:123', path='/abc/def', query='x=y', fragment='z') 74 | >>> u.origin 75 | 'https://example.com:123' 76 | 77 | Origin is ``None`` if unless both :meth:`.scheme` and :meth:`.netloc` 78 | are present. 79 | 80 | >>> u = URL("/a/b/c") 81 | >>> u 82 | URL(scheme='', netloc='', path='/a/b/c', query='', fragment='') 83 | >>> u.origin is None 84 | True 85 | 86 | >>> u = URL("//example.com/a/b/c") 87 | >>> u 88 | URL(scheme='', netloc='example.com', path='/a/b/c', query='', fragment='') 89 | >>> u.origin is None 90 | True 91 | """ 92 | if self.scheme and self.netloc: 93 | return Origin(self.scheme + "://" + self.netloc) 94 | else: 95 | return None 96 | 97 | @property 98 | def query_fields(self) -> Mapping[str, List[str]]: 99 | """ 100 | The URL's :attr:`.query` string parsed into a mapping of name-value fields. 101 | 102 | Values are always lists, even if the field name is not repeated in the 103 | query string. 104 | 105 | >>> URL("?x=y&a=1&b=2&x=z").query_fields 106 | {'x': ['y', 'z'], 'a': ['1'], 'b': ['2']} 107 | 108 | Fields without values are treated as having an empty string value. 109 | 110 | >>> URL("?x=&y&z=1").query_fields 111 | {'x': [''], 'y': [''], 'z': ['1']} 112 | """ 113 | return parse_query(self.query, keep_blank_values = True) 114 | 115 | 116 | def query(fields: Mapping[str, Union[str, List[str]]]) -> str: 117 | """ 118 | Convert query *fields* to a query string. 119 | 120 | >>> query({'x': ['y', 'z'], 'a': ['1'], 'b': ['2']}) 121 | 'x=y&x=z&a=1&b=2' 122 | 123 | >>> query({'x': ['y', 'z'], 'a': '123', 'b': '456'}) 124 | 'x=y&x=z&a=123&b=456' 125 | 126 | >>> query({'x': [''], 'y': '', 'z': '123'}) 127 | 'x=&y=&z=123' 128 | """ 129 | return construct_query(fields, doseq = True, encoding = "utf-8") 130 | 131 | 132 | NEXTSTRAIN_DOT_ORG = URL( 133 | os.environ.get("NEXTSTRAIN_DOT_ORG") 134 | or "https://nextstrain.org") 135 | -------------------------------------------------------------------------------- /nextstrain/cli/volume.py: -------------------------------------------------------------------------------- 1 | """ 2 | Volumes map well-known names to a source path. 3 | """ 4 | 5 | import argparse 6 | from typing import NamedTuple 7 | from pathlib import Path 8 | 9 | 10 | class NamedVolume(NamedTuple): 11 | name: str 12 | src: Path 13 | dir: bool = True 14 | writable: bool = True 15 | 16 | 17 | def store_volume(volume_name): 18 | """ 19 | Generates and returns an argparse.Action subclass for storing named volume 20 | tuples. 21 | 22 | Multiple argparse arguments can use this to cooperatively accept source 23 | path definitions for named volumes. 24 | 25 | Each named volume is stored as a NamedTuple (name, src). The tuple is 26 | stored on the options object as an element in a shared list of volumes, 27 | accessible via the "volumes" attribute on the options object. 28 | 29 | For convenient path manipulation and testing, the "src" value is stored as 30 | a Path object. 31 | """ 32 | class store(argparse.Action): 33 | def __call__(self, parser, namespace, values, option_strings = None): 34 | # Add the new volume to the list of volumes 35 | volumes = getattr(namespace, "volumes", []) 36 | new_volume = NamedVolume(volume_name, Path(values)) if values else None 37 | setattr(namespace, "volumes", [*volumes, new_volume]) 38 | 39 | return store 40 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /pyrightconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "pythonVersion": "3.8", 3 | "include": ["nextstrain"], 4 | "ignore": [ 5 | "nextstrain/cli/markdown.py", 6 | "nextstrain/cli/rst/sphinx.py", 7 | "/**/node_modules/pyright/dist/typeshed-fallback/", 8 | "c:/**/node_modules/pyright/dist/typeshed-fallback/" 9 | ], 10 | "reportMissingImports": false, 11 | "reportMissingModuleSource": true, 12 | "reportUnusedFunction": true, 13 | "reportWildcardImportFromLibrary": true, 14 | "reportUntypedFunctionDecorator": true, 15 | "reportUntypedClassDecorator": true, 16 | "reportUntypedBaseClass": true, 17 | "reportUntypedNamedTuple": true, 18 | "reportPrivateUsage": true, 19 | "reportInvalidStringEscapeSequence": true, 20 | "reportInvalidTypeVarUse": true, 21 | "reportAssertAlwaysTrue": true, 22 | "reportSelfClsParameterName": true, 23 | "reportInvalidStubStatement": true, 24 | "reportIncompleteStub": true, 25 | "reportUnnecessaryCast": true, 26 | "reportUnnecessaryTypeIgnoreComment": true, 27 | "reportUnsupportedDunderAll": true, 28 | "strictParameterNoneValue": false 29 | } 30 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = --doctest-modules 3 | testpaths = nextstrain tests 4 | python_files = *.py 5 | 6 | # Avoid catching test_setup() functions. I've always disliked the name-based 7 | # test discovery anyway. 8 | python_functions = pytest_* 9 | 10 | # Turn warnings into errors. Warnings are signs that something needs 11 | # attention, even if that means choosing to add an explicit ignore filter rule 12 | # for them. Last matching filter wins. 13 | filterwarnings = 14 | error 15 | -------------------------------------------------------------------------------- /tests/command-build.py: -------------------------------------------------------------------------------- 1 | from nextstrain.cli import make_parser 2 | 3 | 4 | def pytest_build_download_options(): 5 | parser = make_parser() 6 | 7 | opts = parser.parse_args(["build", "."]) 8 | assert opts.download is True 9 | 10 | opts = parser.parse_args(["build", "--no-download", "."]) 11 | assert opts.download is False 12 | 13 | opts = parser.parse_args(["build", "--download", "x", "."]) 14 | assert opts.download == ["x"] 15 | 16 | opts = parser.parse_args(["build", "--download", "x", "--download", "y", "."]) 17 | assert opts.download == ["x", "y"] 18 | 19 | opts = parser.parse_args(["build", "--exclude-from-download", "z", "."]) 20 | assert opts.download == ["!z"] 21 | 22 | opts = parser.parse_args(["build", "--exclude-from-download", "z", "--exclude-from-download", "a", "."]) 23 | assert opts.download == ["!z", "!a"] 24 | 25 | opts = parser.parse_args(["build", "--download", "y", "--exclude-from-download", "z", "."]) 26 | assert opts.download == ["y", "!z"] 27 | 28 | opts = parser.parse_args(["build", "--download", "y", "--download", "!z", "."]) 29 | assert opts.download == ["y", "!z"] 30 | -------------------------------------------------------------------------------- /tests/cram.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | from pathlib import Path 4 | from subprocess import run 5 | 6 | testsdir = Path(__file__).resolve().parent 7 | topdir = testsdir.parent 8 | 9 | @pytest.mark.skipif(os.name != "posix", reason = "cram requires a POSIX platform") 10 | @pytest.mark.parametrize("testfile", testsdir.glob("*.cram"), ids = str) 11 | def pytest_cram(testfile): 12 | # Check the exit status ourselves for nicer test output on failure 13 | result = run(["cram", testfile], cwd = topdir) 14 | assert result.returncode == 0, "cram exited with errors" 15 | -------------------------------------------------------------------------------- /tests/data/home/.gitignore: -------------------------------------------------------------------------------- 1 | /lock 2 | -------------------------------------------------------------------------------- /tests/data/home/config: -------------------------------------------------------------------------------- 1 | # Nextstrain CLI config file used in tests, via NEXTSTRAIN_HOME=tests/data/home/. 2 | 3 | [pathogen from-config] 4 | default_version = v1 5 | -------------------------------------------------------------------------------- /tests/data/home/pathogens/with-implicit-default/1.2.3=GEXDELRT/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nextstrain/cli/b5af91bbe581a54730d8d4873db8d817633c4f89/tests/data/home/pathogens/with-implicit-default/1.2.3=GEXDELRT/.gitignore -------------------------------------------------------------------------------- /tests/data/home/pathogens/with-no-implicit-default/1.2.3=GEXDELRT/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nextstrain/cli/b5af91bbe581a54730d8d4873db8d817633c4f89/tests/data/home/pathogens/with-no-implicit-default/1.2.3=GEXDELRT/.gitignore -------------------------------------------------------------------------------- /tests/data/home/pathogens/with-no-implicit-default/4.5.6=GQXDKLRW/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nextstrain/cli/b5af91bbe581a54730d8d4873db8d817633c4f89/tests/data/home/pathogens/with-no-implicit-default/4.5.6=GQXDKLRW/.gitignore -------------------------------------------------------------------------------- /tests/data/markdown/embed-images-001.md: -------------------------------------------------------------------------------- 1 | ![image node](image.png) 2 | ![image node with title](image.png "title") 3 | ---8<--- 4 | ![image node][image.png] 5 | ![image node with title](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4ofAVAAQhAg5PywneAAAAAElFTkSuQmCC "title") 6 | 7 | 8 | [image.png]: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4ofAVAAQhAg5PywneAAAAAElFTkSuQmCC 9 | -------------------------------------------------------------------------------- /tests/data/markdown/embed-images-002.md: -------------------------------------------------------------------------------- 1 | ![full reference][logo] 2 | [logo]: image.png 3 | 4 | ![collapsed reference][] 5 | [collapsed reference]: image.png 6 | 7 | ![shortcut reference] 8 | [shortcut reference]: image.png 9 | 10 | ![label ≠ identifier][case-INSENSITIVE] 11 | [case-insensitive]: image.png 12 | 13 | ![does not exist](does-not-exist.png) 14 | 15 | ![undefined reference][no-definition] 16 | ---8<--- 17 | ![full reference][logo] 18 | 19 | 20 | ![collapsed reference][] 21 | 22 | 23 | ![shortcut reference] 24 | 25 | 26 | ![label ≠ identifier][case-INSENSITIVE] 27 | 28 | 29 | ![does not exist](does-not-exist.png) 30 | 31 | ![undefined reference][no-definition] 32 | 33 | 34 | [logo]: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4ofAVAAQhAg5PywneAAAAAElFTkSuQmCC 35 | [collapsed reference]: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4ofAVAAQhAg5PywneAAAAAElFTkSuQmCC 36 | [shortcut reference]: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4ofAVAAQhAg5PywneAAAAAElFTkSuQmCC 37 | [case-insensitive]: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4ofAVAAQhAg5PywneAAAAAElFTkSuQmCC 38 | -------------------------------------------------------------------------------- /tests/data/markdown/embed-images-003.md: -------------------------------------------------------------------------------- 1 | ![alt](image.png) 2 | 3 | ```auspiceMainDisplayMarkdown 4 | nested markdown 5 | 6 | ![nested image](image.png) 7 | 8 | kinda strange 9 | ``` 10 | 11 | but here we are 12 | ---8<--- 13 | ![alt][image.png] 14 | 15 | ```auspiceMainDisplayMarkdown 16 | nested markdown 17 | 18 | ![nested image][image.png] 19 | 20 | kinda strange 21 | 22 | 23 | [image.png]: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4ofAVAAQhAg5PywneAAAAAElFTkSuQmCC 24 | ``` 25 | 26 | but here we are 27 | 28 | 29 | [image.png]: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4ofAVAAQhAg5PywneAAAAAElFTkSuQmCC 30 | -------------------------------------------------------------------------------- /tests/data/markdown/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nextstrain/cli/b5af91bbe581a54730d8d4873db8d817633c4f89/tests/data/markdown/image.png -------------------------------------------------------------------------------- /tests/data/markdown/roundtrip-001.md: -------------------------------------------------------------------------------- 1 | # Header 2 | 3 | Here's some text and an image 4 | 5 | ![some text](image.png) 6 | ![some text](image.png "a title") 7 | ![some text]("a bogus title") 8 | 9 | and then some more text and another image 10 | 11 | ![some [other] text](image.png "some ) title") 12 | 13 | ```auspiceMainDisplayMarkdown 14 | right hand panel content 15 | 16 | ![some image](image.png "title \\" foo") 17 | 18 | more content 19 | ``` 20 | 21 | now some more advanced images 22 | 23 | ![full][full] 24 | ![collapsed][] 25 | ![shortcut] 26 | 27 | that use references 28 | 29 | [full]: image.png 30 | [collapsed]: image.png 31 | [shortcut]: image.png 32 | [unused]: https://example.com 33 | 34 | another code fence: 35 | 36 | ```python 37 | print("hello markdown") 38 | ``` 39 | 40 | and a code block: 41 | 42 | foo 43 | bar 44 | baz 45 | 46 | and then some weirder image forms: 47 | 48 | ![abc 49 | def][foo] 50 | 51 | ![abc 52 | def][ 53 | foo] 54 | 55 | ![alt]( 56 | https://secure.gravatar.com/avatar/c2f056279f6573478e3b48e95b9b338b 57 | "abc 58 | def") 59 | -------------------------------------------------------------------------------- /tests/data/pathogen-repo/ingest/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nextstrain/cli/b5af91bbe581a54730d8d4873db8d817633c4f89/tests/data/pathogen-repo/ingest/.gitignore -------------------------------------------------------------------------------- /tests/data/pathogen-repo/nextstrain-pathogen.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nextstrain/cli/b5af91bbe581a54730d8d4873db8d817633c4f89/tests/data/pathogen-repo/nextstrain-pathogen.yaml -------------------------------------------------------------------------------- /tests/data/pathogen-repo/phylogenetic/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nextstrain/cli/b5af91bbe581a54730d8d4873db8d817633c4f89/tests/data/pathogen-repo/phylogenetic/.gitignore -------------------------------------------------------------------------------- /tests/doc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | from pathlib import Path 4 | from subprocess import run 5 | 6 | topdir = Path(__file__).resolve().parent.parent 7 | 8 | generator_progs = [ 9 | "devel/generate-command-doc", 10 | "devel/generate-changes-doc", 11 | ] 12 | 13 | @pytest.mark.skipif(os.name != "posix", reason = "doc generation requires a POSIX platform") 14 | @pytest.mark.parametrize("prog", generator_progs, ids = str) 15 | def pytest_generated_doc(prog): 16 | # Check the exit status ourselves for nicer test output on failure 17 | result = run([topdir / prog, "--check", "--diff"]) 18 | assert result.returncode == 0, f"{result.args!r} exited with errors" 19 | -------------------------------------------------------------------------------- /tests/env: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Shebang is not strictly necessary—we only source this file—but it lets our 3 | # GitHub Action for ShellCheck to find it properly. 4 | 5 | export NEXTSTRAIN_RST_STRICT=1 6 | export NEXTSTRAIN_HOME=tests/data/home/ 7 | 8 | # Must do this early, before nextstrain.cli.command.view is loaded 9 | export BROWSER=echo 10 | -------------------------------------------------------------------------------- /tests/flake8.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from subprocess import run 3 | 4 | topdir = Path(__file__).resolve().parent.parent 5 | 6 | def pytest_flake8(): 7 | # Check the exit status ourselves for nicer test output on failure 8 | result = run(["flake8"], cwd = topdir) 9 | assert result.returncode == 0, "flake8 exited with errors" 10 | -------------------------------------------------------------------------------- /tests/help.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | from nextstrain.cli import make_parser 4 | from nextstrain.cli.argparse import walk_commands 5 | from subprocess import run 6 | 7 | 8 | def generate_commands(): 9 | for command, parser in walk_commands(make_parser()): 10 | has_extended_help = any( 11 | any(opt == "--help-all" for opt in action.option_strings) 12 | for action in parser._actions) 13 | 14 | yield (*command, "--help-all" if has_extended_help else "--help") 15 | 16 | 17 | commands = list(generate_commands()) 18 | 19 | 20 | @pytest.mark.parametrize("command", commands, ids = lambda command: " ".join(command)) 21 | def pytest_help(command): 22 | # Check the exit status ourselves for nicer test output on failure 23 | result = run(command) 24 | assert result.returncode == 0, f"{command} exited with error" 25 | 26 | result = run(command, env = {**os.environ, "NEXTSTRAIN_RST_STRICT": "yes"}) 27 | assert result.returncode == 0, f"{command} exited with error with strict rST conversion" 28 | -------------------------------------------------------------------------------- /tests/markdown.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import re 3 | from nextstrain.cli.markdown import parse, generate, embed_images 4 | from pathlib import Path 5 | 6 | testsdir = Path(__file__).resolve(strict = True).parent 7 | topdir = testsdir.parent 8 | datadir = testsdir / "data/markdown/" 9 | 10 | def cases(pattern): 11 | for case in datadir.glob(pattern): 12 | yield pytest.param( 13 | case, 14 | id = str(case.relative_to(topdir))) 15 | 16 | 17 | @pytest.mark.parametrize("case", cases("roundtrip-*.md")) 18 | def pytest_markdown_roundtrip(case): 19 | markdown = case.read_text() 20 | assert generate(parse(markdown)) == markdown 21 | 22 | 23 | @pytest.mark.parametrize("case", cases("embed-images-*.md")) 24 | def pytest_markdown_embed_images(case): 25 | markdown, expected = split_input_expected(case.read_text()) 26 | result = generate(embed_images(parse(markdown), case.parent)) 27 | assert result == expected 28 | 29 | 30 | def split_input_expected(markdown): 31 | return re.split(r'(?m)^---+8<---+$\n', markdown, 1) 32 | -------------------------------------------------------------------------------- /tests/open_browser.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test nextstrain.cli.view.open_browser() works under different multiprocessing 3 | start methods. 4 | 5 | This file is loaded in the global pytest process, but each test function is 6 | marked to run in its own (forked) subprocess, so we modify relevant global 7 | state in each function. 8 | """ 9 | import multiprocessing 10 | import os 11 | import pytest 12 | from nextstrain.cli.command.view import open_browser 13 | 14 | if os.name != "posix": 15 | pytest.skip("@pytest.mark.forked requires a POSIX platform", allow_module_level = True) 16 | 17 | 18 | @pytest.mark.forked 19 | def pytest_open_browser_fork(): 20 | multiprocessing.set_start_method("fork") 21 | assert open_browser("https://nextstrain.org") 22 | 23 | 24 | @pytest.mark.forked 25 | def pytest_open_browser_spawn(): 26 | multiprocessing.set_start_method("spawn") 27 | assert open_browser("https://nextstrain.org") 28 | -------------------------------------------------------------------------------- /tests/pyright.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pathlib import Path 3 | from shutil import which 4 | from subprocess import run 5 | 6 | topdir = Path(__file__).resolve().parent.parent 7 | 8 | pyright = which("pyright") 9 | npx = which("npx") 10 | 11 | if pyright: 12 | pyright = [pyright] 13 | elif npx: 14 | pyright = [npx, "pyright@>=1.1.396"] 15 | else: 16 | pyright = None 17 | 18 | @pytest.mark.skipif(not pyright, reason = "pyright is not available") 19 | def pytest_pyright(): 20 | # Check the exit status ourselves for nicer test output on failure 21 | result = run(pyright, cwd = topdir) 22 | assert result.returncode == 0, "pyright exited with errors" 23 | -------------------------------------------------------------------------------- /tests/remote.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from nextstrain.cli.remote.nextstrain_dot_org import organize_files 3 | 4 | 5 | def pytest_organize_files(): 6 | assert ( 7 | organize_files({ 8 | Path("ncov_open_global.json"), 9 | Path("ncov_open_global_root-sequence.json"), 10 | Path("ncov_open_global_tip-frequencies.json"), 11 | Path("ncov_open_north-america.json"), 12 | Path("ncov_open_north-america_root-sequence.json"), 13 | Path("ncov_open_north-america_tip-frequencies.json"), 14 | Path("ncov_gisaid_global.json"), 15 | }) 16 | == 17 | ( 18 | { 19 | "ncov/open/global": { 20 | "application/vnd.nextstrain.dataset.main+json": Path("ncov_open_global.json"), 21 | "application/vnd.nextstrain.dataset.root-sequence+json": Path("ncov_open_global_root-sequence.json"), 22 | "application/vnd.nextstrain.dataset.tip-frequencies+json": Path("ncov_open_global_tip-frequencies.json"), 23 | }, 24 | "ncov/open/north-america": { 25 | "application/vnd.nextstrain.dataset.main+json": Path("ncov_open_north-america.json"), 26 | "application/vnd.nextstrain.dataset.root-sequence+json": Path("ncov_open_north-america_root-sequence.json"), 27 | "application/vnd.nextstrain.dataset.tip-frequencies+json": Path("ncov_open_north-america_tip-frequencies.json"), 28 | }, 29 | "ncov/gisaid/global": { 30 | "application/vnd.nextstrain.dataset.main+json": Path("ncov_gisaid_global.json"), 31 | }, 32 | }, 33 | {}, 34 | [], 35 | ) 36 | ) 37 | 38 | assert ( 39 | organize_files({ 40 | Path("auspice/A.json"), 41 | Path("auspice/A_root-sequence.json"), 42 | Path("auspice/A_tip-frequencies.json"), 43 | Path("auspice/A_B.json"), 44 | Path("auspice/A_B_root-sequence.json"), 45 | Path("narratives/hello_world.md"), 46 | Path("narratives/hello_universe.md"), 47 | Path("group-overview.md"), 48 | Path("group-logo.png"), 49 | }) 50 | == 51 | ( 52 | { 53 | "A": { 54 | "application/vnd.nextstrain.dataset.main+json": Path("auspice/A.json"), 55 | "application/vnd.nextstrain.dataset.root-sequence+json": Path("auspice/A_root-sequence.json"), 56 | "application/vnd.nextstrain.dataset.tip-frequencies+json": Path("auspice/A_tip-frequencies.json"), 57 | }, 58 | "A/B": { 59 | "application/vnd.nextstrain.dataset.main+json": Path("auspice/A_B.json"), 60 | "application/vnd.nextstrain.dataset.root-sequence+json": Path("auspice/A_B_root-sequence.json"), 61 | }, 62 | }, 63 | { 64 | "hello/world": { 65 | "text/vnd.nextstrain.narrative+markdown": Path("narratives/hello_world.md"), 66 | }, 67 | "hello/universe": { 68 | "text/vnd.nextstrain.narrative+markdown": Path("narratives/hello_universe.md"), 69 | }, 70 | }, 71 | [ 72 | Path("group-logo.png"), 73 | Path("group-overview.md"), 74 | ], 75 | ) 76 | ) 77 | 78 | assert ( 79 | organize_files({ 80 | Path("A_tree.json"), 81 | Path("A_meta.json"), 82 | Path("A_tip-frequencies.json"), 83 | Path("A_B.md"), 84 | Path("A_C.md"), 85 | }) 86 | == 87 | ( 88 | { 89 | "A": { 90 | "application/vnd.nextstrain.dataset.tip-frequencies+json": Path("A_tip-frequencies.json"), 91 | }, 92 | }, 93 | { 94 | "A/B": { 95 | "text/vnd.nextstrain.narrative+markdown": Path("A_B.md"), 96 | }, 97 | "A/C": { 98 | "text/vnd.nextstrain.narrative+markdown": Path("A_C.md"), 99 | }, 100 | }, 101 | [ 102 | Path("A_meta.json"), 103 | Path("A_tree.json"), 104 | ], 105 | ) 106 | ) 107 | -------------------------------------------------------------------------------- /tests/version.cram: -------------------------------------------------------------------------------- 1 | Cram setup. 2 | 3 | $ source "$TESTDIR"/env 4 | 5 | Version command. 6 | 7 | $ nextstrain version 8 | Nextstrain CLI [0-9]+[.][0-9]+[.][0-9]+\S* (re) 9 | 10 | $ python3 -Xnextstrain-cli-is-standalone -m nextstrain.cli version 11 | Nextstrain CLI [0-9]+[.][0-9]+[.][0-9]+\S* \(standalone\) (re) 12 | 13 | Conventional --version flag. 14 | 15 | $ nextstrain --version 16 | Nextstrain CLI [0-9]+[.][0-9]+[.][0-9]+\S* (re) 17 | 18 | $ python3 -Xnextstrain-cli-is-standalone -m nextstrain.cli --version 19 | Nextstrain CLI [0-9]+[.][0-9]+[.][0-9]+\S* \(standalone\) (re) 20 | --------------------------------------------------------------------------------