├── panoptes_cli ├── __init__.py ├── commands │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ └── test_project.py │ ├── inaturalist.py │ ├── info.py │ ├── configure.py │ ├── user.py │ ├── project.py │ ├── workflow.py │ ├── subject.py │ └── subject_set.py └── scripts │ ├── __init__.py │ └── panoptes.py ├── .hound.yml ├── Dockerfile.stable ├── Dockerfile.stable2 ├── .github ├── dependabot.yml └── workflows │ ├── run_tests_CI.yml │ ├── publish-to-pypi.yml │ ├── publish-to-test-pypi.yml │ └── codeql-analysis.yml ├── Dockerfile.dev ├── Dockerfile.dev2 ├── .gitignore ├── setup.py ├── docker-compose.yml ├── CONTRIBUTING.md ├── CHANGELOG.md ├── LICENSE └── README.md /panoptes_cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /panoptes_cli/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /panoptes_cli/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.hound.yml: -------------------------------------------------------------------------------- 1 | python: 2 | enabled: true 3 | -------------------------------------------------------------------------------- /panoptes_cli/commands/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Dockerfile.stable: -------------------------------------------------------------------------------- 1 | FROM python:3-alpine 2 | 3 | RUN apk add --no-cache libmagic 4 | 5 | RUN pip install panoptescli 6 | 7 | CMD [ "panoptes" ] 8 | -------------------------------------------------------------------------------- /Dockerfile.stable2: -------------------------------------------------------------------------------- 1 | FROM python:2.7-alpine 2 | 3 | RUN apk add --no-cache libmagic 4 | RUN pip install panoptescli 5 | 6 | CMD [ "panoptes" ] 7 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "10:00" 8 | open-pull-requests-limit: 10 9 | -------------------------------------------------------------------------------- /Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM python:3.9-alpine 2 | 3 | WORKDIR /usr/src/panoptes-cli 4 | 5 | RUN apk --no-cache add git libmagic 6 | RUN pip install git+https://github.com/zooniverse/panoptes-python-client.git 7 | 8 | COPY . . 9 | 10 | RUN pip install . 11 | 12 | CMD [ "panoptes" ] 13 | -------------------------------------------------------------------------------- /Dockerfile.dev2: -------------------------------------------------------------------------------- 1 | FROM python:2-alpine 2 | 3 | WORKDIR /usr/src/panoptes-cli 4 | 5 | RUN apk --no-cache add git libmagic 6 | RUN pip install git+https://github.com/zooniverse/panoptes-python-client.git 7 | 8 | COPY . . 9 | 10 | RUN pip install . 11 | 12 | CMD [ "panoptes" ] 13 | -------------------------------------------------------------------------------- /.github/workflows/run_tests_CI.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - master 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Set up Python 3.9 16 | uses: actions/setup-python@v2 17 | with: 18 | python-version: 3.9 19 | - uses: actions/cache@v3 20 | with: 21 | path: ${{ env.pythonLocation }} 22 | key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} 23 | - name: Install dependencies 24 | run: | 25 | pip install -U pip 26 | pip install -U . 27 | - name: Run tests 28 | run: python -m unittest discover 29 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPi 2 | 3 | on: workflow_dispatch 4 | 5 | jobs: 6 | build-and-publish: 7 | name: Build python package and publish to PyPi 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@master 11 | - name: Set up Python 3.9 12 | uses: actions/setup-python@v2 13 | with: 14 | python-version: 3.9 15 | - name: Install dependencies 16 | run: | 17 | python -m pip install --upgrade pip 18 | pip install setuptools wheel 19 | - name: Build 20 | run: python setup.py sdist bdist_wheel 21 | - name: Publish to PyPi 22 | uses: pypa/gh-action-pypi-publish@release/v1 23 | with: 24 | user: __token__ 25 | password: ${{ secrets.PYPI_API_TOKEN }} 26 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-test-pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish to Test PyPi 2 | 3 | on: workflow_dispatch 4 | 5 | jobs: 6 | build-and-publish: 7 | name: Build python package and publish to Test PyPi 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@master 11 | - name: Set up Python 3.9 12 | uses: actions/setup-python@v2 13 | with: 14 | python-version: 3.9 15 | - name: Install dependencies 16 | run: | 17 | python -m pip install --upgrade pip 18 | pip install setuptools wheel 19 | - name: Build 20 | run: python setup.py sdist bdist_wheel 21 | - name: Publish to Test PyPi 22 | uses: pypa/gh-action-pypi-publish@release/v1 23 | with: 24 | user: __token__ 25 | password: ${{ secrets.TEST_PYPI_API_TOKEN }} 26 | repository_url: https://test.pypi.org/legacy/ 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | .vscode/ -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | from pathlib import Path 3 | this_directory = Path(__file__).parent 4 | long_description = (this_directory / "README.md").read_text() 5 | 6 | setup( 7 | name='panoptescli', 8 | version='1.2.1', 9 | url='https://github.com/zooniverse/panoptes-cli', 10 | author='Adam McMaster / Zooniverse', 11 | author_email='contact@zooniverse.org', 12 | description=( 13 | 'A command-line client for Panoptes, the API behind the Zooniverse' 14 | ), 15 | long_description=long_description, 16 | long_description_content_type='text/markdown', 17 | packages=find_packages(), 18 | include_package_data=True, 19 | install_requires=[ 20 | 'Click>=6.7,<8.2', 21 | 'PyYAML>=5.1,<6.1', 22 | 'panoptes-client>=1.7,<2.0', 23 | 'humanize>=0.5.1,<4.8', 24 | 'pathvalidate>=0.29.0,<2.6', 25 | ], 26 | entry_points=''' 27 | [console_scripts] 28 | panoptes=panoptes_cli.scripts.panoptes:cli 29 | ''', 30 | ) 31 | -------------------------------------------------------------------------------- /panoptes_cli/commands/inaturalist.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from panoptes_cli.scripts.panoptes import cli 4 | from panoptes_client import Inaturalist 5 | 6 | 7 | @cli.group() 8 | def inaturalist(): 9 | """Contains commands related to iNaturalist integration""" 10 | pass 11 | 12 | 13 | @inaturalist.command(name='import-observations') 14 | @click.option( 15 | '--taxon-id', 16 | help=( 17 | "iNaturalist Taxon ID of the taxa you want to import." 18 | ), 19 | required=True, 20 | type=int, 21 | ) 22 | @click.option( 23 | '--subject-set-id', 24 | help=( 25 | "ID of the Zooniverse subject set to import into." 26 | ), 27 | required=True, 28 | type=int, 29 | ) 30 | @click.option( 31 | '--updated-since', 32 | help=( 33 | "Optional: Import observations since this timestamp" 34 | ), 35 | required=False, 36 | ) 37 | def import_observations(taxon_id, subject_set_id, updated_since=None): 38 | """Requests Panoptes begin an iNaturalist subject import.""" 39 | 40 | click.echo(f'Importing taxon ID {taxon_id} into subject set {subject_set_id}.') 41 | Inaturalist.inat_import(taxon_id, subject_set_id, updated_since) 42 | -------------------------------------------------------------------------------- /panoptes_cli/commands/info.py: -------------------------------------------------------------------------------- 1 | import platform 2 | 3 | import click 4 | import importlib.metadata 5 | 6 | from panoptes_cli.scripts.panoptes import cli 7 | 8 | 9 | @cli.command() 10 | @click.pass_context 11 | def info(ctx): 12 | """Displays version and environment information for debugging.""" 13 | 14 | info = { 15 | 'Panoptes CLI version': ( 16 | importlib.metadata.version("panoptescli") 17 | ), 18 | 'Panoptes client version': ( 19 | importlib.metadata.version("panoptes_client") 20 | ), 21 | 'Operating system': '{} {}'.format( 22 | platform.system(), 23 | platform.release(), 24 | ), 25 | 'Python version': platform.python_version(), 26 | 'API endpoint': ctx.parent.config['endpoint'], 27 | 'Click': importlib.metadata.version("click"), 28 | 'PyYAML': importlib.metadata.version("pyyaml"), 29 | 'requests': importlib.metadata.version("requests"), 30 | 'urllib3': importlib.metadata.version("urllib3") 31 | } 32 | 33 | try: 34 | info['libmagic'] = importlib.metadata.version("python-magic") 35 | except importlib.metadata.PackageNotFoundError: 36 | info['libmagic'] = False 37 | 38 | for k, v in info.items(): 39 | click.echo('{}: {}'.format(k, v)) 40 | -------------------------------------------------------------------------------- /panoptes_cli/commands/configure.py: -------------------------------------------------------------------------------- 1 | import click 2 | import os 3 | import yaml 4 | 5 | from panoptes_cli.scripts.panoptes import cli 6 | 7 | @cli.command() 8 | @click.pass_context 9 | @click.option( 10 | '--edit-all', 11 | '-a', 12 | help=( 13 | "Modify all configuration options (rather than just username and " 14 | "password)." 15 | ), 16 | is_flag=True 17 | ) 18 | def configure(ctx, edit_all): 19 | """Sets default values for configuration options.""" 20 | 21 | if not os.path.isdir(ctx.parent.config_dir): 22 | os.mkdir(ctx.parent.config_dir) 23 | 24 | for opt, value in ctx.parent.config.items(): 25 | if opt == 'endpoint' and not edit_all: 26 | continue 27 | 28 | is_password = opt == 'password' 29 | ctx.parent.config[opt] = click.prompt( 30 | opt, 31 | default=value, 32 | hide_input=is_password, 33 | show_default=not is_password, 34 | ) 35 | 36 | if not ctx.parent.config['endpoint'].startswith('https://'): 37 | click.echo( 38 | 'Error: Invalid endpoint supplied. Endpoint must be an HTTPS URL.' 39 | ) 40 | return -1 41 | 42 | with open(ctx.parent.config_file, 'w') as conf_f: 43 | yaml.dump(ctx.parent.config, conf_f, default_flow_style=False) 44 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | dev: 5 | build: 6 | context: ./ 7 | dockerfile: Dockerfile.dev 8 | volumes: 9 | - ${HOME}/.panoptes/:/root/.panoptes/ 10 | - ${HOME}:${HOME} 11 | working_dir: ${PWD} 12 | 13 | debug: 14 | build: 15 | context: ./ 16 | dockerfile: Dockerfile.dev 17 | volumes: 18 | - ${HOME}/.panoptes/:/root/.panoptes/ 19 | - ${HOME}:${HOME} 20 | environment: 21 | - PANOPTES_DEBUG=true 22 | working_dir: ${PWD} 23 | 24 | stable: 25 | build: 26 | context: ./ 27 | dockerfile: Dockerfile.stable 28 | volumes: 29 | - ${HOME}/.panoptes/:/root/.panoptes/ 30 | - ${HOME}:${HOME} 31 | working_dir: ${PWD} 32 | 33 | dev2: 34 | build: 35 | context: ./ 36 | dockerfile: Dockerfile.dev2 37 | volumes: 38 | - ${HOME}/.panoptes/:/root/.panoptes/ 39 | - ${HOME}:${HOME} 40 | working_dir: ${PWD} 41 | 42 | debug2: 43 | build: 44 | context: ./ 45 | dockerfile: Dockerfile.dev2 46 | volumes: 47 | - ${HOME}/.panoptes/:/root/.panoptes/ 48 | - ${HOME}:${HOME} 49 | environment: 50 | - PANOPTES_DEBUG=true 51 | working_dir: ${PWD} 52 | 53 | stable2: 54 | build: 55 | context: ./ 56 | dockerfile: Dockerfile.stable2 57 | volumes: 58 | - ${HOME}/.panoptes/:/root/.panoptes/ 59 | - ${HOME}:${HOME} 60 | working_dir: ${PWD} 61 | -------------------------------------------------------------------------------- /panoptes_cli/commands/tests/test_project.py: -------------------------------------------------------------------------------- 1 | from click.testing import CliRunner 2 | from panoptes_cli.commands.project import ls 3 | 4 | import unittest 5 | 6 | import warnings 7 | import sys 8 | 9 | class TestProject(unittest.TestCase): 10 | def __init__(self, *args, **kwargs): 11 | super(TestProject, self).__init__(*args, **kwargs) 12 | self.runner = CliRunner() 13 | 14 | def test_ls_id_public(self): 15 | if sys.version_info >= (3, 0): 16 | # avoid client socket warnings polluting our test results 17 | # this can go when the underlying client issue is fixed 18 | # https://github.com/zooniverse/panoptes-python-client/issues/270 19 | warnings.filterwarnings( 20 | action="ignore", message="unclosed", category=ResourceWarning) 21 | 22 | result = self.runner.invoke(ls, ['--project-id', '1']) 23 | self.assertEqual( 24 | result.output, 25 | '1 zooniverse/snapshot-supernova Snapshot Supernova\n' 26 | ) 27 | 28 | def test_ls_id_private_anon(self): 29 | result = self.runner.invoke(ls, ['--project-id', '7']) 30 | self.assertEqual(result.output, '') 31 | 32 | def test_ls_slug_public(self): 33 | result = self.runner.invoke( 34 | ls, 35 | ['--slug', 'zooniverse/snapshot-supernova'] 36 | ) 37 | self.assertEqual( 38 | result.output, 39 | '1 zooniverse/snapshot-supernova Snapshot Supernova\n' 40 | ) 41 | 42 | def test_ls_slug_private_anon(self): 43 | result = self.runner.invoke(ls, ['--slug', 'astopy/testing']) 44 | self.assertEqual(result.output, '') 45 | -------------------------------------------------------------------------------- /panoptes_cli/scripts/panoptes.py: -------------------------------------------------------------------------------- 1 | import click 2 | import os 3 | import yaml 4 | from panoptes_client import Panoptes 5 | 6 | 7 | @click.version_option(prog_name='Panoptes CLI') 8 | @click.group() 9 | @click.option( 10 | '--endpoint', 11 | '-e', 12 | help="Overides the default API endpoint", 13 | type=str, 14 | ) 15 | @click.option( 16 | '--admin', 17 | '-a', 18 | help=( 19 | "Enables admin mode. Ignored if you're not logged in as an " 20 | "administrator." 21 | ), 22 | is_flag=True, 23 | ) 24 | @click.pass_context 25 | def cli(ctx, endpoint, admin): 26 | ctx.config_dir = os.path.expanduser('~/.panoptes/') 27 | ctx.config_file = os.path.join(ctx.config_dir, 'config.yml') 28 | ctx.config = { 29 | 'endpoint': 'https://www.zooniverse.org', 30 | 'username': '', 31 | 'password': '', 32 | } 33 | 34 | try: 35 | with open(ctx.config_file) as conf_f: 36 | ctx.config.update(yaml.full_load(conf_f)) 37 | except IOError: 38 | pass 39 | 40 | if endpoint: 41 | ctx.config['endpoint'] = endpoint 42 | 43 | if ctx.invoked_subcommand != 'configure': 44 | Panoptes.connect( 45 | endpoint=ctx.config['endpoint'], 46 | username=ctx.config['username'], 47 | password=ctx.config['password'], 48 | admin=admin, 49 | ) 50 | 51 | from panoptes_cli.commands.configure import * 52 | from panoptes_cli.commands.info import * 53 | from panoptes_cli.commands.project import * 54 | from panoptes_cli.commands.subject import * 55 | from panoptes_cli.commands.subject_set import * 56 | from panoptes_cli.commands.user import * 57 | from panoptes_cli.commands.workflow import * 58 | from panoptes_cli.commands.inaturalist import * 59 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We welcome pull requests from anyone, so if you have something you'd like to 4 | contribute or an idea for improving this project, that's great! Changes should 5 | generally fit into one of the following categories: 6 | 7 | - Bug fixes 8 | - Implementing additional Panoptes API functionality (there's still a lot to do 9 | here!) 10 | - Improvements/enhancements which will be generally useful to many people who 11 | use the API client. 12 | 13 | If you're unsure about whether your changes would be suitable, please feel free 14 | to open an issue to discuss them _before_ spending too much time implementing 15 | them. It's best to start talking about how (or if) you should do something 16 | early, before a lot of work goes into it. 17 | 18 | ## Getting started 19 | 20 | The first thing you should do is fork this repo and clone your fork to your 21 | local computer. Then create a feature branch for your changes (create a separate 22 | branch for each separate contribution, don't lump unrelated changes together). 23 | 24 | I'd **strongly** recommend using Docker Compose to test your development 25 | version: 26 | 27 | ``` 28 | $ docker-compose build dev 29 | $ docker-compose run dev --help 30 | ``` 31 | 32 | When you're ready, push your changes to a branch in your fork and open a pull 33 | request. After opening the PR, you may get some comments from Hound, which is an 34 | automated service which checks coding style and highlights common mistakes. 35 | Please take note of what it says and make any changes to your code as needed. 36 | 37 | ## Releasing new packages 38 | 39 | If you have access to publish new releases on PyPI, this is a general outline of 40 | the process: 41 | 42 | - Bump the version number in setup.py 43 | - Update CHANGELOG.md 44 | - Update README.md if needed 45 | - Build and upload a new package: 46 | 47 | ``` 48 | python setup.py sdist 49 | twine upload -s dist/panoptescli-* 50 | git tag 51 | git push --tags 52 | ``` 53 | 54 | Note that you'll need to have a GPG key set up so that `twine` can sign the 55 | package. You should also make sure that your public key is published in the key 56 | servers so that people can verify the signature. 57 | -------------------------------------------------------------------------------- /panoptes_cli/commands/user.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | import click 4 | 5 | from panoptes_cli.scripts.panoptes import cli 6 | from panoptes_client import Panoptes, User 7 | 8 | 9 | @cli.group() 10 | def user(): 11 | """Contains commands for retrieving information about users.""" 12 | 13 | pass 14 | 15 | 16 | @user.command() 17 | @click.option( 18 | '--email', 19 | '-e', 20 | help='Search for users by email address (only works if you\'re an admin).', 21 | type=str, 22 | ) 23 | @click.option( 24 | '--login', 25 | '-l', 26 | help='Search for users by login name.', 27 | type=str, 28 | ) 29 | @click.argument('user-id', required=False, type=int) 30 | def info(user_id, email, login): 31 | """ 32 | Displays information about a user. Defaults to the current user if no ID or 33 | search criteria are given. 34 | """ 35 | 36 | if (user_id and email) or (user_id and login) or (email and login): 37 | click.echo( 38 | 'Error: At most only one of user ID, login, or email may be ' 39 | 'specified.', 40 | err=True, 41 | ) 42 | return -1 43 | if user_id: 44 | user = User.find(user_id) 45 | elif email: 46 | try: 47 | user = next(User.where(email=email)) 48 | except StopIteration: 49 | user = None 50 | if getattr(user, 'email', '') != email: 51 | click.echo('User not found', err=True) 52 | return -1 53 | else: 54 | if not login: 55 | login = Panoptes.client().username 56 | try: 57 | user = next(User.where(login=login)) 58 | except StopIteration: 59 | user = None 60 | if getattr(user, 'login', '') != login: 61 | click.echo('User not found', err=True) 62 | return -1 63 | click.echo(yaml.dump(user.raw)) 64 | 65 | 66 | @user.command() 67 | @click.option( 68 | '--force', 69 | '-f', 70 | is_flag=True, 71 | help='Delete without asking for confirmation.', 72 | ) 73 | @click.argument('user-ids', required=True, nargs=-1, type=int) 74 | def delete(force, user_ids): 75 | """ 76 | Deletes a user. Only works if you're an admin. 77 | """ 78 | 79 | for user_id in user_ids: 80 | user = User.find(user_id) 81 | if not force: 82 | click.confirm('Delete user {} ({})?'.format( 83 | user_id, 84 | user.login, 85 | ), abort=True) 86 | user.delete() 87 | 88 | 89 | @user.command() 90 | def token(): 91 | """ 92 | Returns the current oauth token and its expiration date. 93 | """ 94 | 95 | click.echo("Token: {}".format(Panoptes.client().get_bearer_token())) 96 | click.echo("Expiry time: {}".format(Panoptes.client().bearer_expires)) -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | pull_request: 18 | schedule: 19 | - cron: '45 17 * * 5' 20 | 21 | jobs: 22 | analyze: 23 | name: Analyze 24 | runs-on: ubuntu-latest 25 | permissions: 26 | actions: read 27 | contents: read 28 | security-events: write 29 | 30 | strategy: 31 | fail-fast: false 32 | matrix: 33 | language: [ 'python' ] 34 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 35 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 36 | 37 | steps: 38 | - name: Checkout repository 39 | uses: actions/checkout@v3 40 | 41 | # Initializes the CodeQL tools for scanning. 42 | - name: Initialize CodeQL 43 | uses: github/codeql-action/init@v2 44 | with: 45 | languages: ${{ matrix.language }} 46 | # If you wish to specify custom queries, you can do so here or in a config file. 47 | # By default, queries listed here will override any specified in a config file. 48 | # Prefix the list here with "+" to use these queries and those in the config file. 49 | 50 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 51 | # queries: security-extended,security-and-quality 52 | 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v2 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 61 | 62 | # If the Autobuild fails above, remove it and uncomment the following three lines. 63 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 64 | 65 | # - run: | 66 | # echo "Run, Build Application using script" 67 | # ./location_of_script_within_repo/buildscript.sh 68 | 69 | - name: Perform CodeQL Analysis 70 | uses: github/codeql-action/analyze@v2 71 | with: 72 | category: "/language:${{matrix.language}}" 73 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 1.2.1 (2025-06-24) 2 | - Fix: Update batch aggregation file paths 3 | 4 | ## 1.2.0 (2024-03-07) 5 | - Update: Replace pkg_resources with importlib.metadata 6 | - New: Add new package versions to info for debugging 7 | - New: Add subject update-metadata command 8 | - New: Add batch Aggregation 9 | 10 | ## 1.1.5 (2023-07-11) 11 | - Update panoptes-client requirement to >=1.6 12 | - Bump humanize to <4.8 13 | - Fix: Remove deprecated U mode in file open 14 | - Update README with iNat and python 2 deprecation info. 15 | 16 | ## 1.1.4 (2022-12-02) 17 | 18 | - New: support `%` in manifest column headings for indexed subject sets. Manifest headers with the `%` prefix will automatically be added to the subject set configuration `'indexFields'` list. 19 | - New: iNaturalist observation import functionality (see --help for details) 20 | - Update reqs for humanize, click, pyyaml, & pathvalidate 21 | 22 | ## 1.1.3 (2020-12-10) 23 | 24 | - Update panoptes-client requirement to >=1.3 25 | 26 | ## 1.1.2 (2020-05-07) 27 | 28 | - Bump pyyaml to <5.4 29 | - Bump humanize to <1.1 30 | 31 | ## 1.1.1 (2019-11-29) 32 | 33 | - Fix: Bump pyyaml requirement to >=5.1 to fix AttributeError 34 | 35 | ## 1.1 (2019-10-25) 36 | 37 | - New: Use multithreaded subject uploads 38 | - New: Add option to resume subject uploads on failure 39 | - New: Add ID file option to subject set add/remove 40 | - New: Add `--file-column` option to subject uploads 41 | - New: Add `info` commands for all objects 42 | - New: Add `delete` commands for all objects 43 | - New: Allow multiple remote MIME types 44 | - New: Add `user token` command 45 | - Fix: Do not connect to API during `configure` 46 | - Fix: Use `yaml.full_load` instead of `yaml.load` 47 | - Fix: Use `os.path.isfile` instead of `exists` 48 | - Add help text to `workflow download-classifications` 49 | - Abort uploads if manifest doesn't contain any rows 50 | - Don't show password in configure command 51 | - Validate endpoint config 52 | - Validate file sizes before uploading 53 | - Update pyyaml requirement to >=3.12,<5.2 54 | - Update click requirement to >=6.7,<7.1 55 | 56 | ## 1.0.2 (2019-02-20) 57 | 58 | - Update pyyaml requirement to >=3.12,<4.2 59 | 60 | ## 1.0.1 (2018-04-27) 61 | 62 | - Fix: Modifying projects makes them private 63 | 64 | ## 1.0 (2017-11-16) 65 | 66 | - New: Add --version option 67 | - New: Add info command 68 | - New: Add help text for all commands 69 | - New: Add progress bars for data export downloads 70 | - Fix: Modifying project public/private status 71 | - Remove non-functional `--project-id` option from `subject-set modify` 72 | - Rename `workflow download` to `workflow download-classifications` 73 | - Rely on API to validate file types 74 | 75 | ## 0.8 (2017-08-04) 76 | 77 | - New: Set default endpoint to www.zooniverse.org 78 | - New: Standardise options and arguments 79 | - Fix: Fix remote media in Python 3 80 | - Remove default download timeouts 81 | 82 | ## 0.7 (2017-06-20) 83 | 84 | - New: Add 'quiet' option to ls commands 85 | - New: Allow listing multiple subjects by ID 86 | - New: Add short option for subject set id in subject ls 87 | - Fix: Use next(reader) rather than reader.next() 88 | 89 | ## 0.6 (2017-05-11) 90 | 91 | - New: Add support for remote subject media locations 92 | 93 | ## 0.5 (2017-03-22) 94 | 95 | - New: Make `project ls` perform a full-text search 96 | - New: Allow listing subjects in a subject set 97 | - New: Subject to subject set linking 98 | - New: Add commands to activate/deactivate workflows 99 | - New: Add command to download workflow classifications exports 100 | - Fix: Use os.path.expanduser to find config directory 101 | 102 | ## 0.4 (2017-03-13) 103 | 104 | - New: Listing subject sets by project ID and workflow ID 105 | - New: Listing workflows 106 | - New: Adding and removing subject sets to and from workflows 107 | - New: Allow uploading multiple manifests at once (changes arguments for 108 | `subject_set upload_subjects`) 109 | - Increase default timeout for exports to 1 hour 110 | 111 | ## 0.3 (2016-11-21) 112 | 113 | - New: Add all data exports 114 | - New: Add --allow-missing option to upload_subjects 115 | - Fix: JPEG uploading 116 | - Fix: Open manifest file with universal newline mode 117 | - Fix: Don't create subjects with no images 118 | 119 | ## 0.2 (2016-09-02) 120 | 121 | - New: Project classification exports 122 | - New: Subject retirement 123 | - New: Add --launch-approved option to project ls 124 | - Fix: Update `SubjectSet.add_subjects` -> `SubjectSet.add` 125 | 126 | ## 0.1 (2016-06-17) 127 | 128 | - Initial release 129 | - Allows creating and modifying projects and subject sets 130 | - Allows uploading subjects to subject sets 131 | -------------------------------------------------------------------------------- /panoptes_cli/commands/project.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | import click 4 | 5 | from panoptes_cli.scripts.panoptes import cli 6 | from panoptes_client import Project 7 | 8 | @cli.group() 9 | def project(): 10 | """Contains commands for managing projects.""" 11 | pass 12 | 13 | @project.command() 14 | @click.option( 15 | '--project-id', 16 | '-p', 17 | help="Show the project with the given ID.", 18 | required=False, 19 | type=int, 20 | ) 21 | @click.option( 22 | '--display-name', 23 | '-n', 24 | help="Show projects whose display name exactly matches the given string.", 25 | ) 26 | @click.option( 27 | '--launch-approved', 28 | '-a', 29 | help="Only show projects which have been approved by the Zooniverse.", 30 | is_flag=True 31 | ) 32 | @click.option( 33 | '--slug', 34 | '-s', 35 | help="Show the project whose slug exactly matches the given string.", 36 | ) 37 | @click.option( 38 | '--quiet', 39 | '-q', 40 | is_flag=True, 41 | help='Only print project IDs (omit project names).', 42 | ) 43 | @click.argument( 44 | 'search', 45 | required=False, 46 | nargs=-1 47 | ) 48 | def ls(project_id, display_name, launch_approved, slug, quiet, search): 49 | """ 50 | Lists project IDs and names. 51 | 52 | Any given SEARCH terms are used for a full-text search of project titles. 53 | 54 | A "*" before the project ID indicates that the project is private. 55 | """ 56 | 57 | if not launch_approved: 58 | launch_approved = None 59 | 60 | projects = Project.where( 61 | id=project_id, 62 | slug=slug, 63 | display_name=display_name, 64 | launch_approved=launch_approved, 65 | search=" ".join(search) 66 | ) 67 | 68 | if quiet: 69 | click.echo(" ".join([p.id for p in projects])) 70 | else: 71 | for project in projects: 72 | echo_project(project) 73 | 74 | 75 | @project.command() 76 | @click.argument('project-id', required=True) 77 | def info(project_id): 78 | project = Project.find(project_id) 79 | click.echo(yaml.dump(project.raw)) 80 | 81 | 82 | @project.command() 83 | @click.argument('display-name', required=True) 84 | @click.argument('description', required=True) 85 | @click.option( 86 | '--primary-language', 87 | '-l', 88 | help="Sets the language code for the project's primary language.", 89 | default='en' 90 | ) 91 | @click.option( 92 | '--public', 93 | '-p', 94 | help="Makes the project publically accessible.", 95 | is_flag=True 96 | ) 97 | @click.option( 98 | '--quiet', 99 | '-q', 100 | help='Only print project ID (omit name).', 101 | is_flag=True, 102 | ) 103 | def create(display_name, description, primary_language, public, quiet): 104 | """ 105 | Creates a new project. 106 | 107 | Prints the project ID and name of the new project. 108 | """ 109 | 110 | project = Project() 111 | project.display_name = display_name 112 | project.description = description 113 | project.primary_language = primary_language 114 | project.private = not public 115 | project.save() 116 | 117 | if quiet: 118 | click.echo(project.id) 119 | else: 120 | echo_project(project) 121 | 122 | @project.command() 123 | @click.argument('project-id', required=True, type=int) 124 | @click.option( 125 | '--display-name', 126 | '-n', 127 | help="Sets the project's public display name.", 128 | required=False, 129 | ) 130 | @click.option( 131 | '--description', 132 | '-d', 133 | help="Sets the full-text description of the project.", 134 | required=False, 135 | ) 136 | @click.option( 137 | '--primary-language', 138 | '-l', 139 | help="Sets the language code for the project's primary language.", 140 | default='en', 141 | ) 142 | @click.option( 143 | '--public/--private', 144 | '-p/-P', 145 | help="Sets the project to be public or private.", 146 | default=None, 147 | ) 148 | def modify(project_id, display_name, description, primary_language, public): 149 | """ 150 | Changes the attributes of an existing project. 151 | 152 | Any attributes which are not specified are left unchanged. 153 | """ 154 | 155 | project = Project.find(project_id) 156 | if display_name: 157 | project.display_name = display_name 158 | if description: 159 | project.description = description 160 | if primary_language: 161 | project.primary_language = primary_language 162 | if public is not None: 163 | project.private = not public 164 | project.save() 165 | echo_project(project) 166 | 167 | @project.command() 168 | @click.argument('project-id', required=True, type=int) 169 | @click.argument('output-file', required=True, type=click.File('wb')) 170 | @click.option( 171 | '--generate', 172 | '-g', 173 | help="Generates a new export before downloading.", 174 | is_flag=True 175 | ) 176 | @click.option( 177 | '--generate-timeout', 178 | '-T', 179 | help=( 180 | "Time in seconds to wait for new export to be ready. Defaults to " 181 | "unlimited. Has no effect unless --generate is given." 182 | ), 183 | required=False, 184 | type=int, 185 | ) 186 | @click.option( 187 | '--data-type', 188 | '-t', 189 | type=click.Choice([ 190 | 'classifications', 191 | 'subjects', 192 | 'workflows', 193 | 'talk_comments', 194 | 'talk_tags']), 195 | default='classifications' 196 | ) 197 | def download(project_id, output_file, generate, generate_timeout, data_type): 198 | """ 199 | Downloads project-level data exports. 200 | 201 | OUTPUT_FILE will be overwritten if it already exists. Set OUTPUT_FILE to - 202 | to output to stdout. 203 | """ 204 | 205 | project = Project.find(project_id) 206 | 207 | if generate: 208 | click.echo("Generating new export...", err=True) 209 | 210 | export = project.get_export( 211 | data_type, 212 | generate=generate, 213 | wait_timeout=generate_timeout 214 | ) 215 | 216 | with click.progressbar( 217 | export.iter_content(chunk_size=1024), 218 | label='Downloading', 219 | length=(int(export.headers.get('content-length')) / 1024 + 1), 220 | file=click.get_text_stream('stderr'), 221 | ) as chunks: 222 | for chunk in chunks: 223 | output_file.write(chunk) 224 | 225 | 226 | @project.command() 227 | @click.option( 228 | '--force', 229 | '-f', 230 | is_flag=True, 231 | help='Delete without asking for confirmation.', 232 | ) 233 | @click.argument('project-ids', required=True, nargs=-1, type=int) 234 | def delete(force, project_ids): 235 | for project_id in project_ids: 236 | project = Project.find(project_id) 237 | if not force: 238 | click.confirm( 239 | 'Delete project {} ({})?'.format( 240 | project_id, 241 | project.display_name, 242 | ), 243 | abort=True, 244 | ) 245 | project.delete() 246 | 247 | 248 | def echo_project(project): 249 | click.echo( 250 | u'{}{} {} {}'.format( 251 | '*' if project.private else '', 252 | project.id, 253 | project.slug, 254 | project.display_name) 255 | ) 256 | -------------------------------------------------------------------------------- /panoptes_cli/commands/workflow.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | import click 4 | 5 | from panoptes_cli.scripts.panoptes import cli 6 | from panoptes_client import Workflow 7 | from panoptes_client.panoptes import PanoptesAPIException 8 | 9 | @cli.group() 10 | def workflow(): 11 | """Contains commands for managing workflows.""" 12 | pass 13 | 14 | 15 | @workflow.command() 16 | @click.argument('workflow-id', required=False, type=int) 17 | @click.option( 18 | '--project-id', 19 | '-p', 20 | help="List workflows linked to the given project.", 21 | required=False, 22 | type=int, 23 | ) 24 | @click.option( 25 | '--quiet', 26 | '-q', 27 | is_flag=True, 28 | help='Only print workflow IDs (omit names).', 29 | ) 30 | def ls(workflow_id, project_id, quiet): 31 | """Lists workflow IDs and names.""" 32 | 33 | if workflow_id and not project_id: 34 | workflow = Workflow.find(workflow_id) 35 | if quiet: 36 | click.echo(workflow.id) 37 | else: 38 | echo_workflow(workflow) 39 | return 40 | 41 | args = {} 42 | if project_id: 43 | args['project_id'] = project_id 44 | if workflow_id: 45 | args['workflow_id'] = workflow_id 46 | 47 | workflows = Workflow.where(**args) 48 | if quiet: 49 | click.echo(" ".join([w.id for w in workflows])) 50 | else: 51 | for workflow in workflows: 52 | echo_workflow(workflow) 53 | 54 | 55 | @workflow.command() 56 | @click.argument('workflow-id', required=True) 57 | def info(workflow_id): 58 | workflow = Workflow.find(workflow_id) 59 | click.echo(yaml.dump(workflow.raw)) 60 | 61 | 62 | @workflow.command(name='retire-subjects') 63 | @click.argument('workflow-id', type=int) 64 | @click.argument('subject-ids', type=int, nargs=-1) 65 | @click.option( 66 | '--reason', 67 | '-r', 68 | help="The reason for retiring the subject.", 69 | type=click.Choice(( 70 | 'classification_count', 71 | 'flagged', 72 | 'blank', 73 | 'consensus', 74 | 'other' 75 | )), 76 | default='other' 77 | ) 78 | def retire_subjects(workflow_id, subject_ids, reason): 79 | """ 80 | Retires subjects from the given workflow. 81 | 82 | The subjects will no longer be served to volunteers for classification. 83 | """ 84 | 85 | workflow = Workflow.find(workflow_id) 86 | workflow.retire_subjects(subject_ids, reason) 87 | 88 | 89 | @workflow.command(name='unretire-subjects') 90 | @click.argument('workflow-id', type=int) 91 | @click.argument('subject-ids', type=int, nargs=-1, required=True) 92 | def unretire_subjects(workflow_id, subject_ids): 93 | """ 94 | Unretires subjects for the given workflow. 95 | 96 | The subjects will be cleared of retirement data and be available for volunteers to classify. 97 | """ 98 | 99 | workflow = Workflow.find(workflow_id) 100 | workflow.unretire_subjects(subject_ids) 101 | 102 | 103 | @workflow.command(name='unretire-subject-sets') 104 | @click.argument('workflow-id', type=int) 105 | @click.argument('subject-set-ids', type=int, nargs=-1, required=True) 106 | def unretire_subject_sets(workflow_id, subject_set_ids): 107 | """ 108 | Unretires all the subjects in the subject sets for the given workflow. 109 | 110 | All subjects linked to the supplied subject sets will be cleared of retirement data and be available for volunteers to classify. 111 | """ 112 | 113 | workflow = Workflow.find(workflow_id) 114 | workflow.unretire_subjects_by_subject_set(subject_set_ids) 115 | 116 | 117 | @workflow.command(name='add-subject-sets') 118 | @click.argument('workflow-id', type=int) 119 | @click.argument('subject-set-ids', type=int, nargs=-1) 120 | def add_subject_sets(workflow_id, subject_set_ids): 121 | """Links existing subject sets to the given workflow.""" 122 | 123 | workflow = Workflow.find(workflow_id) 124 | workflow.add_subject_sets(subject_set_ids) 125 | 126 | 127 | 128 | @workflow.command(name='remove-subject-sets') 129 | @click.argument('workflow-id', type=int) 130 | @click.argument('subject-set-ids', type=int, nargs=-1) 131 | def remove_subject_sets(workflow_id, subject_set_ids): 132 | """Unlinks the given subject sets from the given workflow.""" 133 | 134 | workflow = Workflow.find(workflow_id) 135 | workflow.remove_subject_sets(subject_set_ids) 136 | 137 | 138 | @workflow.command() 139 | @click.argument('workflow-id', type=int) 140 | def activate(workflow_id): 141 | """Activates the given workflow.""" 142 | 143 | workflow = Workflow.find(workflow_id) 144 | workflow.active = True 145 | workflow.save() 146 | 147 | 148 | @workflow.command() 149 | @click.argument('workflow-id', type=int) 150 | def deactivate(workflow_id): 151 | """Deactivates the given workflow.""" 152 | 153 | workflow = Workflow.find(workflow_id) 154 | workflow.active = False 155 | workflow.save() 156 | 157 | 158 | @workflow.command(name="download-classifications") 159 | @click.argument('workflow-id', required=True, type=int) 160 | @click.argument('output-file', required=True, type=click.File('wb')) 161 | @click.option( 162 | '--generate', 163 | '-g', 164 | help="Generates a new export before downloading.", 165 | is_flag=True 166 | ) 167 | @click.option( 168 | '--generate-timeout', 169 | '-T', 170 | help=( 171 | "Time in seconds to wait for new export to be ready. Defaults to " 172 | "unlimited. Has no effect unless --generate is given." 173 | ), 174 | required=False, 175 | type=int, 176 | ) 177 | def download_classifications( 178 | workflow_id, 179 | output_file, 180 | generate, 181 | generate_timeout 182 | ): 183 | """ 184 | Downloads a workflow-specific classifications export for the given workflow. 185 | 186 | OUTPUT_FILE will be overwritten if it already exists. Set OUTPUT_FILE to - 187 | to output to stdout. 188 | """ 189 | 190 | workflow = Workflow.find(workflow_id) 191 | 192 | if generate: 193 | click.echo("Generating new export...", err=True) 194 | 195 | export = workflow.get_export( 196 | 'classifications', 197 | generate=generate, 198 | wait_timeout=generate_timeout 199 | ) 200 | 201 | with click.progressbar( 202 | export.iter_content(chunk_size=1024), 203 | label='Downloading', 204 | length=(int(export.headers.get('content-length')) / 1024 + 1), 205 | file=click.get_text_stream('stderr'), 206 | ) as chunks: 207 | for chunk in chunks: 208 | output_file.write(chunk) 209 | 210 | 211 | @workflow.command() 212 | @click.option( 213 | '--force', 214 | '-f', 215 | is_flag=True, 216 | help='Delete without asking for confirmation.', 217 | ) 218 | @click.argument('workflow-ids', required=True, nargs=-1, type=int) 219 | def delete(force, workflow_ids): 220 | for workflow_id in workflow_ids: 221 | workflow = Workflow.find(workflow_id) 222 | if not force: 223 | click.confirm( 224 | 'Delete workflow {} ({})?'.format( 225 | workflow_id, 226 | workflow.display_name, 227 | ), 228 | abort=True, 229 | ) 230 | workflow.delete() 231 | 232 | 233 | @workflow.command() 234 | @click.argument('workflow-id', required=True, type=int) 235 | @click.option( 236 | '--user-id', 237 | '-u', 238 | help= 239 | 'ID of user to whom notifications should be sent. ' 240 | 'Default: logged in user', 241 | required=False, 242 | type=int 243 | ) 244 | @click.option( 245 | '--delete-if-exists', 246 | '-d', 247 | is_flag=True, 248 | help='Delete if it exists.' 249 | ) 250 | def run_aggregation(workflow_id, user_id, delete_if_exists): 251 | """Kicks off a new aggregation job.""" 252 | 253 | agg = Workflow(workflow_id).run_aggregation(user_id, delete_if_exists) 254 | try: 255 | click.echo(agg.raw) 256 | except PanoptesAPIException as err: 257 | click.echo(err) 258 | 259 | 260 | @workflow.command() 261 | @click.argument('workflow-id', required=True, type=int) 262 | def get_batch_aggregation(workflow_id): 263 | """Gets workflow's existing batch aggregation.""" 264 | 265 | agg = Workflow(workflow_id).get_batch_aggregation() 266 | click.echo(agg.raw) 267 | 268 | 269 | @workflow.command() 270 | @click.argument('workflow-id', required=True, type=int) 271 | def get_batch_aggregation_status(workflow_id): 272 | """Fetches the run status of existing aggregation.""" 273 | 274 | click.echo(Workflow(workflow_id).get_batch_aggregation_status()) 275 | 276 | 277 | @workflow.command() 278 | @click.argument('workflow-id', required=True, type=int) 279 | def get_batch_aggregation_links(workflow_id): 280 | """Fetches batch aggregation download links.""" 281 | 282 | click.echo(Workflow(workflow_id).get_batch_aggregation_links()) 283 | 284 | 285 | def echo_workflow(workflow): 286 | click.echo( 287 | u'{} {}'.format( 288 | workflow.id, 289 | workflow.display_name 290 | ) 291 | ) 292 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Panoptes CLI 2 | 3 | A command-line interface for [Panoptes](https://github.com/zooniverse/Panoptes), 4 | the API behind [the Zooniverse](https://www.zooniverse.org/). 5 | 6 | ## Installation 7 | 8 | The Panoptes CLI is written in Python, so in order to install it you will need 9 | to install Python 3 along with `pip`. Please note: while still compatible with 10 | Python 2.7, we have ended support for use of the CLI with this deprecated version. 11 | macOS and Linux already come with Python installed, so run this to see if you 12 | already have everything you need: 13 | 14 | ``` 15 | $ python --version && pip --version 16 | ``` 17 | 18 | If you see an error like `python: command not found` or `pip: command not found` 19 | then you will need to install this: 20 | 21 | - [Python installation](https://wiki.python.org/moin/BeginnersGuide/Download) 22 | (or [Miniconda installation](https://docs.conda.io/en/latest/miniconda.html)) 23 | - [Pip installation](https://pip.pypa.io/en/stable/installing/) 24 | 25 | Once these are installed you can just use `pip` to install the latest release of 26 | the CLI: 27 | 28 | ``` 29 | $ pip install panoptescli 30 | ``` 31 | 32 | Alternatively, if you want to preview the next release you can install HEAD 33 | directly from GitHub (though be aware that this may contain 34 | bugs/untested/incomplete features): 35 | 36 | ``` 37 | $ pip install -U git+https://github.com/zooniverse/panoptes-cli.git 38 | ``` 39 | 40 | To upgrade an existing installation to the latest version: 41 | 42 | ``` 43 | pip install -U panoptescli 44 | ``` 45 | 46 | ## Built-in help 47 | 48 | Every command comes with a built in `--help` option, which explains how to use 49 | it. 50 | 51 | ``` 52 | $ panoptes --help 53 | Usage: panoptes [OPTIONS] COMMAND [ARGS]... 54 | 55 | Options: 56 | -e, --endpoint TEXT Overides the default API endpoint 57 | -a, --admin Enables admin mode. Ignored if you're not logged in as 58 | an administrator. 59 | --version Show the version and exit. 60 | --help Show this message and exit. 61 | 62 | Commands: 63 | configure Sets default values for configuration options. 64 | info Displays version and environment information for debugging. 65 | project Contains commands for managing projects. 66 | subject Contains commands for retrieving information about subjects. 67 | subject-set Contains commands for managing subject sets. 68 | user Contains commands for retrieving information about users. 69 | workflow Contains commands for managing workflows. 70 | ``` 71 | 72 | ``` 73 | $ panoptes project --help 74 | Usage: panoptes project [OPTIONS] COMMAND [ARGS]... 75 | 76 | Contains commands for managing projects. 77 | 78 | Options: 79 | --help Show this message and exit. 80 | 81 | Commands: 82 | create Creates a new project. 83 | delete 84 | download Downloads project-level data exports. 85 | info 86 | ls Lists project IDs and names. 87 | modify Changes the attributes of an existing project.. 88 | ``` 89 | 90 | ``` 91 | $ panoptes subject-set upload-subjects --help 92 | Usage: panoptes subject-set upload-subjects [OPTIONS] SUBJECT_SET_ID 93 | MANIFEST_FILES... 94 | 95 | Uploads subjects from each of the given MANIFEST_FILES. 96 | 97 | Example with only local files: 98 | 99 | $ panoptes subject-set upload-subjects 4667 manifest.csv 100 | 101 | Local filenames will be automatically detected in the manifest and 102 | uploaded, or filename columns can be specified with --file-column. 103 | 104 | If you are hosting your media yourself, you can put the URLs in the 105 | manifest and specify the column number(s): 106 | 107 | $ panoptes subject-set upload-subjects -r 1 4667 manifest.csv 108 | 109 | $ panoptes subject-set upload-subjects -r 1 -r 2 4667 manifest.csv 110 | 111 | Any local files will still be detected and uploaded. 112 | 113 | Options: 114 | -M, --allow-missing Do not abort when creating subjects with no 115 | media files. 116 | -r, --remote-location INTEGER Specify a field (by column number) in the 117 | manifest which contains a URL to a remote 118 | media location. Can be used more than once. 119 | -m, --mime-type TEXT MIME type for remote media. Defaults to 120 | image/png. Can be used more than once, in 121 | which case types are mapped one to one with 122 | remote locations in the order they are given. 123 | Has no effect without --remote-location. 124 | -f, --file-column INTEGER Specify a field (by column number) in the 125 | manifest which contains a local file to be 126 | uploaded. Can be used more than once. 127 | Disables auto-detection of filename columns. 128 | --help Show this message and exit. 129 | ``` 130 | 131 | ## Uploading non-image media types 132 | 133 | If you wish to upload subjects with non-image media (e.g. audio or video), 134 | it is desirable to have the `libmagic` library installed for type detection. 135 | If you don't already have `libmagic`, please see the [dependency information 136 | for python-magic](https://github.com/ahupp/python-magic#installation) for more 137 | details. 138 | 139 | To check if `libmagic` is installed, run this command: 140 | 141 | ``` 142 | $ panoptes info 143 | ``` 144 | 145 | If you see `libmagic: False` in the output then it isn't installed. 146 | 147 | If `libmagic` is not installed, assignment of MIME types (e.g., image/jpeg, 148 | video/mp4, text/plain, application/json, etc) will be based on file extensions. 149 | Be aware that if file names and extension aren't accurate, this could lead to 150 | issues when the media is loaded. 151 | 152 | ## Command Line Examples 153 | 154 | This readme does not list everything that the CLI can do. For a full list of 155 | commands and their options, use the built in help as described above. 156 | 157 | ### Log in and optionally set the API endpoint 158 | 159 | ``` 160 | $ panoptes configure 161 | username []: 162 | password: 163 | ``` 164 | 165 | Press enter without typing anything to keep the current value (shown in 166 | brackets). You probably don't need to change the endpoint, unless you're running 167 | your own copy of the Panoptes API. 168 | 169 | ### Create a new project 170 | 171 | ``` 172 | $ panoptes project create "My Project" "This is a description of my project" 173 | *2797 zooniverse/my-project My Project 174 | ``` 175 | 176 | The `*` before the project ID indicates that the project is private. 177 | 178 | ### Create a subject set in your new project 179 | 180 | ``` 181 | $ panoptes subject-set create 2797 "My first subject set" 182 | 4667 My first subject set 183 | ``` 184 | 185 | ### Make your project public 186 | 187 | ``` 188 | $ panoptes project modify --public 2797 189 | 2797 zooniverse/my-project My Project 190 | ``` 191 | 192 | ### Upload subjects 193 | 194 | ``` 195 | $ panoptes subject-set upload-subjects 4667 manifest.csv 196 | ``` 197 | 198 | Local filenames will be automatically detected in the manifest and uploaded. If 199 | you are hosting your media yourself, you can put the URLs in the manifest and 200 | specify the column number(s) and optionally set the file type if you're not 201 | uploading PNG images: 202 | 203 | ``` 204 | $ panoptes subject-set upload-subjects -m image/jpeg -r 1 4667 manifest.csv 205 | $ panoptes subject-set upload-subjects -r 1 -r 2 4667 manifest.csv 206 | ``` 207 | 208 | A manifest is a CSV file which contains the names of local media files to upload (one per column) or remote URLs (matching the `-r` option) 209 | and any other column is recorded as subject metadata, where the column name is the key and the row/column entry is the value, for example: 210 | 211 | file_name_1 | file_name_2 | metadata | !metadata_hidden_from_classification | #metadata_hidden_from_all 212 | -- | -- | -- | -- | -- 213 | local_image_file_1.jpeg | local_image_file_2.jpeg | image_01 | giraffe | kenya_site_1 214 | 215 | ### Resuming a failed upload 216 | 217 | If an upload fails for any reason, the CLI should detect the failure and give you the option of resuming the upload at a later time: 218 | 219 | ``` 220 | $ panoptes subject-set upload-subjects -m image/jpeg -r 1 4667 manifest.csv 221 | Uploading subjects [------------------------------------] 0% 00:41:05 222 | Error: Upload failed. 223 | Would you like to save the upload state to resume the upload later? [Y/n]: y 224 | Enter filename to save to [panoptes-upload-4667.yaml]: 225 | ``` 226 | 227 | This will save a new manifest file which you can use to resume the upload. The new manifest file will be in YAML format rather than CSV, and the YAML file contains all the information about the original upload (including any command-line options you specified) along with a list of the subjects which have not yet been uploaded. 228 | 229 | To resume the upload, simply run the `upload-subjects` command specifying the same subject set ID with the new manifest file. Note that you do not need to include any other options that you originally specified (such as `-r`, `-m`, and so on): 230 | 231 | ``` 232 | $ panoptes subject-set upload-subjects 4667 panoptes-upload-4667.yaml 233 | ``` 234 | 235 | ### Generate and download a classifications export 236 | 237 | ``` 238 | $ panoptes project download --generate 2797 classifications.csv 239 | ``` 240 | 241 | It is also possible to generate and download workflow classification or subject set classification exports 242 | ``` 243 | $ panoptes workflow download-classifications --generate 18706 workflow-18706-classifications.csv 244 | 245 | $ panoptes subject-set download-classifications --generate 79758 subjectset-79759-classifications.csv 246 | ``` 247 | 248 | ### Generate and download a talk comments export 249 | 250 | ``` 251 | $ panoptes project download --generate --data-type talk_comments 2797 2797_comments.tar.gz 252 | ``` 253 | 254 | ### List subject sets in a project 255 | 256 | ``` 257 | $ panoptes subject-set ls -p 2797 258 | ``` 259 | 260 | ### Verify that subject set 4667 is in project 2797 261 | 262 | ``` 263 | $ panoptes subject-set ls -p 2797 4667 264 | ``` 265 | 266 | ### Add known subjects to a subject set 267 | 268 | ``` 269 | # for known subjects with ids 3, 2, 1 and subject set with id 999 270 | $ panoptes subject-set add-subjects 999 3 2 1 271 | ``` 272 | 273 | ### List workflows in your project 274 | 275 | ``` 276 | $ panoptes workflow ls -p 2797 277 | 1579 Example workflow 1 278 | 2251 Example workflow 2 279 | ``` 280 | 281 | ### Add a subject set to a workflow 282 | 283 | ``` 284 | $ panoptes workflow add-subject-sets 1579 4667 285 | ``` 286 | 287 | ### List subject sets in a workflow 288 | 289 | ``` 290 | $ panoptes subject-set ls -w 1579 291 | 4667 My first subject set 292 | ``` 293 | 294 | ### Retire subjects in a workflow 295 | 296 | For known subjects with ids 2001 and 2002, workflow with id 101 297 | ``` 298 | $ panoptes workflow retire-subjects 101 2001 2002 299 | ``` 300 | 301 | ### Un-Retire subjects in a workflow 302 | 303 | To unretire subjects according to subject ID, for known subjects with ids 2001, 2002 and workflow with id 101 304 | ``` 305 | $ panoptes workflow unretire-subjects 101 2001 2002 306 | ``` 307 | 308 | To unretire all subjects in a given subject set, for subject sets with ids 300, 301 and workflow with id 101 309 | ``` 310 | panoptes workflow unretire-subject-sets 101 300 301 311 | ``` 312 | 313 | ### Run aggregations 314 | 315 | To run batch aggregation on workflow with id 101, notify logged-in user (default), and delete existing run 316 | ``` 317 | $ panoptes workflow run-aggregation 101 -d 318 | ``` 319 | 320 | ### Get batch aggregations 321 | 322 | For fetching existing batch aggregation on workflow with id 101 323 | ``` 324 | $ panoptes workflow get-batch-aggregation 101 325 | ``` 326 | 327 | ### Check batch aggregation run status 328 | 329 | For checking status of batch aggregation run on workflow with id 101 330 | ``` 331 | $ panoptes workflow get-batch-aggregation-status 101 332 | ``` 333 | 334 | ### Get batch aggregation links 335 | 336 | For fetching download URLs for the run aggregation on workflow with id 101 337 | ``` 338 | $ panoptes workflow get-batch-aggregation-links 101 339 | ``` 340 | 341 | ### Importing iNaturalist observations 342 | 343 | Importing iNaturalist observations to the Zooniverse as subjects is possible via an API endpoint, which is accessible via this client. 344 | 345 | This command initiates a background job on the Zooniverse platform to import Observations. The request will return a 200 upon success, and the import will begin as the Zooniverse and iNaturalist APIs talk to each other. Once the command is issued, the work is being done remotely and you can refresh the subject set in the project builder to check its progress. The authenticated user will receive an email when this job is completed; you don't have to keep the terminal open. 346 | 347 | This command imports “verifiable” observations, which according to the iNat docs means “observations with a quality_grade of either `needs_id` or `research`." Project owners and collaborators can use this CLI to send a request to begin that import process: 348 | 349 | ``` 350 | # Requires an iNaturalist taxon id and a Zooniverse subject set (both integers). This will import all observations for that taxon id. 351 | $ panoptes inaturalist import-observations --taxon-id 46017 --subject-set-id 999999 352 | ``` 353 | 354 | Optional: include an updated_since timestamp (string) to include only observations updated after that date: 355 | 356 | ``` 357 | $ panoptes inaturalist import-observations --taxon-id 46017 --subject-set-id 999999 --updated-since 2022-12-03 358 | ``` 359 | 360 | The `--updated-since` argument is a standard ISO timestamp, such as '2022-12-03' or `2022-12-03T18:56:06+00:00'. It is passed directly to the iNat Observations v2 API as the 'updated_since' query parameter. 361 | 362 | 363 | 364 | ## Debugging 365 | 366 | To view the various requests as sent to the Panoptes API as well as other info, 367 | include the env var `PANOPTES_DEBUG=true` before your command, like so: 368 | 369 | `PANOPTES_DEBUG=true panoptes workflow ls -p 1234` 370 | 371 | ### Usage 372 | 373 | 1. Run `docker-compose build` to build the containers. Note there are mulitple containers for different envs, see docker-compose.yml for more details 374 | 375 | 2. Create and run all the containers with `docker-compose up` 376 | 377 | ### Testing 378 | 379 | 1. Use docker to run a testing environment bash shell and run test commands . 380 | 1. Run `docker-compose run --rm dev sh` to start an interactive shell in the container 381 | 1. Run `python -m unittest discover` to run the full test suite 382 | -------------------------------------------------------------------------------- /panoptes_cli/commands/subject.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import yaml 3 | import copy 4 | import os 5 | import time 6 | 7 | 8 | 9 | import click 10 | import humanize 11 | 12 | from pathvalidate import is_valid_filename, sanitize_filename 13 | from panoptes_cli.scripts.panoptes import cli 14 | from panoptes_client import Subject 15 | from panoptes_client.panoptes import PanoptesAPIException 16 | 17 | MAX_PENDING_SUBJECTS_ATTACHED_MEDIA = 50 18 | MAX_UPLOAD_FILE_SIZE = 1024 * 1024 19 | CURRENT_STATE_VERSION = 1 20 | 21 | 22 | @cli.group() 23 | def subject(): 24 | """Contains commands for managing subjects.""" 25 | 26 | pass 27 | 28 | 29 | @subject.command() 30 | @click.option( 31 | "--subject-set-id", 32 | "-s", 33 | help="List subjects from the given subject set.", 34 | type=int, 35 | required=False, 36 | ) 37 | @click.option( 38 | "--quiet", 39 | "-q", 40 | is_flag=True, 41 | help="Only print subject IDs (omit media URLs).", 42 | ) 43 | @click.argument("subject-ids", type=int, required=False, nargs=-1) 44 | def ls(subject_set_id, quiet, subject_ids): 45 | """ 46 | Lists subject IDs and their media URLs. 47 | """ 48 | 49 | if subject_ids: 50 | for subject_id in subject_ids: 51 | subject = Subject.find(subject_id) 52 | if quiet: 53 | click.echo(subject.id) 54 | else: 55 | echo_subject(subject) 56 | return 57 | 58 | subjects = Subject.where(subject_set_id=subject_set_id) 59 | if quiet: 60 | click.echo(" ".join([s.id for s in subjects])) 61 | else: 62 | for subject in subjects: 63 | echo_subject(subject) 64 | 65 | 66 | @subject.command(name='upload-subjects-attached-media') 67 | @click.argument('manifest-files', required=True, nargs=-1) 68 | @click.option( 69 | '--allow-missing', 70 | '-M', 71 | help=("Do not abort when creating subjects with no media files."), 72 | is_flag=True, 73 | ) 74 | @click.option( 75 | '--remote-location', 76 | '-r', 77 | help=( 78 | "Specify a field (by column number) in the manifest which contains a " 79 | "URL to a remote media location. Can be used more than once." 80 | ), 81 | multiple=True, 82 | type=int, 83 | required=False, 84 | ) 85 | @click.option( 86 | '--mime-type', 87 | '-m', 88 | help=( 89 | "MIME type for remote media. Defaults to image/png. Can be used more " 90 | "than once, in which case types are mapped one to one with remote " 91 | "locations in the order they are given. Has no effect without " 92 | "--remote-location." 93 | ), 94 | type=str, 95 | required=False, 96 | default=('image/png',), 97 | multiple=True 98 | ) 99 | @click.option( 100 | '--file-column', 101 | '-f', 102 | help=( 103 | "Specify a field (by column number) in the manifest which contains a " 104 | "local file to be uploaded. Can be used more than once. Disables auto-" 105 | "detection of filename columns." 106 | ), 107 | multiple=True, 108 | type=int, 109 | required=False, 110 | ) 111 | def upload_subjects_attached_media( 112 | manifest_files, 113 | allow_missing, 114 | remote_location, 115 | mime_type, 116 | file_column 117 | ): 118 | """ 119 | Uploads attached_media associated to subjects from each of the given MANIFEST_FILES. 120 | 121 | Example with only local files: 122 | 123 | $ panoptes subject upload-subjects-attached-media manifest.csv 124 | 125 | $ panoptes subject upload-subjects-attached-media -f 1 -f 2 manifest.csv 126 | 127 | 128 | eg csv 129 | subject_id | file_name_1 | file_name_2 | metadata 130 | 131 | Local filenames will be automatically detected in the manifest and 132 | uploaded, or filename columns can be specified with --file-column. 133 | 134 | If you are hosting your media yourself, you can put the URLs in the 135 | manifest and specify the column number(s): 136 | 137 | $ panoptes subject upload-subjects-attached-media -r 1 manifest.csv 138 | 139 | $ panoptes subject upload-subjects-attached-media -r 1 -r 2 manifest.csv 140 | 141 | Any local files will still be detected and uploaded. 142 | """ 143 | if ( 144 | len(manifest_files) > 1 145 | and any(map(lambda m: m.endswith('.yaml'), manifest_files)) 146 | ): 147 | click.echo( 148 | 'Error: YAML manifests must be processed one at a time.', 149 | err=True, 150 | ) 151 | return -1 152 | elif manifest_files[0].endswith('.yaml'): 153 | with open(manifest_files[0], 'r') as yaml_manifest: 154 | upload_state = yaml.load(yaml_manifest, Loader=yaml.FullLoader) 155 | if upload_state['state_version'] > CURRENT_STATE_VERSION: 156 | click.echo( 157 | 'Error: {} was generated by a newer version of the Panoptes ' 158 | 'CLI and is not compatible with this version.'.format( 159 | manifest_files[0], 160 | ), 161 | err=True, 162 | ) 163 | return -1 164 | resumed_upload = True 165 | else: 166 | upload_state = { 167 | 'state_version': CURRENT_STATE_VERSION, 168 | 'manifest_files': manifest_files, 169 | 'allow_missing': allow_missing, 170 | 'remote_location': remote_location, 171 | 'mime_type': mime_type, 172 | 'file_column': file_column, 173 | 'waiting_to_upload': [] 174 | } 175 | resumed_upload = False 176 | 177 | remote_location_count = len(upload_state['remote_location']) 178 | mime_type_count = len(upload_state['mime_type']) 179 | if remote_location_count > 1 and mime_type_count == 1: 180 | upload_state['mime_type'] = ( 181 | upload_state['mime_type'] * remote_location_count 182 | ) 183 | elif remote_location_count > 0 and mime_type_count != remote_location_count: 184 | click.echo( 185 | 'Error: The number of MIME types given must be either 1 or equal ' 186 | 'to the number of remote locations.', 187 | err=True, 188 | ) 189 | return -1 190 | 191 | if not resumed_upload: 192 | subject_rows = [] 193 | subjects_by_id = {} 194 | for manifest_file in upload_state['manifest_files']: 195 | with open(manifest_file) as manifest_f: 196 | file_root = os.path.dirname(manifest_file) 197 | r = csv.reader(manifest_f, skipinitialspace=True) 198 | headers = next(r) 199 | for row_number, row in enumerate(r, start=1): 200 | metadata = dict(zip(headers, row)) 201 | files = [] 202 | if not metadata.get('subject_id'): 203 | click.echo(f'Missing subject_id in row:{row_number}', err=True) 204 | return -1 205 | subject_id = metadata.pop('subject_id') 206 | try: 207 | subject_id = int(subject_id) 208 | except ValueError: 209 | click.echo(f'Invalid subject_id in row: {row_number}') 210 | return -1 211 | try: 212 | subject = Subject.find(subject_id) 213 | subjects_by_id[subject_id] = subject 214 | except PanoptesAPIException: 215 | click.echo(f'Subject {subject_id} does not exist on row: {row_number}') 216 | return -1 217 | if not upload_state['file_column']: 218 | upload_state['file_column'] = [] 219 | for field_num, col in enumerate(row, start=1): 220 | file_path = os.path.join(file_root, col) 221 | if os.path.exists(file_path): 222 | upload_state['file_column'].append(field_num,) 223 | if _validate_file(file_path): 224 | files.append(file_path) 225 | elif not upload_state['allow_missing']: 226 | return -1 227 | else: 228 | for field_num in upload_state['file_column']: 229 | file_path = os.path.join( 230 | file_root, 231 | row[field_num - 1] 232 | ) 233 | if _validate_file(file_path): 234 | files.append(file_path) 235 | elif not upload_state['allow_missing']: 236 | return -1 237 | 238 | for field_num, _mime_type in zip(upload_state['remote_location'], upload_state['mime_type'],): 239 | files.append({_mime_type: row[field_num - 1]}) 240 | 241 | if len(files) == 0: 242 | click.echo( 243 | 'Could not find any files in row:', 244 | err=True, 245 | ) 246 | click.echo(','.join(row), err=True) 247 | if not upload_state['allow_missing']: 248 | return -1 249 | else: 250 | continue 251 | subject_rows.append((subject_id, files, metadata)) 252 | 253 | if not subject_rows: 254 | click.echo( 255 | 'File {} did not contain any rows.'.format( 256 | manifest_file, 257 | ), 258 | err=True, 259 | ) 260 | return -1 261 | subject_rows = list(enumerate(subject_rows)) 262 | upload_state['waiting_to_upload'] = copy.deepcopy(subject_rows) 263 | else: 264 | subjects_by_id = {} 265 | for _, (subject_id, _, _) in upload_state['waiting_to_upload']: 266 | try: 267 | subject = Subject.find(subject_id) 268 | subjects_by_id[subject_id] = subject 269 | except PanoptesAPIException: 270 | click.echo(f'Subject {subject_id} does not exist', err=True) 271 | return -1 272 | subject_rows = copy.deepcopy(upload_state['waiting_to_upload']) 273 | 274 | pending_attached_media = [] 275 | 276 | def move_created(limit): 277 | while len(pending_attached_media) > limit: 278 | for media_futures, subject_row in pending_attached_media: 279 | if all(media_future.done() for media_future in media_futures): 280 | pending_attached_media.remove((media_futures, subject_row)) 281 | upload_state['waiting_to_upload'].remove(subject_row) 282 | time.sleep(0.5) 283 | 284 | with click.progressbar( 285 | subject_rows, 286 | length=len(subject_rows), 287 | label='Uploading subject attached media', 288 | ) as _subject_rows: 289 | try: 290 | with Subject.async_saves(): 291 | for subject_row in _subject_rows: 292 | _, (subject_id, files, metadata) = subject_row 293 | subject = subjects_by_id[subject_id] 294 | media_futures = [] 295 | for media_file in files: 296 | media_futures.append(subject.save_attached_image(media_file, metadata=metadata)) 297 | pending_attached_media.append((media_futures, subject_row)) 298 | 299 | move_created(MAX_PENDING_SUBJECTS_ATTACHED_MEDIA) 300 | 301 | move_created(0) 302 | finally: 303 | if (len(pending_attached_media) > 0): 304 | click.echo('Error: Upload failed.', err=True) 305 | if click.confirm( 306 | 'Would you like to save the upload state to resume the ' 307 | 'upload later?', 308 | default=True, 309 | ): 310 | while True: 311 | state_file_name = 'panoptes-upload-attached-media.yaml' 312 | 313 | state_file_name = click.prompt('Enter filename to save to', default=state_file_name,) 314 | 315 | if not state_file_name.endswith('.yaml'): 316 | click.echo( 317 | 'Error: File name must end in ".yaml".', 318 | err=True, 319 | ) 320 | if click.confirm( 321 | 'Save to {}.yaml?'.format(state_file_name), 322 | default=True, 323 | ): 324 | state_file_name += '.yaml' 325 | else: 326 | continue 327 | if not is_valid_filename(state_file_name): 328 | click.echo( 329 | f'Error: {state_file_name} is not a valid file name', 330 | err=True, 331 | ) 332 | sanitized_filename = sanitize_filename(state_file_name,) 333 | if click.confirm(f'Save to {sanitized_filename}', default=True,): 334 | state_file_name = sanitized_filename 335 | else: 336 | continue 337 | if os.path.exists(state_file_name): 338 | if not click.confirm(f'File {state_file_name} already exists. Overwrite?', default=False,): 339 | continue 340 | break 341 | 342 | with open(state_file_name, 'w') as state_file: 343 | yaml.dump(upload_state, state_file) 344 | 345 | @subject.command() 346 | @click.argument("subject-id", required=True) 347 | def info(subject_id): 348 | subject = Subject.find(subject_id) 349 | click.echo(yaml.dump(subject.raw)) 350 | 351 | 352 | @subject.command() 353 | @click.option( 354 | "--force", 355 | "-f", 356 | is_flag=True, 357 | help="Delete without asking for confirmation.", 358 | ) 359 | @click.argument("subject-ids", required=True, nargs=-1, type=int) 360 | def delete(force, subject_ids): 361 | for subject_id in subject_ids: 362 | if not force: 363 | click.confirm("Delete subject {}?".format(subject_id), abort=True) 364 | Subject.find(subject_id).delete() 365 | 366 | 367 | @subject.command() 368 | @click.option( 369 | "--replace", 370 | "-r", 371 | is_flag=True, 372 | help="Replace all existing metadata rather than merging.", 373 | ) 374 | @click.argument("metadata-file", required=True, nargs=1) 375 | def update_metadata(replace, metadata_file): 376 | """ 377 | Updates subject metadata from a CSV file. 378 | 379 | The CSV file should contain a "subject_id" column listing subject IDs. 380 | All other column names are taken to be metadata keys. 381 | """ 382 | with open(metadata_file, "r") as metadata_f: 383 | total_tows = len(metadata_f.readlines()) 384 | metadata_f.seek(0) 385 | metadata_rows = csv.DictReader(metadata_f) 386 | with click.progressbar( 387 | metadata_rows, 388 | length=total_tows, 389 | label="Update subject metadata", 390 | ) as _metadata_rows: 391 | for metadata in _metadata_rows: 392 | subject_id = metadata.pop("subject_id") 393 | try: 394 | subject = Subject.find(subject_id) 395 | if replace: 396 | subject.metadata = dict(metadata) 397 | else: 398 | subject.metadata.update(metadata) 399 | subject.save() 400 | except Exception as e: 401 | click.echo(f"Failed to update subject {subject_id}: {e}", err=True) 402 | 403 | 404 | def echo_subject(subject): 405 | m = map(lambda l: list(l.values())[0], subject.locations) 406 | click.echo("{} {}".format(subject.id, " ".join(m))) 407 | 408 | 409 | def _validate_file(file_path): 410 | if not os.path.isfile(file_path): 411 | click.echo( 412 | 'Error: File "{}" could not be found.'.format( 413 | file_path, 414 | ), 415 | err=True, 416 | ) 417 | return False 418 | 419 | file_size = os.path.getsize(file_path) 420 | if file_size == 0: 421 | click.echo( 422 | 'Error: File "{}" is empty.'.format( 423 | file_path, 424 | ), 425 | err=True, 426 | ) 427 | return False 428 | elif file_size > MAX_UPLOAD_FILE_SIZE: 429 | click.echo( 430 | 'Error: File "{}" is {}, larger than the maximum {}.'.format( 431 | file_path, 432 | humanize.naturalsize(file_size), 433 | humanize.naturalsize(MAX_UPLOAD_FILE_SIZE), 434 | ), 435 | err=True, 436 | ) 437 | return False 438 | return True 439 | -------------------------------------------------------------------------------- /panoptes_cli/commands/subject_set.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import copy 3 | import os 4 | import re 5 | import sys 6 | import time 7 | import yaml 8 | 9 | import click 10 | import humanize 11 | 12 | from pathvalidate import is_valid_filename, sanitize_filename 13 | 14 | from panoptes_cli.scripts.panoptes import cli 15 | from panoptes_client import SubjectSet 16 | from panoptes_client.panoptes import PanoptesAPIException 17 | 18 | LINK_BATCH_SIZE = 10 19 | MAX_PENDING_SUBJECTS = 50 20 | MAX_UPLOAD_FILE_SIZE = 1024 * 1024 21 | CURRENT_STATE_VERSION = 1 22 | 23 | @cli.group(name='subject-set') 24 | def subject_set(): 25 | """Contains commands for managing subject sets.""" 26 | pass 27 | 28 | 29 | @subject_set.command() 30 | @click.argument('subject-set-id', required=False, type=int) 31 | @click.option( 32 | '--project-id', 33 | '-p', 34 | help="Show subject sets belonging to the given project.", 35 | required=False, 36 | type=int 37 | ) 38 | @click.option( 39 | '--workflow-id', 40 | '-w', 41 | help="Show subject sets linked to the given workflow.", 42 | required=False, 43 | type=int 44 | ) 45 | @click.option( 46 | '--quiet', 47 | '-q', 48 | help='Only print subject set IDs (omit names).', 49 | is_flag=True, 50 | ) 51 | def ls(subject_set_id, project_id, workflow_id, quiet): 52 | """Lists subject set IDs and names""" 53 | 54 | if subject_set_id and not project_id and not workflow_id: 55 | subject_set = SubjectSet.find(subject_set_id) 56 | if quiet: 57 | click.echo(subject_set.id) 58 | else: 59 | echo_subject_set(subject_set) 60 | return 61 | 62 | args = {} 63 | if project_id: 64 | args['project_id'] = project_id 65 | if workflow_id: 66 | args['workflow_id'] = workflow_id 67 | if subject_set_id: 68 | args['subject_set_id'] = subject_set_id 69 | 70 | subject_sets = SubjectSet.where(**args) 71 | 72 | if quiet: 73 | click.echo(" ".join([s.id for s in subject_sets])) 74 | else: 75 | for subject_set in subject_sets: 76 | echo_subject_set(subject_set) 77 | 78 | 79 | @subject_set.command() 80 | @click.argument('subject-set-id', required=True) 81 | def info(subject_set_id): 82 | subject_set = SubjectSet.find(subject_set_id) 83 | click.echo(yaml.dump(subject_set.raw)) 84 | 85 | 86 | @subject_set.command() 87 | @click.option( 88 | '--quiet', 89 | '-q', 90 | help='Only print subject set ID (omit name).', 91 | is_flag=True, 92 | ) 93 | @click.argument('project-id', required=True, type=int) 94 | @click.argument('display-name', required=True) 95 | def create(quiet, project_id, display_name): 96 | """ 97 | Creates a new subject set. 98 | 99 | Prints the subject set ID and name of the new subject set. 100 | """ 101 | 102 | subject_set = SubjectSet() 103 | subject_set.links.project = project_id 104 | subject_set.display_name = display_name 105 | subject_set.save() 106 | 107 | if quiet: 108 | click.echo(subject_set.id) 109 | else: 110 | echo_subject_set(subject_set) 111 | 112 | 113 | @subject_set.command() 114 | @click.argument('subject-set-id', required=True, type=int) 115 | @click.option( 116 | '--display-name', 117 | '-n', 118 | help="Sets the subject set's public display name.", 119 | required=False 120 | ) 121 | def modify(subject_set_id, display_name): 122 | """ 123 | Changes the attributes of an existing subject set. 124 | 125 | Any attributes which are not specified are left unchanged. 126 | """ 127 | subject_set = SubjectSet.find(subject_set_id) 128 | if display_name: 129 | subject_set.display_name = display_name 130 | subject_set.save() 131 | echo_subject_set(subject_set) 132 | 133 | 134 | @subject_set.command(name='upload-subjects') 135 | @click.argument('subject-set-id', required=True, type=int) 136 | @click.argument('manifest-files', required=True, nargs=-1) 137 | @click.option( 138 | '--allow-missing', 139 | '-M', 140 | help=("Do not abort when creating subjects with no media files."), 141 | is_flag=True, 142 | ) 143 | @click.option( 144 | '--remote-location', 145 | '-r', 146 | help=( 147 | "Specify a field (by column number) in the manifest which contains a " 148 | "URL to a remote media location. Can be used more than once." 149 | ), 150 | multiple=True, 151 | type=int, 152 | required=False, 153 | ) 154 | @click.option( 155 | '--mime-type', 156 | '-m', 157 | help=( 158 | "MIME type for remote media. Defaults to image/png. Can be used more " 159 | "than once, in which case types are mapped one to one with remote " 160 | "locations in the order they are given. Has no effect without " 161 | "--remote-location." 162 | ), 163 | type=str, 164 | required=False, 165 | default=('image/png',), 166 | multiple=True 167 | ) 168 | @click.option( 169 | '--file-column', 170 | '-f', 171 | help=( 172 | "Specify a field (by column number) in the manifest which contains a " 173 | "local file to be uploaded. Can be used more than once. Disables auto-" 174 | "detection of filename columns." 175 | ), 176 | multiple=True, 177 | type=int, 178 | required=False, 179 | ) 180 | def upload_subjects( 181 | subject_set_id, 182 | manifest_files, 183 | allow_missing, 184 | remote_location, 185 | mime_type, 186 | file_column, 187 | ): 188 | """ 189 | Uploads subjects from each of the given MANIFEST_FILES. 190 | 191 | Example with only local files: 192 | 193 | $ panoptes subject-set upload-subjects 4667 manifest.csv 194 | 195 | Local filenames will be automatically detected in the manifest and 196 | uploaded, or filename columns can be specified with --file-column. 197 | 198 | If you are hosting your media yourself, you can put the URLs in the 199 | manifest and specify the column number(s): 200 | 201 | $ panoptes subject-set upload-subjects -r 1 4667 manifest.csv 202 | 203 | $ panoptes subject-set upload-subjects -r 1 -r 2 4667 manifest.csv 204 | 205 | Any local files will still be detected and uploaded. 206 | """ 207 | if ( 208 | len(manifest_files) > 1 209 | and any(map(lambda m: m.endswith('.yaml'), manifest_files)) 210 | ): 211 | click.echo( 212 | 'Error: YAML manifests must be processed one at a time.', 213 | err=True, 214 | ) 215 | return -1 216 | elif manifest_files[0].endswith('.yaml'): 217 | with open(manifest_files[0], 'r') as yaml_manifest: 218 | upload_state = yaml.load(yaml_manifest, Loader=yaml.FullLoader) 219 | if upload_state['state_version'] > CURRENT_STATE_VERSION: 220 | click.echo( 221 | 'Error: {} was generated by a newer version of the Panoptes ' 222 | 'CLI and is not compatible with this version.'.format( 223 | manifest_files[0], 224 | ), 225 | err=True, 226 | ) 227 | return -1 228 | if upload_state['subject_set_id'] != subject_set_id: 229 | click.echo( 230 | 'Warning: You specified subject set {} but this YAML ' 231 | 'manifest is for subject set {}.'.format( 232 | subject_set_id, 233 | upload_state['subject_set_id'], 234 | ), 235 | err=True, 236 | ) 237 | click.confirm( 238 | 'Upload {} to subject set {} ({})?'.format( 239 | manifest_files[0], 240 | subject_set_id, 241 | SubjectSet.find(subject_set_id).display_name, 242 | ), 243 | abort=True 244 | ) 245 | upload_state['subject_set_id'] = subject_set_id 246 | resumed_upload = True 247 | else: 248 | upload_state = { 249 | 'state_version': CURRENT_STATE_VERSION, 250 | 'subject_set_id': subject_set_id, 251 | 'manifest_files': manifest_files, 252 | 'allow_missing': allow_missing, 253 | 'remote_location': remote_location, 254 | 'mime_type': mime_type, 255 | 'file_column': file_column, 256 | 'waiting_to_upload': [], 257 | 'waiting_to_link': {}, 258 | } 259 | resumed_upload = False 260 | 261 | remote_location_count = len(upload_state['remote_location']) 262 | mime_type_count = len(upload_state['mime_type']) 263 | if remote_location_count > 1 and mime_type_count == 1: 264 | upload_state['mime_type'] = ( 265 | upload_state['mime_type'] * remote_location_count 266 | ) 267 | elif remote_location_count > 0 and mime_type_count != remote_location_count: 268 | click.echo( 269 | 'Error: The number of MIME types given must be either 1 or equal ' 270 | 'to the number of remote locations.', 271 | err=True, 272 | ) 273 | return -1 274 | 275 | def validate_file(file_path): 276 | if not os.path.isfile(file_path): 277 | click.echo( 278 | 'Error: File "{}" could not be found.'.format( 279 | file_path, 280 | ), 281 | err=True, 282 | ) 283 | return False 284 | 285 | file_size = os.path.getsize(file_path) 286 | if file_size == 0: 287 | click.echo( 288 | 'Error: File "{}" is empty.'.format( 289 | file_path, 290 | ), 291 | err=True, 292 | ) 293 | return False 294 | elif file_size > MAX_UPLOAD_FILE_SIZE: 295 | click.echo( 296 | 'Error: File "{}" is {}, larger than the maximum {}.'.format( 297 | file_path, 298 | humanize.naturalsize(file_size), 299 | humanize.naturalsize(MAX_UPLOAD_FILE_SIZE), 300 | ), 301 | err=True, 302 | ) 303 | return False 304 | return True 305 | 306 | def get_index_fields(headers): 307 | index_fields = [header.lstrip('%') for header in headers if header.startswith('%')] 308 | return ",".join(str(field) for field in index_fields) 309 | 310 | subject_set = SubjectSet.find(upload_state['subject_set_id']) 311 | if not resumed_upload: 312 | subject_rows = [] 313 | for manifest_file in upload_state['manifest_files']: 314 | with open(manifest_file) as manifest_f: 315 | file_root = os.path.dirname(manifest_file) 316 | r = csv.reader(manifest_f, skipinitialspace=True) 317 | headers = next(r) 318 | # update set metadata for indexed sets 319 | index_fields = get_index_fields(headers) 320 | if index_fields: 321 | subject_set.metadata['indexFields'] = index_fields 322 | subject_set.save() 323 | # remove leading % from subject metadata headings 324 | cleaned_headers = [header.lstrip('%') for header in headers] 325 | for row in r: 326 | metadata = dict(zip(cleaned_headers, row)) 327 | files = [] 328 | if not upload_state['file_column']: 329 | upload_state['file_column'] = [] 330 | for field_number, col in enumerate(row, start=1): 331 | file_path = os.path.join(file_root, col) 332 | if os.path.exists(file_path): 333 | upload_state['file_column'].append( 334 | field_number, 335 | ) 336 | if validate_file(file_path): 337 | files.append(file_path) 338 | elif not upload_state['allow_missing']: 339 | return -1 340 | else: 341 | for field_number in upload_state['file_column']: 342 | file_path = os.path.join( 343 | file_root, 344 | row[field_number - 1] 345 | ) 346 | if validate_file(file_path): 347 | files.append(file_path) 348 | elif not upload_state['allow_missing']: 349 | return -1 350 | 351 | for field_number, _mime_type in zip( 352 | upload_state['remote_location'], 353 | upload_state['mime_type'], 354 | ): 355 | files.append({_mime_type: row[field_number - 1]}) 356 | 357 | if len(files) == 0: 358 | click.echo( 359 | 'Could not find any files in row:', 360 | err=True, 361 | ) 362 | click.echo(','.join(row), err=True) 363 | if not upload_state['allow_missing']: 364 | return -1 365 | else: 366 | continue 367 | subject_rows.append((files, metadata)) 368 | 369 | if not subject_rows: 370 | click.echo( 371 | 'File {} did not contain any rows.'.format( 372 | manifest_file, 373 | ), 374 | err=True, 375 | ) 376 | return -1 377 | 378 | subject_rows = list(enumerate(subject_rows)) 379 | upload_state['waiting_to_upload'] = copy.deepcopy(subject_rows) 380 | else: 381 | for subject_id, subject_row in upload_state['waiting_to_link'].items(): 382 | try: 383 | subject = Subject.find(subject_id) 384 | except PanoptesAPIException: 385 | upload_state['waiting_to_upload'].append(subject_row) 386 | del upload_state['waiting_to_link'][subject_id] 387 | subject_rows = copy.deepcopy(upload_state['waiting_to_upload']) 388 | 389 | pending_subjects = [] 390 | 391 | def move_created(limit): 392 | while len(pending_subjects) > limit: 393 | for subject, subject_row in pending_subjects: 394 | if subject.async_save_result: 395 | pending_subjects.remove((subject, subject_row)) 396 | upload_state['waiting_to_upload'].remove(subject_row) 397 | upload_state['waiting_to_link'][subject.id] = subject_row 398 | time.sleep(0.5) 399 | 400 | def link_subjects(limit): 401 | if len(upload_state['waiting_to_link']) > limit: 402 | subject_set.add(list(upload_state['waiting_to_link'].keys())) 403 | upload_state['waiting_to_link'].clear() 404 | 405 | with click.progressbar( 406 | subject_rows, 407 | length=len(subject_rows), 408 | label='Uploading subjects', 409 | ) as _subject_rows: 410 | try: 411 | with Subject.async_saves(): 412 | for subject_row in _subject_rows: 413 | count, (files, metadata) = subject_row 414 | subject = Subject() 415 | subject.links.project = subject_set.links.project 416 | for media_file in files: 417 | subject.add_location(media_file) 418 | subject.metadata.update(metadata) 419 | subject.save() 420 | 421 | pending_subjects.append((subject, subject_row)) 422 | 423 | move_created(MAX_PENDING_SUBJECTS) 424 | link_subjects(LINK_BATCH_SIZE) 425 | 426 | move_created(0) 427 | link_subjects(0) 428 | finally: 429 | if ( 430 | len(pending_subjects) > 0 431 | or len(upload_state['waiting_to_link']) > 0 432 | ): 433 | click.echo('Error: Upload failed.', err=True) 434 | if click.confirm( 435 | 'Would you like to save the upload state to resume the ' 436 | 'upload later?', 437 | default=True, 438 | ): 439 | while True: 440 | state_file_name = 'panoptes-upload-{}.yaml'.format( 441 | subject_set_id, 442 | ) 443 | state_file_name = click.prompt( 444 | 'Enter filename to save to', 445 | default=state_file_name, 446 | ) 447 | 448 | if not state_file_name.endswith('.yaml'): 449 | click.echo( 450 | 'Error: File name must end in ".yaml".', 451 | err=True, 452 | ) 453 | if click.confirm( 454 | 'Save to {}.yaml?'.format(state_file_name), 455 | default=True, 456 | ): 457 | state_file_name += '.yaml' 458 | else: 459 | continue 460 | if not is_valid_filename(state_file_name): 461 | click.echo( 462 | 'Error: {} is not a valid file name'.format( 463 | state_file_name, 464 | ), 465 | err=True, 466 | ) 467 | sanitized_filename = sanitize_filename( 468 | state_file_name, 469 | ) 470 | if click.confirm( 471 | 'Save to {}?'.format( 472 | sanitized_filename, 473 | ), 474 | default=True, 475 | ): 476 | state_file_name = sanitized_filename 477 | else: 478 | continue 479 | if os.path.exists(state_file_name): 480 | if not click.confirm( 481 | 'File {} already exists. Overwrite?'.format( 482 | state_file_name, 483 | ), 484 | default=False, 485 | ): 486 | continue 487 | break 488 | 489 | with open(state_file_name, 'w') as state_file: 490 | yaml.dump(upload_state, state_file) 491 | 492 | 493 | @subject_set.command(name='add-subjects') 494 | @click.argument('subject-set-id', required=True, type=int) 495 | @click.argument('subject-ids', required=False, nargs=-1) 496 | @click.option( 497 | '--id-file', 498 | '-f', 499 | type=click.File('r'), 500 | help=( 501 | "Specify a filename which contains a list of subject IDs, one per line." 502 | ), 503 | ) 504 | def add_subjects(subject_set_id, subject_ids, id_file): 505 | """ 506 | Links existing subjects to this subject set. 507 | 508 | This command is useful mainly for adding previously uploaded subjects to 509 | additional subject sets. 510 | 511 | See the upload-subjects command to create new subjects in a subject set. 512 | """ 513 | s = SubjectSet.find(subject_set_id) 514 | if id_file: 515 | s.add([l.strip() for l in id_file.readlines()]) 516 | if subject_ids: 517 | s.add(subject_ids) 518 | 519 | 520 | @subject_set.command(name='remove-subjects') 521 | @click.argument('subject-set-id', required=True, type=int) 522 | @click.argument('subject-ids', required=False, nargs=-1) 523 | @click.option( 524 | '--id-file', 525 | '-f', 526 | type=click.File('r'), 527 | help=( 528 | "Specify a filename which contains a list of subject IDs, one per line." 529 | ), 530 | ) 531 | def remove_subjects(subject_set_id, subject_ids, id_file): 532 | """ 533 | Unlinks subjects from this subject set. 534 | 535 | The subjects themselves are not deleted or modified in any way and will 536 | still be present in any other sets they're linked to. 537 | """ 538 | 539 | s = SubjectSet.find(subject_set_id) 540 | if id_file: 541 | s.remove([l.strip() for l in id_file.readlines()]) 542 | if subject_ids: 543 | s.remove(subject_ids) 544 | 545 | 546 | @subject_set.command() 547 | @click.option( 548 | '--force', 549 | '-f', 550 | is_flag=True, 551 | help='Delete without asking for confirmation.', 552 | ) 553 | @click.argument('subject-set-ids', required=True, nargs=-1, type=int) 554 | def delete(force, subject_set_ids): 555 | for subject_set_id in subject_set_ids: 556 | subject_set = SubjectSet.find(subject_set_id) 557 | if not force: 558 | click.confirm( 559 | 'Delete subject set {} ({})?'.format( 560 | subject_set_id, 561 | subject_set.display_name, 562 | ), 563 | abort=True, 564 | ) 565 | subject_set.delete() 566 | 567 | 568 | @subject_set.command(name="download-classifications") 569 | @click.argument('subject-set-id', required=True, type=int) 570 | @click.argument('output-file', required=True, type=click.File('wb')) 571 | @click.option( 572 | '--generate', 573 | '-g', 574 | help="Generates a new export before downloading.", 575 | is_flag=True 576 | ) 577 | @click.option( 578 | '--generate-timeout', 579 | '-T', 580 | help=( 581 | "Time in seconds to wait for new export to be ready. Defaults to " 582 | "unlimited. Has no effect unless --generate is given." 583 | ), 584 | required=False, 585 | type=int, 586 | ) 587 | def download_classifications( 588 | subject_set_id, 589 | output_file, 590 | generate, 591 | generate_timeout 592 | ): 593 | """ 594 | Downloads a subject-set specific classifications export for the given subject set. 595 | 596 | OUTPUT_FILE will be overwritten if it already exists. Set OUTPUT_FILE to - 597 | to output to stdout. 598 | """ 599 | 600 | subject_set = SubjectSet.find(subject_set_id) 601 | 602 | if generate: 603 | click.echo("Generating new export...", err=True) 604 | 605 | export = subject_set.get_export( 606 | 'classifications', 607 | generate=generate, 608 | wait_timeout=generate_timeout 609 | ) 610 | 611 | with click.progressbar( 612 | export.iter_content(chunk_size=1024), 613 | label='Downloading', 614 | length=(int(export.headers.get('content-length')) / 1024 + 1), 615 | file=click.get_text_stream('stderr'), 616 | ) as chunks: 617 | for chunk in chunks: 618 | output_file.write(chunk) 619 | 620 | 621 | def echo_subject_set(subject_set): 622 | click.echo( 623 | u'{} {}'.format( 624 | subject_set.id, 625 | subject_set.display_name 626 | ) 627 | ) 628 | 629 | 630 | from panoptes_client import Subject 631 | --------------------------------------------------------------------------------