├── .coveragerc ├── .dockerignore ├── .github ├── CODEOWNERS ├── pull_request_template.md └── workflows │ ├── build-docs.yml │ ├── build.yml │ └── codeql-analysis.yml ├── .gitignore ├── .pep8speaks.yml ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── ISSUE_TEMPLATE.md ├── LICENSE ├── MANIFEST.in ├── Pipfile ├── Pipfile.lock ├── README.md ├── SECURITY.md ├── docs ├── .nojekyll ├── CNAME ├── assets │ ├── favicon.png │ ├── synapse_logo.png │ └── under_construction.png ├── css │ └── custom.css ├── explanations │ ├── README │ ├── access_control.md │ ├── assets │ │ └── annotation_example_1.png │ ├── benchmarking.md │ ├── domain_models_of_synapse.md │ ├── home.md │ ├── manifest_tsv.md │ ├── properties_vs_annotations.md │ └── structuring_your_project.md ├── guides │ ├── README │ ├── accessing_the_rest_api.md │ ├── data_storage.md │ ├── home.md │ ├── validate_annotations.md │ └── views.md ├── index.html ├── index.md ├── news.md ├── reference │ ├── README │ ├── activity.md │ ├── annotations.md │ ├── cli.md │ ├── client.md │ ├── core.md │ ├── docker_repository.md │ ├── entity.md │ ├── evaluation.md │ ├── exceptions.md │ ├── experimental │ │ ├── async │ │ │ ├── activity.md │ │ │ ├── agent.md │ │ │ ├── dataset.md │ │ │ ├── dataset_collection.md │ │ │ ├── entityview.md │ │ │ ├── file.md │ │ │ ├── folder.md │ │ │ ├── materializedview.md │ │ │ ├── project.md │ │ │ ├── submissionview.md │ │ │ ├── table.md │ │ │ ├── team.md │ │ │ ├── user_profile.md │ │ │ └── virtualtable.md │ │ ├── functional_interfaces.md │ │ ├── mixins │ │ │ ├── access_controllable.md │ │ │ ├── asynchronous_communicator.md │ │ │ ├── failure_strategy.md │ │ │ └── storable_container.md │ │ └── sync │ │ │ ├── activity.md │ │ │ ├── agent.md │ │ │ ├── dataset.md │ │ │ ├── dataset_collection.md │ │ │ ├── entityview.md │ │ │ ├── file.md │ │ │ ├── folder.md │ │ │ ├── materializedview.md │ │ │ ├── project.md │ │ │ ├── submissionview.md │ │ │ ├── table.md │ │ │ ├── team.md │ │ │ ├── user_profile.md │ │ │ └── virtualtable.md │ ├── file.md │ ├── folder.md │ ├── json_schema.md │ ├── link.md │ ├── permissions.md │ ├── project.md │ ├── rest_apis.md │ ├── synapse_utils.md │ ├── table_schema.md │ ├── tables.md │ ├── teams.md │ ├── view_schema.md │ └── wiki.md ├── requirements.txt ├── scripts │ ├── downloadBenchmark.py │ ├── object_orientated_programming_poc │ │ ├── oop_poc_activity.py │ │ ├── oop_poc_agent.py │ │ ├── oop_poc_file.py │ │ ├── oop_poc_folder.py │ │ ├── oop_poc_project.py │ │ ├── oop_poc_table.py │ │ ├── oop_poc_team.py │ │ ├── oop_poc_user_and_permission.py │ │ └── synapse_project.py │ ├── print_release_issues.py │ ├── uploadBenchmark.py │ └── uploadTestFiles.py └── tutorials │ ├── README │ ├── authentication.md │ ├── command_line_client.md │ ├── configuration.md │ ├── file_versioning.md │ ├── home.md │ ├── installation.md │ ├── python │ ├── activity.md │ ├── annotation.md │ ├── dataset.md │ ├── dataset_collection.md │ ├── download_data_in_bulk.md │ ├── entityview.md │ ├── file.md │ ├── folder.md │ ├── materializedview.md │ ├── migrate_data_to_other_storage_locations.md │ ├── move_files_and_folders.md │ ├── project.md │ ├── sharing_settings.md │ ├── submissionview.md │ ├── table.md │ ├── table_crud.md │ ├── team.md │ ├── tutorial_screenshots │ │ ├── annotation.png │ │ ├── dataset_collection_default_schema.png │ │ ├── dataset_collection_with_datasets.png │ │ ├── dataset_default_schema.png │ │ ├── dataset_with_files.png │ │ ├── edit_provenance_button.png │ │ ├── edit_provenance_screen.png │ │ ├── entityview.png │ │ ├── file.png │ │ └── folder.png │ ├── tutorial_scripts │ │ ├── annotation.py │ │ ├── dataset.py │ │ ├── dataset_collection.py │ │ ├── download_data_in_bulk.py │ │ ├── entityview.py │ │ ├── file.py │ │ ├── folder.py │ │ ├── materializedview.py │ │ ├── project.py │ │ ├── submissionview.py │ │ ├── upload_data_in_bulk.py │ │ └── virtualtable.py │ ├── upload_data_in_bulk.md │ ├── versions.md │ ├── virtualtable.md │ └── wiki.md │ ├── python_client.md │ ├── reticulate.md │ ├── sample_files │ ├── README │ └── my_ad_project │ │ ├── experiment_notes │ │ ├── notes_2022 │ │ │ ├── fileA.txt │ │ │ └── fileB.txt │ │ └── notes_2023 │ │ │ ├── fileC.txt │ │ │ └── fileD.txt │ │ ├── single_cell_RNAseq_batch_1 │ │ ├── SRR12345678_R1.fastq.gz │ │ ├── SRR12345678_R2.fastq.gz │ │ ├── SRR92345678_R1.fastq.gz │ │ └── SRR92345678_R2.fastq.gz │ │ └── single_cell_RNAseq_batch_2 │ │ ├── SRR12345678_R1.fastq.gz │ │ └── SRR12345678_R2.fastq.gz │ └── tables.md ├── mkdocs.yml ├── pylintrc ├── pyproject.toml ├── pytest.ini ├── setup.cfg ├── setup.py ├── sonar-project.properties ├── synapseclient ├── .synapseConfig ├── __init__.py ├── __main__.py ├── activity.py ├── annotations.py ├── api │ ├── __init__.py │ ├── agent_services.py │ ├── annotations.py │ ├── configuration_services.py │ ├── entity_bundle_services_v2.py │ ├── entity_factory.py │ ├── entity_services.py │ ├── file_services.py │ └── table_services.py ├── client.py ├── core │ ├── __init__.py │ ├── async_utils.py │ ├── cache.py │ ├── config.py │ ├── constants │ │ ├── __init__.py │ │ ├── concrete_types.py │ │ ├── config_file_constants.py │ │ ├── limits.py │ │ └── method_flags.py │ ├── credentials │ │ ├── __init__.py │ │ ├── cred_data.py │ │ └── credential_provider.py │ ├── cumulative_transfer_progress.py │ ├── download │ │ ├── __init__.py │ │ ├── download_async.py │ │ └── download_functions.py │ ├── dozer.py │ ├── exceptions.py │ ├── lock.py │ ├── logging_setup.py │ ├── models │ │ ├── __init__.py │ │ ├── custom_json.py │ │ ├── dict_object.py │ │ └── permission.py │ ├── multithread_download │ │ ├── __init__.py │ │ └── download_threads.py │ ├── pool_provider.py │ ├── remote_file_storage_wrappers.py │ ├── retry.py │ ├── sts_transfer.py │ ├── transfer_bar.py │ ├── upload │ │ ├── __init__.py │ │ ├── multipart_upload.py │ │ ├── multipart_upload_async.py │ │ ├── upload_functions.py │ │ ├── upload_functions_async.py │ │ └── upload_utils.py │ ├── utils.py │ └── version_check.py ├── entity.py ├── evaluation.py ├── models │ ├── __init__.py │ ├── activity.py │ ├── agent.py │ ├── annotations.py │ ├── dataset.py │ ├── entityview.py │ ├── evaluation.py │ ├── file.py │ ├── folder.py │ ├── materializedview.py │ ├── mixins │ │ ├── __init__.py │ │ ├── access_control.py │ │ ├── asynchronous_job.py │ │ ├── storable_container.py │ │ └── table_components.py │ ├── project.py │ ├── protocols │ │ ├── __init__.py │ │ ├── access_control_protocol.py │ │ ├── activity_protocol.py │ │ ├── agent_protocol.py │ │ ├── annotations_protocol.py │ │ ├── file_protocol.py │ │ ├── folder_protocol.py │ │ ├── project_protocol.py │ │ ├── storable_container_protocol.py │ │ ├── table_protocol.py │ │ ├── team_protocol.py │ │ └── user_protocol.py │ ├── services │ │ ├── __init__.py │ │ ├── search.py │ │ ├── storable_entity.py │ │ └── storable_entity_components.py │ ├── submissionview.py │ ├── table.py │ ├── table_components.py │ ├── team.py │ ├── user.py │ ├── virtualtable.py │ └── wiki.py ├── services │ ├── __init__.py │ └── json_schema.py ├── synapsePythonClient ├── table.py ├── team.py └── wiki.py ├── synapseutils ├── __init__.py ├── copy_functions.py ├── describe_functions.py ├── migrate_functions.py ├── monitor.py ├── sync.py └── walk_functions.py ├── test.synapseConfig.enc └── tests ├── __init__.py ├── integration ├── __init__.py ├── conftest.py ├── synapse_creds.sh ├── synapseclient │ ├── __init__.py │ ├── core │ │ ├── __init__.py │ │ ├── test_REST_request_helpers.py │ │ ├── test_caching.py │ │ ├── test_download.py │ │ ├── test_external_storage.py │ │ ├── test_version_check.py │ │ └── upload │ │ │ ├── Dockerfile_sftp │ │ │ ├── __init__.py │ │ │ ├── test_multipart_upload.py │ │ │ ├── test_multipart_upload_async.py │ │ │ └── test_sftp_upload.py │ ├── integration_test.py │ ├── integration_test_Entity.py │ ├── models │ │ ├── async │ │ │ ├── test_activity_async.py │ │ │ ├── test_agent_async.py │ │ │ ├── test_dataset_async.py │ │ │ ├── test_entityview_async.py │ │ │ ├── test_file_async.py │ │ │ ├── test_folder_async.py │ │ │ ├── test_materializedview_async.py │ │ │ ├── test_permissions_async.py │ │ │ ├── test_project_async.py │ │ │ ├── test_submissionview_async.py │ │ │ ├── test_table_async.py │ │ │ ├── test_team_async.py │ │ │ ├── test_user_async.py │ │ │ └── test_virtualtable_async.py │ │ └── synchronous │ │ │ ├── test_activity.py │ │ │ ├── test_agent.py │ │ │ ├── test_dataset.py │ │ │ ├── test_entityview.py │ │ │ ├── test_file.py │ │ │ ├── test_folder.py │ │ │ ├── test_materializedview.py │ │ │ ├── test_permissions.py │ │ │ ├── test_project.py │ │ │ ├── test_submissionview.py │ │ │ ├── test_table.py │ │ │ ├── test_team.py │ │ │ ├── test_user.py │ │ │ └── test_virtualtable.py │ ├── test_command_line_client.py │ ├── test_evaluations.py │ ├── test_json_schema_services.py │ ├── test_tables.py │ └── test_wikis.py └── synapseutils │ ├── __init__.py │ ├── test_synapseutils_copy.py │ ├── test_synapseutils_migrate.py │ ├── test_synapseutils_sync.py │ └── test_synapseutils_walk.py ├── test_utils.py └── unit ├── __init__.py ├── conftest.py ├── synapseclient ├── __init__.py ├── core │ ├── __init__.py │ ├── credentials │ │ ├── __init__.py │ │ ├── unit_test_cred_data.py │ │ └── unit_test_cred_provider.py │ ├── download │ │ ├── unit_test_download_async.py │ │ └── unit_test_download_functions.py │ ├── models │ │ ├── __init__.py │ │ ├── unit_test_DictObject.py │ │ └── unit_test_custom_json.py │ ├── multithread_download │ │ ├── __init__.py │ │ └── unit_test_download_threads.py │ ├── unit_test_Cache.py │ ├── unit_test_cumulative_transfer_progress.py │ ├── unit_test_download.py │ ├── unit_test_doze.py │ ├── unit_test_lock.py │ ├── unit_test_pool_provider.py │ ├── unit_test_remote_storage_file_wrappers.py │ ├── unit_test_retry.py │ ├── unit_test_sts_transfer.py │ ├── unit_test_utils.py │ ├── unit_test_version_check.py │ └── upload │ │ ├── __init__.py │ │ ├── unit_test_multipart_upload.py │ │ └── unit_test_upload_functions.py ├── mixins │ ├── async │ │ └── unit_test_asynchronous_job.py │ └── unit_test_table_components.py ├── models │ ├── async │ │ ├── unit_test_activity_async.py │ │ ├── unit_test_agent_async.py │ │ ├── unit_test_dataset_async.py │ │ ├── unit_test_file_async.py │ │ ├── unit_test_folder_async.py │ │ ├── unit_test_permissions_async.py │ │ ├── unit_test_project_async.py │ │ ├── unit_test_team_async.py │ │ └── unit_test_user_async.py │ ├── synchronous │ │ ├── unit_test_activity.py │ │ ├── unit_test_agent.py │ │ ├── unit_test_file.py │ │ ├── unit_test_folder.py │ │ ├── unit_test_project.py │ │ ├── unit_test_team.py │ │ └── unit_test_user.py │ ├── unit_test_entityview.py │ ├── unit_test_permissions.py │ ├── unit_test_table.py │ └── unit_test_virtualtable.py ├── services │ ├── __init__.py │ └── unit_test_json_schema.py ├── unit_test_Entity.py ├── unit_test_Evaluation.py ├── unit_test_Wiki.py ├── unit_test_activity.py ├── unit_test_annotations.py ├── unit_test_client.py ├── unit_test_commandline.py ├── unit_test_get_permissions.py ├── unit_test_get_user_profile.py ├── unit_test_tables.py └── unit_tests.py ├── synapseutils ├── __init__.py ├── unit_test_synapseutils_copy.py ├── unit_test_synapseutils_describe.py ├── unit_test_synapseutils_migrate.py ├── unit_test_synapseutils_monitor.py ├── unit_test_synapseutils_sync.py └── unit_test_synapseutils_walk.py └── test_utils ├── __init__.py └── unit_utils.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | docs/* 4 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Git 2 | .git 3 | .gitignore 4 | 5 | # CI 6 | .codeclimate.yml 7 | .travis.yml 8 | 9 | # Docker 10 | docker-compose.yml 11 | .docker 12 | 13 | # Byte-compiled / optimized / DLL files 14 | __pycache__/ 15 | */__pycache__/ 16 | */*/__pycache__/ 17 | */*/*/__pycache__/ 18 | *.py[cod] 19 | */*.py[cod] 20 | */*/*.py[cod] 21 | */*/*/*.py[cod] 22 | 23 | # C extensions 24 | *.so 25 | 26 | # Distribution / packaging 27 | .Python 28 | env/ 29 | build/ 30 | develop-eggs/ 31 | dist/ 32 | downloads/ 33 | eggs/ 34 | lib/ 35 | lib64/ 36 | parts/ 37 | sdist/ 38 | var/ 39 | *.egg-info/ 40 | .installed.cfg 41 | *.egg 42 | 43 | # PyInstaller 44 | # Usually these files are written by a python script from a template 45 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 46 | *.manifest 47 | *.spec 48 | 49 | # Installer logs 50 | pip-log.txt 51 | pip-delete-this-directory.txt 52 | 53 | # Unit test / coverage reports 54 | htmlcov/ 55 | .tox/ 56 | .coverage 57 | .cache 58 | nosetests.xml 59 | coverage.xml 60 | 61 | # Translations 62 | *.mo 63 | *.pot 64 | 65 | # Django stuff: 66 | *.log 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Virtual environment 75 | .env/ 76 | .venv/ 77 | venv/ 78 | 79 | # PyCharm 80 | .idea 81 | 82 | # Python mode for VIM 83 | .ropeproject 84 | */.ropeproject 85 | */*/.ropeproject 86 | */*/*/.ropeproject 87 | 88 | # Vim swap files 89 | *.swp 90 | */*.swp 91 | */*/*.swp 92 | */*/*/*.swp 93 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @Sage-Bionetworks/python-client-developers 2 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # **Problem:** 2 | 3 | - Describe the specific technical problem that this PR solves. 4 | - Include details such as: 5 | - How you reproduced the problem 6 | - What code is affected 7 | - Any relevant background information 8 | 9 | # **Solution:** 10 | 11 | - Describe the specific technical solution that this PR provides. 12 | - Include details such as: 13 | - How you debugged the problem 14 | - Why you chose this solution 15 | - Does the solution make any impact beyond the immediate problem 16 | - Any necessary technical debt incurred 17 | - Any important design decisions/links to design documentation (if applicable) 18 | 19 | # **Testing:** 20 | 21 | - Describe any testing that was performed to validate the solution. 22 | - Include details such as: 23 | - How you tested the solution 24 | - Any relevant test results (Do not include any test results that may contain sensitive information) 25 | - Any automated tests that were added to the codebase 26 | -------------------------------------------------------------------------------- /.github/workflows/build-docs.yml: -------------------------------------------------------------------------------- 1 | name: Build Docs 2 | on: 3 | push: 4 | branches: 5 | - master 6 | 7 | jobs: 8 | deploy: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - name: Configure Git Credentials 13 | run: | 14 | git config user.name github-actions[bot] 15 | git config user.email 41898282+github-actions[bot]@users.noreply.github.com 16 | - uses: actions/setup-python@v4 17 | with: 18 | python-version: 3.x 19 | - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 20 | - uses: actions/cache@v4 21 | with: 22 | key: mkdocs-material-${{ env.cache_id }} 23 | path: .cache 24 | restore-keys: | 25 | mkdocs-material- 26 | - run: pip install mkdocs mkdocs-material mkdocstrings mkdocstrings-python termynal markdown-include mkdocs-open-in-new-tab 27 | - run: mkdocs gh-deploy --force 28 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "develop", master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "develop" ] 20 | schedule: 21 | - cron: '20 14 * * 1' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v4 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v3 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v3 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v3 73 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | pypi.info.txt 4 | dist 5 | virtualenvs 6 | examples_temp 7 | .DS_Store 8 | deploy.sh 9 | 10 | junk/ 11 | nose.cfg 12 | notes.md 13 | notes/ 14 | examples/ 15 | jenkins/ 16 | 17 | .idea/* 18 | docs/build/doctrees/* 19 | docs/build/html/_sources/* 20 | build/* 21 | 22 | /venv 23 | 24 | .vscode/ 25 | CONFIGFILE 26 | 27 | htmlcov/ 28 | .coverage 29 | .coverage.* 30 | cov.xml 31 | coverage.xml 32 | 33 | .ipynb_checkpoints 34 | *.ipynb 35 | -------------------------------------------------------------------------------- /.pep8speaks.yml: -------------------------------------------------------------------------------- 1 | scanner: 2 | diff_only: True # If False, the entire file touched by the Pull Request is scanned for errors. If True, only the diff is scanned. 3 | linter: flake8 4 | 5 | # Configure to support black code formatting 6 | flake8: 7 | max-line-length: 88 8 | ignore: 9 | - E203 # whitespace before ',' 10 | - W503 # line break before binary operator 11 | 12 | no_blank_comment: False # If True, no comment is made on PR without any errors. 13 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: "^docs/conf.py|^docs/build/" 2 | 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v4.4.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: check-added-large-files 9 | - id: check-ast 10 | - id: check-json 11 | - id: check-merge-conflict 12 | - id: check-xml 13 | - id: check-yaml 14 | exclude: ^mkdocs\.yml$ 15 | - id: debug-statements 16 | - id: end-of-file-fixer 17 | - id: requirements-txt-fixer 18 | - id: mixed-line-ending 19 | args: ["--fix=auto"] # replace 'auto' with 'lf' to enforce Linux/Mac line endings or 'crlf' for Windows 20 | 21 | - repo: https://github.com/astral-sh/ruff-pre-commit 22 | rev: "v0.0.270" 23 | hooks: 24 | - id: ruff 25 | 26 | - repo: https://github.com/pycqa/isort 27 | rev: 5.13.2 28 | hooks: 29 | - id: isort 30 | name: isort (python) 31 | 32 | - repo: https://github.com/psf/black 33 | rev: 23.3.0 34 | hooks: 35 | - id: black 36 | language_version: python3 37 | 38 | - repo: https://github.com/PyCQA/bandit 39 | rev: 1.7.5 40 | hooks: 41 | - id: bandit 42 | args: ["-c", "pyproject.toml"] 43 | additional_dependencies: ["bandit[toml]"] 44 | # - repo: https://github.com/asottile/blacken-docs 45 | # rev: v1.12.0 46 | # hooks: 47 | # - id: blacken-docs 48 | # additional_dependencies: [black] 49 | 50 | # - repo: https://github.com/pre-commit/mirrors-mypy 51 | # rev: 'v1.0.1' 52 | # hooks: 53 | # - id: mypy 54 | # additional_dependencies: [pydantic~=1.10] 55 | 56 | # Checks for missing docstrings 57 | # - repo: https://github.com/econchick/interrogate 58 | # rev: 1.5.0 59 | # hooks: 60 | # - id: interrogate 61 | # exclude: ^(docs/conf.py|setup.py|tests) 62 | # args: [--config=pyproject.toml] 63 | 64 | # finds dead python code 65 | # - repo: https://github.com/jendrikseipp/vulture 66 | # rev: 'v2.7' 67 | # hooks: 68 | # - id: vulture 69 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for MkDocs projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the version of Python and other tools you might need 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | # alias for the latest 3.x version available on Read the Docs 12 | python: "3" 13 | 14 | mkdocs: 15 | configuration: mkdocs.yml 16 | 17 | # Optionally declare the Python requirements required to build your docs 18 | python: 19 | install: 20 | - requirements: docs/requirements.txt 21 | 22 | # See https://docs.readthedocs.com/platform/stable/config-file/v2.html#search for more options 23 | search: 24 | ranking: 25 | reference/*: -3 26 | reference/experimental/*: -2 27 | reference/experimental/sync/*: -1 28 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | We subscribe to [Sage Bionetwork's Code of Conduct](https://sagebionetworks.org/code-of-conduct/), which is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/). 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9-slim 2 | RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections 3 | 4 | RUN apt-get update \ 5 | && apt-get install --no-install-recommends -y \ 6 | gcc \ 7 | procps \ 8 | python3-dev \ 9 | python3-setuptools \ 10 | python3-pip \ 11 | && rm -rf /var/lib/apt/lists/* 12 | 13 | RUN pip3 install --upgrade pip 14 | 15 | WORKDIR /synapsePythonClient 16 | COPY . . 17 | 18 | RUN pip install --no-cache-dir .[pandas] 19 | 20 | 21 | LABEL org.opencontainers.image.source='https://github.com/Sage-Bionetworks/synapsePythonClient' 22 | -------------------------------------------------------------------------------- /ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Bug Report 2 | 3 | *Github issues is reserved for bug report. If you have a question, please don't use this form. 4 | Instead, please ask your question on the [Synapse Help Forum](https://www.synapse.org/#!SynapseForum:default).* 5 | 6 | ## Operating system 7 | 8 | *Which operating system are you working on? E.g., Ubuntu Linux, Max OSX, Windows 10, etc.* 9 | 10 | ## Client version 11 | *Output of:* 12 | ``` 13 | import synapseclient 14 | synapseclient.__version__ 15 | ``` 16 | 17 | ## Description of the problem 18 | 19 | *Provide a description of the problem, and if possible a minimal reproducible example.* 20 | 21 | A minimal reproducible example: 22 | * uses a minimal dataset necessary to reproduce the error 23 | * the minimal runnable code necessary to reproduce the error, which can be run on the given dataset 24 | * often do not include extra imports 25 | 26 | *Please don't share private data and credentials with us.* 27 | 28 | ### Expected behavior 29 | 30 | *What did you expect to happen?* 31 | 32 | ### Actual behavior 33 | 34 | *What actually happened? Provide output or error messages from the console, if applicable.* 35 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include synapseclient/synapsePythonClient 3 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [packages] 7 | synapseclient = {file = ".", editable = true, path = "."} 8 | 9 | [requires] 10 | python_version = "3.12.6" 11 | 12 | [dev-packages] 13 | synapseclient = {file = ".", editable = true, path = ".", extras = ["dev", "tests", "pandas", "pysftp", "boto3", "docs"]} 14 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | To report a security issue, please email `sagesecurity@sasebase.org` with a description of the 6 | issue, the steps you took to create the issue, affected versions, and, if known, mitigations for the 7 | issue. This project follows a 90 day disclosure timeline. 8 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/.nojekyll -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | python-docs.synapse.org 2 | -------------------------------------------------------------------------------- /docs/assets/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/assets/favicon.png -------------------------------------------------------------------------------- /docs/assets/synapse_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/assets/synapse_logo.png -------------------------------------------------------------------------------- /docs/assets/under_construction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/assets/under_construction.png -------------------------------------------------------------------------------- /docs/css/custom.css: -------------------------------------------------------------------------------- 1 | /* Indentation. */ 2 | div.doc-contents { 3 | padding-left: 25px; 4 | border-left: .05rem solid var(--md-typeset-table-color); 5 | } 6 | 7 | .md-typeset table tbody { 8 | font-size: .7rem 9 | } 10 | 11 | .md-grid { 12 | max-width: 1700px; 13 | } 14 | -------------------------------------------------------------------------------- /docs/explanations/README: -------------------------------------------------------------------------------- 1 | Explanations: 2 | The purpose of the documents in this folder is to dive further into the 3 | detail of specific concepts. It is meant to give those reading the 4 | information a deeper knowledge of how something works, why something was built, or the 5 | reason for decisions. These articles are meant to be linked to from other articles where 6 | it would not fit to go into further discussion about a topic. 7 | 8 | Guideline: 9 | - Provide detail to understand a problem, solution, and testing around that solution 10 | 11 | 12 | 13 | The reason for x is because historically, y… 14 | Explain. 15 | 16 | W is better than z, because… 17 | Offer judgements and even opinions where appropriate.. 18 | 19 | An x in system y is analogous to a w in system z. However… 20 | Provide context that helps the reader. 21 | 22 | Some users prefer w (because z). This can be a good approach, but… 23 | Weigh up alternatives. 24 | 25 | An x interacts with a y as follows:… 26 | Unfold the machinery’s internal secrets, to help understand why something does what it does. 27 | -------------------------------------------------------------------------------- /docs/explanations/access_control.md: -------------------------------------------------------------------------------- 1 | # Access Control 2 | 3 | By default, data sets in Synapse are private to your user account, but they can easily be shared with specific users, groups, or the public. 4 | 5 | See: 6 | 7 | - [synapseclient.Synapse.getPermissions][] 8 | - [synapseclient.Synapse.setPermissions][] 9 | -------------------------------------------------------------------------------- /docs/explanations/assets/annotation_example_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/explanations/assets/annotation_example_1.png -------------------------------------------------------------------------------- /docs/explanations/home.md: -------------------------------------------------------------------------------- 1 | # Further Reading 2 | Welcome! 3 | 4 | This is where you'll find information that gives you a deeper understanding of best practices and advanced use cases for interacting with Synapse. 5 | -------------------------------------------------------------------------------- /docs/explanations/properties_vs_annotations.md: -------------------------------------------------------------------------------- 1 | # Properties and annotations, implementation details 2 | 3 | In Synapse, entities have both properties and annotations. Properties are used by the system, whereas annotations are completely user defined. In the Python client, we try to present this situation as a normal object, with one set of properties. 4 | 5 | Printing an entity will show the division between properties and annotations. 6 | 7 | ```python 8 | print(entity) 9 | ``` 10 | 11 | Under the covers, an Entity object has two dictionaries, one for properties and one for annotations. These two namespaces are distinct, so there is a possibility of collisions. It is recommended to avoid defining annotations with names that collide with properties, but this is not enforced. 12 | 13 | ```python 14 | ## don't do this! 15 | entity.properties['description'] = 'One thing' 16 | entity.annotations['description'] = 'A different thing' 17 | ``` 18 | 19 | In case of conflict, properties will take precedence. 20 | 21 | ```python 22 | print(entity.description) 23 | #> One thing 24 | ``` 25 | 26 | Some additional ambiguity is entailed in the use of dot notation. Entity objects have their own internal properties which are not persisted to Synapse. As in all Python objects, these properties are held in object.__dict__. For example, this dictionary holds the keys 'properties' and 'annotations' whose values are both dictionaries themselves. 27 | 28 | The rule, for either getting or setting is: first look in the object then look in properties, then look in annotations. If the key is not found in any of these three, a get results in a `KeyError` and a set results in a new annotation being created. Thus, the following results in a new annotation that will be persisted in Synapse: 29 | 30 | ```python 31 | entity.foo = 'bar' 32 | ``` 33 | 34 | To create an object member variable, which will *not* be persisted in Synapse, this unfortunate notation is required: 35 | 36 | ```python 37 | entity.__dict__['foo'] = 'bar' 38 | ``` 39 | 40 | As mentioned previously, name collisions are entirely possible. Keys in the three namespaces can be referred to unambiguously like so: 41 | 42 | ```python 43 | entity.__dict__['key'] 44 | 45 | entity.properties.key 46 | entity.properties['key'] 47 | 48 | entity.annotations.key 49 | entity.annotations['key'] 50 | ``` 51 | 52 | Most of the time, users should be able to ignore these distinctions and treat Entities like normal Python objects. End users should never need to manipulate items in __dict__. 53 | 54 | See also: 55 | 56 | - [synapseclient.annotations][] 57 | -------------------------------------------------------------------------------- /docs/guides/README: -------------------------------------------------------------------------------- 1 | Guides: 2 | The purpose of the documents in this folder is to focus on the technincal detail for a 3 | specific real problem that one might use the client to solve. It is meant to frame the 4 | picture of the capabilities that are possible with the client. 5 | 6 | Guideline: 7 | - Solve a problem 8 | - Don't explain concepts: Link to them 9 | - Adapt to a real-world use-case if possible 10 | - Be specific with naming and titles 11 | 12 | 13 | 14 | This guide shows you how to… 15 | Describe clearly the problem or task that the guide shows the user how to solve. 16 | 17 | If you want x, do y. To achieve w, do z. 18 | Use conditional imperatives. 19 | 20 | Refer to the x reference guide for a full list of options. 21 | Don’t pollute your practical how-to guide with every possible thing the user might do related to x. 22 | -------------------------------------------------------------------------------- /docs/guides/accessing_the_rest_api.md: -------------------------------------------------------------------------------- 1 | # Accessing the Rest API 2 | 3 | These methods enable access to the Synapse REST(ish) API taking care of details like endpoints and authentication. 4 | See the [REST API documentation](https://rest-docs.synapse.org/rest/). 5 | 6 | See: 7 | 8 | - [synapseclient.Synapse.restGET][] 9 | - [synapseclient.Synapse.restPOST][] 10 | - [synapseclient.Synapse.restPUT][] 11 | - [synapseclient.Synapse.restDELETE][] 12 | -------------------------------------------------------------------------------- /docs/guides/home.md: -------------------------------------------------------------------------------- 1 | # How-To Guides 2 | Welcome! 3 | 4 | In this module you'll find extensive examples of all the things you can do with the 5 | Synapse Python/Command line client. 6 | -------------------------------------------------------------------------------- /docs/guides/validate_annotations.md: -------------------------------------------------------------------------------- 1 | # Validate Annotations 2 | 3 | > **Warning:** This is a beta implementation and is subject to change. Use at your own risk. 4 | 5 | Validate annotations on your Synapse entities by leveraging the JSON schema services. 6 | Here are the steps you must take to set up the JSON Schema service. 7 | 8 | ## Create a JSON Schema organization 9 | 10 | Set up Synapse client and JSON Schema service: 11 | 12 | ```python 13 | import synapseclient 14 | syn = synapseclient.login() 15 | syn.get_available_services() # Output: ['json_schema'] 16 | js = syn.service("json_schema") 17 | ``` 18 | 19 | Create, manage, and delete a JSON Schema organization: 20 | 21 | ```python 22 | my_org_name = 23 | my_org = js.JsonSchemaOrganization(my_org_name) 24 | my_org # Output: JsonSchemaOrganization(name=my_org_name) 25 | my_org.create() 26 | my_org.get_acl() 27 | my_org.set_acl([syn.getUserProfile().ownerId]) 28 | # my_org.update_acl([syn.getUserProfile().ownerId]) 29 | my_org.delete() 30 | ``` 31 | 32 | Retrieve existing organization and associated JSON schemas: 33 | 34 | ```python 35 | orgs = js.list_organizations() 36 | sage_org = js.JsonSchemaOrganization("sage.annotations") 37 | schemas = sage_org.list_json_schemas() 38 | schema1 = next(schemas) 39 | schema2 = sage_org.get_json_schema(schema1.name) 40 | assert schema1 is schema2 # True 41 | schema1 # Output: JsonSchema(org='sage.annotations', name='analysis.alignmentMethod') 42 | ``` 43 | 44 | Manage a specific version of a JSON schema: 45 | 46 | ```python 47 | versions = schema1.list_versions() 48 | version1 = next(versions) 49 | raw_body = version1.body 50 | full_body = version1.expand() 51 | version1 52 | # Output: JsonSchemaVersion(org='sage.annotations', name='analysis.alignmentMethod', version='0.0.1') 53 | ``` 54 | 55 | Create a new JSON schema version for an existing organization: 56 | 57 | ```python 58 | from random import randint 59 | rint = randint(0, 100000) 60 | schema_name = "my.schema" 61 | 62 | # Method 1 63 | my_org = js.JsonSchemaOrganization(my_org_name) 64 | new_version1 = my_org.create_json_schema(raw_body, schema_name, f"0.{rint}.1") 65 | 66 | # Method 2 67 | my_schema = js.JsonSchema(my_org, schema_name) 68 | new_version2 = my_schema.create(raw_body, f"0.{rint}.2") 69 | 70 | # Method 3 71 | my_version = js.JsonSchemaVersion(my_org, schema_name, f"0.{rint}.3") 72 | new_version3 = my_version.create(raw_body) 73 | ``` 74 | 75 | Test validation on a Synapse entity: 76 | 77 | ```python 78 | from time import sleep 79 | synapse_id = "syn25922647" 80 | js.bind_json_schema(new_version1.uri, synapse_id) 81 | js.get_json_schema(synapse_id) 82 | sleep(3) 83 | js.validate(synapse_id) 84 | js.validate_children(synapse_id) 85 | js.validation_stats(synapse_id) 86 | js.unbind_json_schema(synapse_id) 87 | ``` 88 | 89 | Access to low-level API functions: 90 | 91 | ```python 92 | js.create_organization(organization_name) 93 | js.get_organization(organization_name) 94 | js.list_organizations() 95 | js.delete_organization(organization_id) 96 | js.get_organization_acl(organization_id) 97 | js.update_organization_acl(organization_id, resource_access, etag) 98 | js.list_json_schemas(organization_name) 99 | js.list_json_schema_versions(organization_name, json_schema_name) 100 | js.create_json_schema(json_schema_body, dry_run) 101 | js.get_json_schema_body(json_schema_uri) 102 | js.delete_json_schema(json_schema_uri) 103 | js.json_schema_validation(json_schema_uri) 104 | js.bind_json_schema_to_entity(synapse_id, json_schema_uri) 105 | js.get_json_schema_from_entity(synapse_id) 106 | js.delete_json_schema_from_entity(synapse_id) 107 | js.validate_entity_with_json_schema(synapse_id) 108 | js.get_json_schema_validation_statistics(synapse_id) 109 | js.get_invalid_json_schema_validation(synapse_id) 110 | ``` 111 | -------------------------------------------------------------------------------- /docs/guides/views.md: -------------------------------------------------------------------------------- 1 | # Views 2 | 3 | A view is a view of all entities (File, Folder, Project, Table, Docker Repository, View) within one or more Projects or Folders. Views can: 4 | 5 | * Provide a way of isolating or linking data based on similarities 6 | * Provide the ability to link entities together by their annotations 7 | * Allow view/editing entities attributes in bulk 8 | * Allow entities to be easily searched and queried 9 | 10 | Let's go over some examples to demonstrate how view works. First, create a new project and add some files: 11 | 12 | ```python 13 | import synapseclient 14 | from synapseclient import Project, File, Column, Table, EntityViewSchema, EntityViewType 15 | syn = synapseclient.Synapse() 16 | syn.login() 17 | 18 | # Create a new project 19 | project = syn.store(Project("test view")) 20 | 21 | # Create some files 22 | file1 = syn.store(File(path="path/to/file1.txt", parent=project)) 23 | file2 = syn.store(File(path="path/to/file2.txt", parent=project)) 24 | 25 | # add some annotations 26 | syn.setAnnotations(file1, {"contributor":"Sage", "class":"V"}) 27 | syn.setAnnotations(file2, {"contributor":"UW", "rank":"X"}) 28 | ``` 29 | 30 | ## Creating a View 31 | 32 | To create a view, defines its name, columns, parent, scope, and the type of the view: 33 | 34 | ```python 35 | view = EntityViewSchema(name="my first file view", 36 | columns=[ 37 | Column(name="contributor", columnType="STRING"), 38 | Column(name="class", columnType="STRING"), 39 | Column(name="rank", columnType="STRING")), 40 | parent=project['id'], 41 | scopes=project['id'], 42 | includeEntityTypes=[EntityViewType.FILE, EntityViewType.FOLDER], 43 | addDefaultViewColumns=True) 44 | view = syn.store(view) 45 | ``` 46 | 47 | We support the following entity type in a View: 48 | 49 | * EntityViewType.FILE 50 | * EntityViewType.PROJECT 51 | * EntityViewType.TABLE 52 | * EntityViewType.FOLDER 53 | * EntityViewType.VIEW 54 | * EntityViewType.DOCKER 55 | 56 | To see the content of your newly created View, use [syn.tableQuery()][synapseclient.Synapse.tableQuery]: 57 | 58 | ```python 59 | query_results = syn.tableQuery("select * from %s" % view['id']) 60 | data = query_results.asDataFrame() 61 | ``` 62 | 63 | ## Updating Annotations using View 64 | 65 | To update `class` annotation for `file2`, simply update the view: 66 | 67 | ```python 68 | # Retrieve the view data using table query 69 | query_results = syn.tableQuery("select * from %s" % view['id']) 70 | data = query_results.asDataFrame() 71 | 72 | # Modify the annotations by modifying the view data and store it 73 | data["class"] = ["V", "VI"] 74 | syn.store(Table(view['id'], data)) 75 | ``` 76 | 77 | The change in annotations reflect in synGetAnnotations(): 78 | 79 | ```python 80 | syn.getAnnotations(file2['id']) 81 | ``` 82 | 83 | A View is a Table. Please visit [Tables][synapseclient.table] to see how to change schema, update content, and other operations that can be done on View. 84 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Synapse Python/Command Line Client Documentation 2 | 3 | [![Supported Python Versions](https://img.shields.io/pypi/pyversions/synapseclient.svg)](https://pypi.org/project/synapseclient/) 4 | 5 | ### **Notice for the upcoming v5.0 release:** 6 | 7 | - The upcoming v5.0 release will include a number of breaking changes. Take a look at 8 | this [pubpub](https://sagebionetworks.pubpub.org/pub/828a3x4k/release/1) article 9 | detailing some of the changes. 10 | - A release date has not been set. A number of these changes will be available within 11 | the 4.x.x versions hidden behind optional feature flags or different import paths. Any 12 | breaking changes will not be included until v5.0. 13 | 14 | The `synapseclient` package provides an interface to [Synapse](http://www.synapse.org), a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate, providing support for: 15 | 16 | - integrated presentation of data, code and text 17 | - fine grained access control 18 | - provenance tracking 19 | 20 | The `synapseclient` package lets you communicate with the cloud-hosted Synapse service to access data and create shared data analysis projects from within Python scripts or at the interactive Python console. Other Synapse clients exist for [R](https://r-docs.synapse.org/), [Java](https://github.com/Sage-Bionetworks/Synapse-Repository-Services/tree/develop/client/synapseJavaClient), and the [web](https://www.synapse.org/). The Python client can also be used from the [command line](tutorials/command_line_client.md). 21 | 22 | Installing this package will install `synapseclient`, `synapseutils` and the command line client. `synapseutils` contains beta features and the behavior of these features are subject to change. 23 | 24 | ## What’s on this docs site for you? 25 | * [Installation](./tutorials/installation.md), [Authentication](./tutorials/authentication.md), and [Configuration](./tutorials/configuration.md) of the `synapseclient` 26 | * [Tutorials](./tutorials/home.md) to get you and your team sharing, organizing, and discussing your scientific research 27 | * [How-To Guides](./guides/home.md) showcasing the full power and functionality available to you 28 | * [API Reference](./reference/client.md) of the programatic interfaces 29 | * [Further Reading](./explanations/home.md) to gain a deeper understanding of best practices and advanced use cases 30 | * Our [release notes](./news.md) 31 | 32 | ## Additional Background 33 | 34 | * Read [about Synapse](https://help.synapse.org/docs/About-Synapse.2058846607.html)—how it got started and how it fits into the bigger data-sharing picture 35 | * Gain a better understanding of [Sage Bionetworks](https://help.synapse.org/docs/About-Synapse.2058846607.html#AboutSynapse-WhomanagesSynapse?) (that’s us—the nonprofit organization that created Synapse) and our other platforms that coincide with Synapse (such as portals) 36 | * Learn about [Synapse governance](https://help.synapse.org/docs/Synapse-Governance.2004255211.html) and how it protects data privacy 37 | * Look up an unfamiliar term or acronym in our [glossary](https://help.synapse.org/docs/Glossary.2667938103.html) 38 | * See our [help section](https://help.synapse.org/docs/Help.2650865669.html) for further assistance via the FAQ page, discussion forum, or contact information to get in touch 39 | -------------------------------------------------------------------------------- /docs/reference/README: -------------------------------------------------------------------------------- 1 | Reference: 2 | The purpose of the documents in this folder is to focus on the interfaces input and output 3 | when working with the client. Documents here are meant to be consulted with to solve 4 | a problem. 5 | 6 | Guideline: 7 | - Be consistent 8 | - Provide brief to-the-point examples 9 | - Describe what something does and link to tutorials/how-to guides/explanations if needed 10 | - Be complete, if you give an example provide the preconditions 11 | 12 | 13 | 14 | X is an example of y. W needs to be initialised using z. This option does that. 15 | State facts about the machinery and its behaviour. 16 | 17 | Sub-commands are: a, b, c, d, e, f. 18 | List commands, options, operations, features, flags, limitations, error messages, etc. 19 | 20 | You must use a. You must not apply b unless c. Never d. 21 | Provide warnings where appropriate. 22 | -------------------------------------------------------------------------------- /docs/reference/activity.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.activity 2 | -------------------------------------------------------------------------------- /docs/reference/annotations.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.annotations 2 | -------------------------------------------------------------------------------- /docs/reference/cli.md: -------------------------------------------------------------------------------- 1 | # Command Line Client 2 | 3 | The Synapse Python Client can be used from the command line via the **synapse** command. 4 | 5 | > **Note:** The command line client is installed along with [installation](../tutorials/installation.md) of the Synapse Python client. 6 | 7 | ## Usage 8 | 9 | For help, type: 10 | 11 | ```bash 12 | synapse -h 13 | ``` 14 | -------------------------------------------------------------------------------- /docs/reference/client.md: -------------------------------------------------------------------------------- 1 | # Client 2 | 3 | 5 | ::: synapseclient.Synapse 6 | options: 7 | members: 8 | - login 9 | - logout 10 | - get 11 | - store 12 | - move 13 | - delete 14 | - get_annotations 15 | - set_annotations 16 | - tableQuery 17 | - createColumns 18 | - getColumn 19 | - getColumns 20 | - getTableColumns 21 | - downloadTableColumns 22 | - get_acl 23 | - get_permissions 24 | - getPermissions 25 | - setPermissions 26 | - getProvenance 27 | - setProvenance 28 | - deleteProvenance 29 | - updateActivity 30 | - findEntityId 31 | - getChildren 32 | - getTeam 33 | - getTeamMembers 34 | - invite_to_team 35 | - get_membership_status 36 | - get_team_open_invitations 37 | - send_membership_invitation 38 | - submit 39 | - getConfigFile 40 | - setEndpoints 41 | - invalidateAPIKey 42 | - get_user_profile_by_username 43 | - get_user_profile_by_id 44 | - getUserProfile 45 | - is_certified 46 | - is_synapse_id 47 | - onweb 48 | - printEntity 49 | - get_available_services 50 | - service 51 | - clear_download_list 52 | - create_external_s3_file_handle 53 | - getMyStorageLocationSetting 54 | - createStorageLocationSetting 55 | - create_s3_storage_location 56 | - setStorageLocation 57 | - get_sts_storage_token 58 | - create_snapshot_version 59 | - getConfigFile 60 | - getEvaluation 61 | - getEvaluationByContentSource 62 | - getEvaluationByName 63 | - create_team 64 | - delete_team 65 | - getProjectSetting 66 | - getSubmission 67 | - getSubmissions 68 | - getSubmissionBundles 69 | - getSubmissionStatus 70 | - getWiki 71 | - getWikiAttachments 72 | - getWikiHeaders 73 | - get_download_list 74 | - get_download_list_manifest 75 | - remove_from_download_list 76 | - md5Query 77 | - sendMessage 78 | - uploadFileHandle 79 | - restGET 80 | - restPOST 81 | - restPUT 82 | - restDELETE 83 | - rest_get_async 84 | - rest_post_async 85 | - rest_put_async 86 | - rest_delete_async 87 | 88 | ## More information 89 | 90 | See also the [Synapse API documentation](https://rest-docs.synapse.org/) 91 | -------------------------------------------------------------------------------- /docs/reference/core.md: -------------------------------------------------------------------------------- 1 | # Core 2 | 3 | This section is for super users / developers only. 4 | These functions are subject to change as they are internal development 5 | functions. Use at your own risk. 6 | 7 | ## Upload 8 | ::: synapseclient.core.upload.upload_functions 9 | 10 | ## Multipart Upload 11 | 12 | ::: synapseclient.core.upload.multipart_upload 13 | options: 14 | members: 15 | - UploadAttempt 16 | - executor 17 | - multipart_upload_file 18 | - multipart_upload_string 19 | - multipart_copy 20 | - _multipart_upload 21 | 22 | 23 | ## Upload Async 24 | ::: synapseclient.core.upload.upload_functions_async 25 | 26 | ## Multipart Upload Async 27 | 28 | ::: synapseclient.core.upload.multipart_upload_async 29 | 30 | ## Multithreaded Downloading 31 | ::: synapseclient.core.multithread_download 32 | 33 | ## Download Functions 34 | ::: synapseclient.core.download.download_functions 35 | 36 | ## Async managed Multithreaded Downloads 37 | ::: synapseclient.core.download.download_async 38 | 39 | ## Cache 40 | ::: synapseclient.core.cache 41 | 42 | ## Credentials 43 | ::: synapseclient.core.credentials.cred_data.SynapseCredentials 44 | ::: synapseclient.core.credentials.cred_data.SynapseAuthTokenCredentials 45 | ::: synapseclient.core.credentials.credential_provider 46 | 47 | ## Remote File Storage Wrappers 48 | ::: synapseclient.core.remote_file_storage_wrappers 49 | 50 | ## Retry 51 | 52 | ::: synapseclient.core.retry 53 | 54 | ## Utils 55 | 56 | ::: synapseclient.core.utils 57 | 58 | ## Async Utils 59 | 60 | ::: synapseclient.core.async_utils 61 | 62 | ## Versions 63 | ::: synapseclient.core.version_check 64 | 65 | ## STS Transfer 66 | ::: synapseclient.core.sts_transfer 67 | -------------------------------------------------------------------------------- /docs/reference/docker_repository.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.entity.DockerRepository 2 | -------------------------------------------------------------------------------- /docs/reference/entity.md: -------------------------------------------------------------------------------- 1 | #Entity 2 | The Entity class is the base class for all entities, including Project, Folder, File, and Link. 3 | 4 | Entities are dictionary-like objects in which both object and dictionary notation (`entity.foo` or `entity['foo']`) can be 5 | used interchangeably. 6 | 7 | ::: synapseclient.entity.Entity 8 | ::: synapseclient.entity.Versionable 9 | -------------------------------------------------------------------------------- /docs/reference/evaluation.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.evaluation 2 | -------------------------------------------------------------------------------- /docs/reference/exceptions.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.core.exceptions 2 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/activity.md: -------------------------------------------------------------------------------- 1 | # Activity 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API Reference 8 | 9 | [](){ #activity-reference-async } 10 | ::: synapseclient.models.Activity 11 | options: 12 | members: 13 | - from_parent_async 14 | - store_async 15 | - delete_async 16 | - disassociate_from_entity_async 17 | --- 18 | [](){ #used-entity-reference-async } 19 | ::: synapseclient.models.UsedEntity 20 | options: 21 | filters: 22 | - "!" 23 | --- 24 | [](){ #used-url-reference-async } 25 | ::: synapseclient.models.UsedURL 26 | options: 27 | filters: 28 | - "!" 29 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/agent.md: -------------------------------------------------------------------------------- 1 | # Agent 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API reference 8 | 9 | [](){ #agent-reference-async } 10 | ::: synapseclient.models.Agent 11 | options: 12 | members: 13 | - register_async 14 | - get_async 15 | - start_session_async 16 | - get_session_async 17 | - prompt_async 18 | - get_chat_history 19 | --- 20 | [](){ #agent-session-reference-async } 21 | ::: synapseclient.models.AgentSession 22 | options: 23 | members: 24 | - start_async 25 | - get_async 26 | - update_async 27 | - prompt_async 28 | --- 29 | [](){ #agent-prompt-reference-async } 30 | ::: synapseclient.models.AgentPrompt 31 | options: 32 | inherited_members: true 33 | members: 34 | - send_job_and_wait_async 35 | --- 36 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/dataset.md: -------------------------------------------------------------------------------- 1 | # Dataset 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API reference 8 | 9 | [](){ #dataset-reference-async } 10 | ::: synapseclient.models.Dataset 11 | options: 12 | inherited_members: true 13 | members: 14 | - add_item_async 15 | - remove_item_async 16 | - store_async 17 | - get_async 18 | - delete_async 19 | - update_rows_async 20 | - snapshot_async 21 | - query_async 22 | - query_part_mask_async 23 | - add_column 24 | - delete_column 25 | - reorder_column 26 | - rename_column 27 | - get_permissions_async 28 | - get_acl_async 29 | - set_permissions_async 30 | - delete_permissions_async 31 | --- 32 | [](){ #entity-ref-reference-async } 33 | ::: synapseclient.models.EntityRef 34 | --- 35 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/dataset_collection.md: -------------------------------------------------------------------------------- 1 | # Dataset Collection 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API reference 8 | 9 | [](){ #dataset-collection-reference-async } 10 | ::: synapseclient.models.DatasetCollection 11 | options: 12 | inherited_members: true 13 | members: 14 | - add_item_async 15 | - remove_item_async 16 | - store_async 17 | - get_async 18 | - delete_async 19 | - update_rows_async 20 | - snapshot_async 21 | - query_async 22 | - query_part_mask_async 23 | - add_column 24 | - delete_column 25 | - reorder_column 26 | - rename_column 27 | - get_permissions_async 28 | - get_acl_async 29 | - set_permissions_async 30 | - delete_permissions_async 31 | --- 32 | [](){ #entity-ref-dataset-collection-reference-async } 33 | ::: synapseclient.models.EntityRef 34 | --- 35 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/entityview.md: -------------------------------------------------------------------------------- 1 | # EntityView 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API reference 8 | 9 | [](){ #entityview-reference-async } 10 | ::: synapseclient.models.EntityView 11 | options: 12 | inherited_members: true 13 | members: 14 | - store_async 15 | - get_async 16 | - delete_async 17 | - update_rows_async 18 | - query_async 19 | - query_part_mask_async 20 | - snapshot_async 21 | - add_column 22 | - reorder_column 23 | - delete_column 24 | - get_acl_async 25 | - get_permissions_async 26 | - set_permissions_async 27 | - delete_permissions_async 28 | --- 29 | 30 | [](){ #view-type-mask-reference } 31 | ::: synapseclient.api.ViewTypeMask 32 | --- 33 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/file.md: -------------------------------------------------------------------------------- 1 | # File 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API Reference 8 | 9 | [](){ #file-reference-async } 10 | ::: synapseclient.models.File 11 | options: 12 | inherited_members: true 13 | members: 14 | - get_async 15 | - store_async 16 | - copy_async 17 | - delete_async 18 | - from_id_async 19 | - from_path_async 20 | - change_metadata_async 21 | - get_permissions_async 22 | - get_acl_async 23 | - set_permissions_async 24 | - delete_permissions_async 25 | --- 26 | [](){ #filehandle-reference-async } 27 | ::: synapseclient.models.file.FileHandle 28 | options: 29 | filters: 30 | - "!" 31 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/folder.md: -------------------------------------------------------------------------------- 1 | # Folder 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API Reference 8 | 9 | ::: synapseclient.models.Folder 10 | options: 11 | inherited_members: true 12 | members: 13 | - get_async 14 | - store_async 15 | - delete_async 16 | - copy_async 17 | - sync_from_synapse_async 18 | - get_permissions_async 19 | - get_acl_async 20 | - set_permissions_async 21 | - delete_permissions_async 22 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/materializedview.md: -------------------------------------------------------------------------------- 1 | # MaterializedView 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API reference 8 | 9 | ::: synapseclient.models.MaterializedView 10 | options: 11 | inherited_members: true 12 | members: 13 | - store_async 14 | - get_async 15 | - delete_async 16 | - query_async 17 | - query_part_mask_async 18 | - get_permissions_async 19 | - get_acl_async 20 | - set_permissions_async 21 | - delete_permissions_async 22 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/project.md: -------------------------------------------------------------------------------- 1 | # Project 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API reference 8 | 9 | ::: synapseclient.models.Project 10 | options: 11 | inherited_members: true 12 | members: 13 | - get_async 14 | - store_async 15 | - delete_async 16 | - sync_from_synapse_async 17 | - get_permissions_async 18 | - get_acl_async 19 | - set_permissions_async 20 | - delete_permissions_async 21 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/submissionview.md: -------------------------------------------------------------------------------- 1 | # SubmissionView 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API reference 8 | 9 | ::: synapseclient.models.SubmissionView 10 | options: 11 | inherited_members: true 12 | members: 13 | - store_async 14 | - get_async 15 | - delete_async 16 | - query_async 17 | - query_part_mask_async 18 | - snapshot_async 19 | - add_column 20 | - reorder_column 21 | - delete_column 22 | - get_acl_async 23 | - get_permissions_async 24 | - set_permissions_async 25 | --- 26 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/table.md: -------------------------------------------------------------------------------- 1 | # Table 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API Reference 8 | 9 | [](){ #table-reference-async } 10 | ::: synapseclient.models.Table 11 | options: 12 | inherited_members: true 13 | members: 14 | - get_async 15 | - store_async 16 | - delete_async 17 | - query_async 18 | - query_part_mask_async 19 | - store_rows_async 20 | - upsert_rows_async 21 | - delete_rows_async 22 | - snapshot_async 23 | - delete_column 24 | - add_column 25 | - reorder_column 26 | - get_permissions_async 27 | - get_acl_async 28 | - set_permissions_async 29 | - delete_permissions_async 30 | 31 | [](){ #column-reference-async } 32 | ::: synapseclient.models.Column 33 | options: 34 | members: 35 | 36 | [](){ #schema-storage-strategy-reference-async } 37 | ::: synapseclient.models.SchemaStorageStrategy 38 | [](){ #column-expansion-strategy-reference-async } 39 | ::: synapseclient.models.ColumnExpansionStrategy 40 | 41 | [](){ #facet-type-reference-async } 42 | ::: synapseclient.models.FacetType 43 | [](){ #column-type-reference-async } 44 | ::: synapseclient.models.ColumnType 45 | [](){ #json-sub-column-reference-async } 46 | ::: synapseclient.models.JsonSubColumn 47 | 48 | 49 | [](){ #column-change-reference-async } 50 | ::: synapseclient.models.ColumnChange 51 | [](){ #partial-row-reference-async } 52 | ::: synapseclient.models.PartialRow 53 | [](){ #partial-row-set-reference-async } 54 | ::: synapseclient.models.PartialRowSet 55 | [](){ #table-schema-change-request-reference-async } 56 | ::: synapseclient.models.TableSchemaChangeRequest 57 | [](){ #appendable-row-set-request-reference-async } 58 | ::: synapseclient.models.AppendableRowSetRequest 59 | [](){ #upload-to-table-request-reference-async } 60 | ::: synapseclient.models.UploadToTableRequest 61 | [](){ #table-update-transaction-reference-async } 62 | ::: synapseclient.models.TableUpdateTransaction 63 | [](){ #csv-table-descriptor-reference-async } 64 | ::: synapseclient.models.CsvTableDescriptor 65 | [](){ #csv-to-pandas-df-reference-async } 66 | ::: synapseclient.models.mixins.table_components.csv_to_pandas_df 67 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/team.md: -------------------------------------------------------------------------------- 1 | # Team 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API Reference 8 | 9 | [](){ #team-reference-async } 10 | ::: synapseclient.models.Team 11 | options: 12 | members: 13 | - create_async 14 | - delete_async 15 | - from_id_async 16 | - from_name_async 17 | - members_async 18 | - invite_async 19 | - open_invitations_async 20 | --- 21 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/user_profile.md: -------------------------------------------------------------------------------- 1 | # UserProfile 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API Reference 8 | 9 | [](){ #user-profile-reference-async } 10 | ::: synapseclient.models.UserProfile 11 | options: 12 | inherited_members: true 13 | members: 14 | - get_async 15 | - from_id_async 16 | - from_username_async 17 | - is_certified_async 18 | --- 19 | [](){ #user-preference-reference-async } 20 | ::: synapseclient.models.UserPreference 21 | --- 22 | -------------------------------------------------------------------------------- /docs/reference/experimental/async/virtualtable.md: -------------------------------------------------------------------------------- 1 | # VirtualTable 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API reference 8 | 9 | ::: synapseclient.models.VirtualTable 10 | options: 11 | inherited_members: true 12 | members: 13 | - store_async 14 | - get_async 15 | - delete_async 16 | - query_async 17 | - query_part_mask_async 18 | - get_permissions_async 19 | - get_acl_async 20 | - set_permissions_async 21 | - delete_permissions_async 22 | -------------------------------------------------------------------------------- /docs/reference/experimental/functional_interfaces.md: -------------------------------------------------------------------------------- 1 | 2 | ::: synapseclient.api 3 | options: 4 | inherited_members: true 5 | members: 6 | - delete_entity 7 | -------------------------------------------------------------------------------- /docs/reference/experimental/mixins/access_controllable.md: -------------------------------------------------------------------------------- 1 | # AccessControllable 2 | 3 | ::: synapseclient.models.mixins.AccessControllable 4 | -------------------------------------------------------------------------------- /docs/reference/experimental/mixins/asynchronous_communicator.md: -------------------------------------------------------------------------------- 1 | # AsynchronousCommunicator 2 | 3 | ::: synapseclient.models.mixins.AsynchronousCommunicator 4 | -------------------------------------------------------------------------------- /docs/reference/experimental/mixins/failure_strategy.md: -------------------------------------------------------------------------------- 1 | # FailureStrategy 2 | 3 | ::: synapseclient.models.FailureStrategy 4 | -------------------------------------------------------------------------------- /docs/reference/experimental/mixins/storable_container.md: -------------------------------------------------------------------------------- 1 | # StorableContainer 2 | 3 | ::: synapseclient.models.mixins.StorableContainer 4 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/activity.md: -------------------------------------------------------------------------------- 1 | [](){ #activity-reference-sync } 2 | # Activity 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## Example Script 9 | 10 |
11 | Working with activities 12 | 13 | ```python 14 | {!docs/scripts/object_orientated_programming_poc/oop_poc_activity.py!} 15 | ``` 16 |
17 | 18 | ## API Reference 19 | 20 | ::: synapseclient.models.Activity 21 | options: 22 | inherited_members: true 23 | members: 24 | - from_parent 25 | - store 26 | - delete 27 | --- 28 | [](){ #used-entity-reference-sync } 29 | ::: synapseclient.models.UsedEntity 30 | options: 31 | filters: 32 | - "!" 33 | --- 34 | [](){ #used-url-reference-sync } 35 | ::: synapseclient.models.UsedURL 36 | options: 37 | filters: 38 | - "!" 39 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/agent.md: -------------------------------------------------------------------------------- 1 | [](){ #agent-reference-sync } 2 | # Agent 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## Example Script: 9 | 10 |
11 | Working with Synapse agents 12 | 13 | ```python 14 | {!docs/scripts/object_orientated_programming_poc/oop_poc_agent.py!} 15 | ``` 16 |
17 | 18 | ## API Reference 19 | 20 | ::: synapseclient.models.Agent 21 | options: 22 | inherited_members: true 23 | members: 24 | - register 25 | - get 26 | - start_session 27 | - get_session 28 | - prompt 29 | - get_chat_history 30 | --- 31 | [](){ #agent-session-reference-sync } 32 | ::: synapseclient.models.AgentSession 33 | options: 34 | inherited_members: true 35 | members: 36 | - start 37 | - get 38 | - update 39 | - prompt 40 | --- 41 | [](){ #agent-prompt-reference-sync } 42 | ::: synapseclient.models.AgentPrompt 43 | options: 44 | inherited_members: true 45 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/dataset.md: -------------------------------------------------------------------------------- 1 | [](){ #dataset-reference-sync } 2 | # Dataset 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## API reference 9 | 10 | ::: synapseclient.models.Dataset 11 | options: 12 | inherited_members: true 13 | members: 14 | - add_item 15 | - remove_item 16 | - store 17 | - get 18 | - delete 19 | - update_rows 20 | - snapshot 21 | - query 22 | - query_part_mask 23 | - add_column 24 | - delete_column 25 | - reorder_column 26 | - rename_column 27 | - get_permissions 28 | - get_acl 29 | - set_permissions 30 | - delete_permissions 31 | --- 32 | [](){ #entity-ref-reference-sync } 33 | ::: synapseclient.models.EntityRef 34 | --- 35 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/dataset_collection.md: -------------------------------------------------------------------------------- 1 | [](){ #dataset-collection-reference-sync } 2 | # Dataset Collection 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## API reference 9 | 10 | ::: synapseclient.models.DatasetCollection 11 | options: 12 | inherited_members: true 13 | members: 14 | - add_item 15 | - remove_item 16 | - store 17 | - get 18 | - delete 19 | - update_rows 20 | - snapshot 21 | - query 22 | - query_part_mask 23 | - add_column 24 | - delete_column 25 | - reorder_column 26 | - rename_column 27 | - get_permissions 28 | - get_acl 29 | - set_permissions 30 | - delete_permissions 31 | --- 32 | [](){ #entity-ref-dataset-collection-reference-sync } 33 | ::: synapseclient.models.EntityRef 34 | --- 35 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/entityview.md: -------------------------------------------------------------------------------- 1 | [](){ #entityview-reference-sync } 2 | # EntityView 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## API reference 9 | 10 | ::: synapseclient.models.EntityView 11 | options: 12 | inherited_members: true 13 | members: 14 | - store 15 | - get 16 | - delete 17 | - update_rows 18 | - query 19 | - query_part_mask 20 | - snapshot 21 | - add_column 22 | - reorder_column 23 | - delete_column 24 | - get_acl 25 | - get_permissions 26 | - set_permissions 27 | - delete_permissions 28 | --- 29 | 30 | [](){ #view-type-mask-reference-sync } 31 | ::: synapseclient.api.ViewTypeMask 32 | --- 33 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/file.md: -------------------------------------------------------------------------------- 1 | [](){ #file-reference-sync } 2 | # File 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## Example Script 9 | 10 |
11 | Working with files 12 | 13 | ```python 14 | {!docs/scripts/object_orientated_programming_poc/oop_poc_file.py!} 15 | ``` 16 |
17 | 18 | ## API Reference 19 | 20 | ::: synapseclient.models.File 21 | options: 22 | inherited_members: true 23 | members: 24 | - get 25 | - store 26 | - copy 27 | - delete 28 | - from_id 29 | - from_path 30 | - change_metadata 31 | - get_permissions 32 | - get_acl 33 | - set_permissions 34 | - delete_permissions 35 | --- 36 | [](){ #filehandle-reference-sync } 37 | ::: synapseclient.models.file.FileHandle 38 | options: 39 | filters: 40 | - "!" 41 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/folder.md: -------------------------------------------------------------------------------- 1 | [](){ #folder-reference-sync } 2 | # Folder 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## Example Script 9 | 10 |
11 | Working with folders 12 | 13 | ```python 14 | {!docs/scripts/object_orientated_programming_poc/oop_poc_folder.py!} 15 | ``` 16 |
17 | 18 | ## API Reference 19 | 20 | ::: synapseclient.models.Folder 21 | options: 22 | inherited_members: true 23 | members: 24 | - get 25 | - store 26 | - delete 27 | - copy 28 | - sync_from_synapse 29 | - get_permissions 30 | - get_acl 31 | - set_permissions 32 | - delete_permissions 33 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/materializedview.md: -------------------------------------------------------------------------------- 1 | [](){ #materializedview-reference-sync } 2 | # MaterializedView 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## API reference 9 | 10 | ::: synapseclient.models.MaterializedView 11 | options: 12 | inherited_members: true 13 | members: 14 | - store 15 | - get 16 | - delete 17 | - query 18 | - query_part_mask 19 | - get_permissions 20 | - get_acl 21 | - set_permissions 22 | - delete_permissions 23 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/project.md: -------------------------------------------------------------------------------- 1 | [](){ #project-reference-sync } 2 | # Project 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## Example Script 9 | 10 |
11 | Working with a project 12 | 13 | ```python 14 | {!docs/scripts/object_orientated_programming_poc/oop_poc_project.py!} 15 | ``` 16 |
17 | 18 | ## API reference 19 | 20 | ::: synapseclient.models.Project 21 | options: 22 | inherited_members: true 23 | members: 24 | - get 25 | - store 26 | - delete 27 | - sync_from_synapse 28 | - get_permissions 29 | - get_acl 30 | - set_permissions 31 | - delete_permissions 32 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/submissionview.md: -------------------------------------------------------------------------------- 1 | [](){ #submissionview-reference-sync } 2 | # SubmissionView 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## API reference 9 | 10 | ::: synapseclient.models.SubmissionView 11 | options: 12 | inherited_members: true 13 | members: 14 | - store 15 | - get 16 | - delete 17 | - query 18 | - query_part_mask 19 | - snapshot 20 | - add_column 21 | - reorder_column 22 | - delete_column 23 | - get_acl 24 | - get_permissions 25 | - set_permissions 26 | - delete_permissions 27 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/table.md: -------------------------------------------------------------------------------- 1 | [](){ #table-reference-sync } 2 | # Table 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## Example Script 9 | 10 |
11 | Working with tables 12 | 13 | ```python 14 | {!docs/scripts/object_orientated_programming_poc/oop_poc_table.py!} 15 | ``` 16 |
17 | 18 | ## API Reference 19 | 20 | 21 | ::: synapseclient.models.Table 22 | options: 23 | inherited_members: true 24 | members: 25 | - get 26 | - store 27 | - delete 28 | - query 29 | - query_part_mask 30 | - store_rows 31 | - upsert_rows 32 | - delete_rows 33 | - snapshot 34 | - delete_column 35 | - add_column 36 | - reorder_column 37 | - get_permissions 38 | - get_acl 39 | - set_permissions 40 | - delete_permissions 41 | 42 | [](){ #column-reference-sync } 43 | ::: synapseclient.models.Column 44 | options: 45 | members: 46 | 47 | [](){ #schema-storage-strategy-reference-sync } 48 | ::: synapseclient.models.SchemaStorageStrategy 49 | [](){ #column-expansion-strategy-reference-sync } 50 | ::: synapseclient.models.ColumnExpansionStrategy 51 | 52 | [](){ #facet-type-reference-sync } 53 | ::: synapseclient.models.FacetType 54 | [](){ #column-type-reference-sync } 55 | ::: synapseclient.models.ColumnType 56 | [](){ #json-sub-column-reference-sync } 57 | ::: synapseclient.models.JsonSubColumn 58 | 59 | 60 | [](){ #column-change-reference-sync } 61 | ::: synapseclient.models.ColumnChange 62 | [](){ #partial-row-reference-sync } 63 | ::: synapseclient.models.PartialRow 64 | [](){ #partial-row-set-reference-sync } 65 | ::: synapseclient.models.PartialRowSet 66 | [](){ #table-schema-change-request-reference-sync } 67 | ::: synapseclient.models.TableSchemaChangeRequest 68 | [](){ #appendable-row-set-request-reference-sync } 69 | ::: synapseclient.models.AppendableRowSetRequest 70 | [](){ #upload-to-table-request-reference-sync } 71 | ::: synapseclient.models.UploadToTableRequest 72 | [](){ #table-update-transaction-reference-sync } 73 | ::: synapseclient.models.TableUpdateTransaction 74 | [](){ #csv-table-descriptor-reference-sync } 75 | ::: synapseclient.models.CsvTableDescriptor 76 | [](){ #csv-to-pandas-df-reference-sync } 77 | ::: synapseclient.models.mixins.table_components.csv_to_pandas_df 78 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/team.md: -------------------------------------------------------------------------------- 1 | [](){ #team-reference-sync } 2 | # Team 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## Example Script 9 | 10 |
11 | Working with teams 12 | 13 | ```python 14 | {!docs/scripts/object_orientated_programming_poc/oop_poc_team.py!} 15 | ``` 16 |
17 | 18 | ## API Reference 19 | 20 | ::: synapseclient.models.Team 21 | options: 22 | inherited_members: true 23 | members: 24 | - create 25 | - delete 26 | - from_id 27 | - from_name 28 | - members 29 | - invite 30 | - open_invitations 31 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/user_profile.md: -------------------------------------------------------------------------------- 1 | [](){ #user-profile-reference-sync } 2 | # UserProfile 3 | 4 | Contained within this file are experimental interfaces for working with the Synapse Python 5 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 6 | at your own risk. 7 | 8 | ## API Reference 9 | 10 | ::: synapseclient.models.UserProfile 11 | options: 12 | inherited_members: true 13 | members: 14 | - get 15 | - from_id 16 | - from_username 17 | - is_certified 18 | --- 19 | [](){ #user-preference-reference-sync } 20 | ::: synapseclient.models.UserPreference 21 | --- 22 | -------------------------------------------------------------------------------- /docs/reference/experimental/sync/virtualtable.md: -------------------------------------------------------------------------------- 1 | # VirtualTable 2 | 3 | Contained within this file are experimental interfaces for working with the Synapse Python 4 | Client. Unless otherwise noted these interfaces are subject to change at any time. Use 5 | at your own risk. 6 | 7 | ## API reference 8 | 9 | ::: synapseclient.models.VirtualTable 10 | options: 11 | inherited_members: true 12 | members: 13 | - store 14 | - get 15 | - delete 16 | - query 17 | - query_part_mask 18 | - get_permissions 19 | - get_acl 20 | - set_permissions 21 | - delete_permissions 22 | -------------------------------------------------------------------------------- /docs/reference/file.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.entity.File 2 | -------------------------------------------------------------------------------- /docs/reference/folder.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.entity.Folder 2 | -------------------------------------------------------------------------------- /docs/reference/json_schema.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.services.json_schema 2 | -------------------------------------------------------------------------------- /docs/reference/link.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.entity.Link 2 | -------------------------------------------------------------------------------- /docs/reference/permissions.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.Permissions 2 | options: 3 | members: 4 | - from_dict 5 | - access_types 6 | -------------------------------------------------------------------------------- /docs/reference/project.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.entity.Project 2 | -------------------------------------------------------------------------------- /docs/reference/rest_apis.md: -------------------------------------------------------------------------------- 1 | # Rest APIs 2 | 3 | This section is for super users / developers only. 4 | These functions are subject to change as they are internal development 5 | functions. Use at your own risk. 6 | 7 | 8 | The functions provided in these documents are generally direct representations of the 9 | Synapse REST API. They're used within the Synapse Python Client to interact with the 10 | Synapse servers. These will eventually be removed from the Synapse Python Client in 11 | favor of an auto-generated client derived from the 12 | [Synapse Open API Spec](https://rest-docs.synapse.org/rest/openapi/openapispecification.json). 13 | 14 | 15 | ::: synapseclient.api.entity_bundle_services_v2 16 | ::: synapseclient.api.annotations 17 | -------------------------------------------------------------------------------- /docs/reference/synapse_utils.md: -------------------------------------------------------------------------------- 1 | # Synapse Utils 2 | 3 | ::: synapseutils 4 | 5 | ::: synapseutils.sync 6 | options: 7 | members: 8 | - syncFromSynapse 9 | - syncToSynapse 10 | - generateManifest 11 | - generate_sync_manifest 12 | - readManifestFile 13 | 14 | ::: synapseutils.copy_functions 15 | options: 16 | members: 17 | - copy 18 | - changeFileMetaData 19 | - copyFileHandles 20 | - copyWiki 21 | 22 | ::: synapseutils.walk_functions 23 | options: 24 | members: 25 | - walk 26 | 27 | ::: synapseutils.monitor 28 | options: 29 | members: 30 | - notifyMe 31 | - with_progress_bar 32 | 33 | ::: synapseutils.migrate_functions 34 | options: 35 | members: 36 | - MigrationResult 37 | - index_files_for_migration 38 | - migrate_indexed_files 39 | 40 | ::: synapseutils.describe_functions 41 | options: 42 | members: 43 | - describe 44 | -------------------------------------------------------------------------------- /docs/reference/table_schema.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.table.Schema 2 | -------------------------------------------------------------------------------- /docs/reference/tables.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.table 2 | -------------------------------------------------------------------------------- /docs/reference/teams.md: -------------------------------------------------------------------------------- 1 | # Teams 2 | 3 | ::: synapseclient.team 4 | -------------------------------------------------------------------------------- /docs/reference/view_schema.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.table.EntityViewSchema 2 | -------------------------------------------------------------------------------- /docs/reference/wiki.md: -------------------------------------------------------------------------------- 1 | ::: synapseclient.wiki 2 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | markdown-include 2 | mkdocs 3 | mkdocs-material 4 | mkdocs-open-in-new-tab 5 | mkdocstrings 6 | mkdocstrings-python 7 | termynal 8 | -------------------------------------------------------------------------------- /docs/scripts/object_orientated_programming_poc/oop_poc_team.py: -------------------------------------------------------------------------------- 1 | """The purpose of this script is to demonstrate how to use the new OOP interface for teams. 2 | The following actions are shown in this script: 3 | 1. Creating a Team 4 | 2. Instantiating a Team object from Synapse 5 | 3. Getting information about the members of a Team 6 | 4. Inviting a user to a Team 7 | 5. Checking on invitations to join a Team 8 | 6. Deleting a Team 9 | """ 10 | 11 | import time 12 | 13 | import synapseclient 14 | from synapseclient.models.team import Team 15 | 16 | syn = synapseclient.Synapse(debug=True) 17 | syn.login() 18 | 19 | 20 | def new_team(): 21 | # Create a team 22 | new_team = Team( 23 | name="python-client-test-team", 24 | description="testing OOP interface", 25 | can_public_join=False, 26 | ) 27 | my_synapse_team = new_team.create() 28 | print(my_synapse_team) 29 | 30 | # Instantiate a Team object from a Synapse team 31 | my_team = Team.from_id(id=my_synapse_team.id) 32 | print(my_team) 33 | 34 | # Sleep because the API for retrieving a team by name is eventually consistent. from_id works right away, however. 35 | time.sleep(5) 36 | my_team = Team.from_name(name=my_synapse_team.name) 37 | print(my_team) 38 | 39 | # Refresh the team to get the latest information 40 | my_team.get() 41 | print(my_team) 42 | 43 | # Get information about the members of a Team 44 | members = my_team.members() 45 | print(members) 46 | 47 | # Invite a user to a Team 48 | invite = my_team.invite( 49 | user="test_account_synapse_client", 50 | message="testing OOP interface (do not accept)", 51 | ) 52 | print(invite) 53 | 54 | # Get open invitations for the Team 55 | invitations = my_team.open_invitations() 56 | print(invitations) 57 | 58 | # Delete the Team 59 | my_team.delete() 60 | 61 | 62 | new_team() 63 | -------------------------------------------------------------------------------- /docs/scripts/object_orientated_programming_poc/oop_poc_user_and_permission.py: -------------------------------------------------------------------------------- 1 | """The purpose of this script is to demonstrate how to use the OOP interface for 2 | UserProfile and permissions on some entities. 3 | The following actions are shown in this script: 4 | 1. Get a UserProfile by name and id 5 | 2. Check if a UserProfile is certified 6 | 3. Set permissions on a File 7 | 4. Get permissions on a File 8 | """ 9 | 10 | import os 11 | 12 | import synapseclient 13 | from synapseclient.models import File, UserProfile 14 | 15 | PROJECT_ID = "syn52948289" 16 | TEST_ACCOUNT_NAME = "bfauble_synapse_test_account" 17 | TEST_ACCOUNT_ID = 3489192 18 | 19 | syn = synapseclient.Synapse(debug=True) 20 | syn.login() 21 | 22 | 23 | def create_random_file( 24 | path: str, 25 | ) -> None: 26 | """Create a random file with random data. 27 | 28 | :param path: The path to create the file at. 29 | """ 30 | with open(path, "wb") as f: 31 | f.write(os.urandom(1)) 32 | 33 | 34 | def user_profile(): 35 | test_profile = UserProfile(username=TEST_ACCOUNT_NAME).get() 36 | print(f"Getting account by name: {test_profile}") 37 | 38 | test_profile = UserProfile(id=TEST_ACCOUNT_ID).get() 39 | print(f"Getting account by id: {test_profile}") 40 | 41 | test_profile = UserProfile.from_username(username=TEST_ACCOUNT_NAME) 42 | print(f"Getting account by name: {test_profile}") 43 | 44 | test_profile = UserProfile.from_id(user_id=TEST_ACCOUNT_ID) 45 | print(f"Getting account by id: {test_profile}") 46 | 47 | print(f"is certified - By ID?: {UserProfile(id=TEST_ACCOUNT_ID).is_certified()}") 48 | print( 49 | f"is certified - By name?: {UserProfile(username=TEST_ACCOUNT_NAME).is_certified()}" 50 | ) 51 | 52 | 53 | def set_permissions(): 54 | name_of_file = "my_file_with_read_permissions_for_service_account.txt" 55 | path_to_file = os.path.join(os.path.expanduser("~/temp"), name_of_file) 56 | create_random_file(path_to_file) 57 | 58 | file_with_read = File( 59 | path=path_to_file, 60 | name=name_of_file, 61 | parent_id=PROJECT_ID, 62 | ).store() 63 | 64 | name_of_file = "my_file_with_default_permissions_for_service_account.txt" 65 | path_to_file = os.path.join(os.path.expanduser("~/temp"), name_of_file) 66 | create_random_file(path_to_file) 67 | 68 | file_with_default = File( 69 | path=path_to_file, 70 | name=name_of_file, 71 | parent_id=PROJECT_ID, 72 | ).store() 73 | 74 | file_with_read.set_permissions(principal_id=TEST_ACCOUNT_ID, access_type=["READ"]) 75 | file_with_default.set_permissions(principal_id=TEST_ACCOUNT_ID) 76 | 77 | print( 78 | f"Read ACL for service account: {file_with_read.get_acl(principal_id=TEST_ACCOUNT_ID)}" 79 | ) 80 | print( 81 | f"Read/Download ACL for service acccount: {file_with_default.get_acl(principal_id=TEST_ACCOUNT_ID)}" 82 | ) 83 | print(f"Admin permissions for self: {file_with_read.get_permissions()}") 84 | print(f"Admin permissions for self: {file_with_default.get_permissions()}") 85 | 86 | 87 | user_profile() 88 | set_permissions() 89 | -------------------------------------------------------------------------------- /docs/tutorials/README: -------------------------------------------------------------------------------- 1 | Tutorials: 2 | The purpose of the documents in this folder is to focus on achieving a basic competence 3 | with the client. The focus is learning how to do something. 4 | 5 | Guideline: 6 | - Provide a clear goal of each lesson 7 | - Ensure the steps are reliable 8 | - Provide visible results often and immediately 9 | - Describe concrete steps 10 | - Offer only minimum, nessecary, explanation - Anything further should be linked to 11 | - Ignore options and alternatives. Stay on one branch to reach a conclusion. 12 | 13 | 14 | 15 | In this tutorial, you will… 16 | Describe what the learner will accomplish (note - not: “you will learn…”). 17 | 18 | First, do x. Now, do y. Now that you have done y, do z. 19 | No room for ambiguity or doubt. 20 | 21 | We must always do x before we do y because… (see Explanation for more details). 22 | Provide minimal explanation of actions in the most basic language possible. Link to more detailed explanation. 23 | 24 | The output should look something like this… 25 | Give your learner clear expectations. 26 | 27 | Notice that… Remember that… 28 | Give your learner plenty of clues to help confirm they are on the right track and orient themselves. 29 | 30 | You have built a secure, three-layer hylomorphic stasis engine… 31 | Describe (and admire, in a mild way) what your learner has accomplished (note - not: “you have learned…”) 32 | -------------------------------------------------------------------------------- /docs/tutorials/configuration.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | The Synapse Python client can be configured either programmatically or by using a configuration file. 4 | 5 | **The default configuration file does not need to be modified for most use-cases**. 6 | 7 | 8 | When installing the Synapse Python client, the `.synapseConfig` is added to your home directory. This configuration file is used to store a number of configuration options, including your Synapse authtoken, cache, and multi-threading settings. 9 | 10 | A full example `.synapseConfig` can be found in the [github repository](https://github.com/Sage-Bionetworks/synapsePythonClient/blob/develop/synapseclient/.synapseConfig). 11 | 12 | ## `.synapseConfig` sections 13 | 14 | ### `[authentication]` 15 | 16 | See details on this section in the [authentication](./authentication.md) document. 17 | 18 | ### `[cache]` 19 | 20 | Your downloaded files are cached to avoid repeat downloads of the same file. Change 'location' to use a different folder on your computer as the cache location 21 | 22 | ### `[endpoints]` 23 | 24 | Configuring these will cause the Python client to use these as Synapse service endpoints instead of the default prod endpoints. 25 | 26 | ### `[transfer]` 27 | 28 | Settings to configure how Synapse uploads/downloads data. 29 | 30 | You may also set the `max_threads` programmatically via: 31 | 32 | ```python 33 | import synapseclient 34 | syn = synapseclient.login() 35 | syn.max_threads = 10 36 | ``` 37 | -------------------------------------------------------------------------------- /docs/tutorials/file_versioning.md: -------------------------------------------------------------------------------- 1 | # File Upload 2 | 3 | Files in Synapse are versionable. Please see [Versioning](https://help.synapse.org/docs/Versioning.2668134540.html) for more information about how versions in Files works. 4 | 5 | ## Uploading a New Version 6 | 7 | Uploading a new version follows the same steps as uploading a file for the first time - use the same file name and store it in the same location (e.g., the same parentId). It is recommended to add a comment to the new version in order to easily track differences at a glance. The example file `raw_data.txt` will now have a version of 2 and a comment describing the change. 8 | 9 | Explicit example: 10 | 11 | ```python 12 | import synapseclient 13 | 14 | syn = synapseclient.login() 15 | 16 | # fetch the file in Synapse 17 | file_to_update = syn.get('syn2222', downloadFile=False) 18 | 19 | # save the local path to the new version of the file 20 | file_to_update.path = '/path/to/new/version/of/raw_data.txt' 21 | 22 | # add a version comment 23 | file_to_update.versionComment = 'Added 5 random normally distributed numbers.' 24 | 25 | # store the new file 26 | updated_file = syn.store(file_to_update) 27 | ``` 28 | 29 | Implicit example: 30 | 31 | ```python 32 | # Assuming that there is a file created with: 33 | syn.store(File('path/to/old/raw_data.txt', parentId='syn123456')) 34 | 35 | # To create a new version of that file, make sure you store it with the exact same name 36 | new_file = syn.store(File('path/to/new_version/raw_data.txt', parentId='syn123456')) 37 | ``` 38 | 39 | ## Updating Annotations or Provenance without Changing Versions 40 | 41 | Any change to a File will automatically update its version. If this isn’t the desired behavior, such as minor changes to the metadata, you can set `forceVersion=False` with the Python client. For command line, the commands `set-annotations` and `set-provenance` will update the metadata without creating a new version. Adding/updating annotations and provenance in the web client will also not cause a version change. 42 | 43 | **Important: Because Provenance is tracked by version, set `forceVersion=False` for minor changes to avoid breaking Provenance.** 44 | 45 | Setting annotations without changing version: 46 | 47 | ```python 48 | # Get file from Synapse, set download=False since we are only updating annotations 49 | file = syn.get('syn56789', download=False) 50 | 51 | # Add annotations 52 | file.annotations = {"fileType":"bam", "assay":"RNA-seq"} 53 | 54 | # Store the file without creating a new version 55 | file = syn.store(file, forceVersion=False) 56 | ``` 57 | 58 | ## Setting Provenance without Changing Version 59 | 60 | To set Provenance without changing the file version: 61 | 62 | ```python 63 | from synapseclient import Activity 64 | 65 | # Get file from Synapse, set download=False since we are only updating provenance 66 | file = syn.get('syn56789', download=False) 67 | 68 | # Add provenance (passing in activity) and store the file without creating a new version 69 | file = syn.store( 70 | file, 71 | activity = Activity(used = 'some_valid_file_synapse_id'), 72 | forceVersion=False 73 | ) 74 | ``` 75 | 76 | ## Downloading a Specific Version 77 | 78 | By default, the File downloaded will always be the most recent version. However, a specific version can be downloaded by passing the version parameter: 79 | 80 | ```python 81 | entity = syn.get("syn3260973", version=1) 82 | ``` 83 | -------------------------------------------------------------------------------- /docs/tutorials/home.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | Welcome! 3 | 4 | In this module you'll find everything you need to get you and your team sharing, 5 | organizing, and discussing your scientific research. 6 | 7 | 8 | ## What do you need to get started? 9 | First off, to browse Synapse and access its most basic level of functionality, you 10 | will need to [register for a Synapse account](https://www.synapse.org/#!RegisterAccount:0). 11 | In doing so, you will be agreeing to the 12 | [Synapse Code of Conduct and Synapse Terms and Conditions of Use](https://help.synapse.org/docs/Synapse-Governance.2004255211.html). 13 | 14 | If you’re simply looking to view a public wiki page, or browse our public project or 15 | file catalog, you can do so without registering for a Synapse account. 16 | 17 | To perform other functions within Synapse, such as exploring, downloading, uploading, 18 | and organizing data, there are additional steps to take: 19 | 20 | 21 | ## Exploring and downloading data 22 | 23 | As a 24 | [registered user](https://help.synapse.org/docs/Synapse-User-Account-Types.2007072795.html#SynapseUserAccountTypes-RegisteredUser), 25 | you can view and download any data that is publicly available. 26 | 27 | However, some data is considered controlled access and requires additional protections 28 | for who can access it and how it can be used. This data will be clearly labelled with 29 | its specific 30 | [conditions for use](https://help.synapse.org/docs/Sharing-Settings,-Permissions,-and-Conditions-for-Use.2024276030.html). 31 | Occasionally, access to controlled data requires additional steps, like having your 32 | analysis plan approved by an ethics board or IRB. Controlled access data can not be 33 | shared. Anyone wishing to gain access to said data must individually agree to the 34 | conditions for use. 35 | 36 | 37 | ## Uploading, organizing, and curating data 38 | 39 | To work with data of your own or your team’s, you must become a 40 | [certified user](https://help.synapse.org/docs/Synapse-User-Account-Types.2007072795.html#SynapseUserAccountTypes-CertifiedUser). To become certified, you must pass a 41 | [certification quiz](https://www.synapse.org/#!Quiz:Certification), 42 | which will ask you about information that can be found throughout this documentation site. 43 | 44 | 45 | ## Accessing mHealth data 46 | 47 | To access mHealth data, you must become a 48 | [validated user](https://help.synapse.org/docs/Synapse-User-Account-Types.2007072795.html#SynapseUserAccountTypes-ValidatedUsers). 49 | 50 | ## Debugging in the client 51 | Erorrs happen, and when they do it is useful to turn on debug level logging within the 52 | Synapse client. In order to turn on debug level logging see: 53 | 54 | - [Debug flag for the programmatic interface](../reference/client.md?h=debug#synapseclient.Synapse) 55 | - [Debug flag for the command line interface](./command_line_client.md?h=debug#options) 56 | -------------------------------------------------------------------------------- /docs/tutorials/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | By following the instructions below, you are installing the `synapseclient`, `synapseutils` and the command line client. 4 | 5 | ## TL;DR For Experts 6 | 1. Set up your Python development environment in your preferred manner (e.g. with `conda`, `pyenv`, etc). 7 | 2. Run 8 | ``` 9 | pip install --upgrade synapseclient 10 | ``` 11 | 3. Verify your installation 12 | ``` 13 | pip show synapseclient 14 | ``` 15 | 16 | [](){ #pypi } 17 | ## Installation Guide For: PyPI Users 18 | 19 | The [synapseclient](https://pypi.python.org/pypi/synapseclient/) package is available from PyPI. It can be installed or upgraded with pip. Due to the nature of Python, we highly recommend you set up your python environment with [conda](https://www.anaconda.com/products/distribution) or [pyenv](https://github.com/pyenv/pyenv) and create virtual environments to control your Python dependencies for your work. 20 | 21 | - conda: Please follow instructions [here](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) to manage environments: 22 | 23 | ```bash 24 | conda create -n synapseclient python=3.9 25 | conda activate synapseclient 26 | 27 | # Here are a few ways to install the client. Choose the one that fits your use-case 28 | # sudo may optionally be needed depending on your setup 29 | 30 | pip install --upgrade synapseclient 31 | pip install --upgrade "synapseclient[pandas]" 32 | pip install --upgrade "synapseclient[pandas, pysftp, boto3]" 33 | ``` 34 | 35 | > **NOTE**
36 | > The `synapseclient` package may require loading shared libraries located in your system's `/usr/local/lib` directory. Some users working 37 | with `conda` have experienced issues with shared libraries not being found due to the system searching in the wrong locations. Although 38 | not recommended, one solution for this is manually configuring the `LD_LIBRARY_PATH` environment variable to point 39 | to the `/usr/local/lib` directory. [See here](https://github.com/conda/conda/issues/12800) for more context on this solution, and for alternatives. 40 | 41 | ---- 42 | 43 | - pyenv: Use [virtualenv](https://virtualenv.pypa.io/en/latest/) to manage your python environment: 44 | 45 | ```bash 46 | pyenv install -v 3.9.13 47 | pyenv global 3.9.13 48 | python -m venv env 49 | source env/bin/activate 50 | 51 | # Here are a few ways to install the client. Choose the one that fits your use-case 52 | # sudo may optionally be needed depending on your setup 53 | 54 | python -m pip install --upgrade synapseclient 55 | python -m pip install --upgrade "synapseclient[pandas]" 56 | python -m pip install --upgrade "synapseclient[pandas, pysftp, boto3]" 57 | 58 | python3 -m pip3 install --upgrade synapseclient 59 | python3 -m pip3 install --upgrade "synapseclient[pandas]" 60 | python3 -m pip3 install --upgrade "synapseclient[pandas, pysftp, boto3]" 61 | ``` 62 | 63 | The dependencies on pandas, pysftp, and boto3 are optional. The Synapse `synapseclient.table` feature integrates with Pandas. Support for sftp is required for users of SFTP file storage. Both require native libraries to be compiled or installed separately from prebuilt binaries. 64 | 65 | ## Installation Guide For: Git Users 66 | 67 | Source code and development versions are [available on Github](https://github.com/Sage-Bionetworks/synapsePythonClient). Installing from source: 68 | 69 | ```bash 70 | git clone https://github.com/Sage-Bionetworks/synapsePythonClient.git 71 | cd synapsePythonClient 72 | ``` 73 | 74 | You can stay on the master branch to get the latest stable release or check out the develop branch or a tagged revision: 75 | 76 | ```bash 77 | git checkout 78 | ``` 79 | 80 | Next, either install the package in the site-packages directory `pip install .` or `pip install -e .` to make the installation follow the head without having to reinstall: 81 | 82 | ```bash 83 | pip install . 84 | ``` 85 | -------------------------------------------------------------------------------- /docs/tutorials/python/activity.md: -------------------------------------------------------------------------------- 1 | # Activity/Provenance 2 | [See the current available tutorial](../python_client.md#provenance) 3 | 4 | ![Under Construction](../../assets/under_construction.png) 5 | 6 | Provenance is a concept describing the origin of something. In Synapse, it is used to describe the connections between the workflow steps used to create a particular file or set of results. Data analysis often involves multiple steps to go from a raw data file to a finished analysis. Synapse’s provenance tools allow users to keep track of each step involved in an analysis and share those steps with other users. 7 | 8 | The model Synapse uses for provenance is based on the [W3C provenance spec](https://www.w3.org/TR/prov-n/) where items are derived from an activity which has components that were **used** and components that were **executed**. Think of the **used** items as input files and **executed** items as software or code. Both **used** and **executed** items can reside in Synapse or in URLs such as a link to a GitHub commit or a link to a specific version of a software tool. 9 | 10 | [Dive into Activity/Provenance further here](../../explanations/domain_models_of_synapse.md#activityprovenance) 11 | 12 | ## Tutorial Purpose 13 | In this tutorial you will: 14 | 15 | 1. Add a new Activity to your File 16 | 1. Add a new Activity to a specific version of your File 17 | 1. Print stored activities on your File 18 | 1. Delete an activity 19 | 20 | ## Prerequisites 21 | - In order to follow this tutorial you will need to have a [Project](./project.md) created with at least one [File](./file.md) with multiple [Versions](./versions.md). 22 | -------------------------------------------------------------------------------- /docs/tutorials/python/folder.md: -------------------------------------------------------------------------------- 1 | # Folders in Synapse 2 | Similar to Projects, Folders are “containers” that offer an additional way to organize your data. Instead of uploading a bunch of single files into your project, you can create folders to separate your data in a systematic way. 3 | 4 | Folders in Synapse always have a “parent”, which could be a project or a folder. You can organize collections of folders and sub-folders, just as you would on your local computer. 5 | 6 | [Read more about Folders](../../explanations/domain_models_of_synapse.md#folders) 7 | 8 | 9 | **Note:** You may optionally follow the [Uploading data in bulk](./upload_data_in_bulk.md) 10 | tutorial instead. The bulk tutorial may fit your needs better as it limits the amount 11 | of code that you are required to write and maintain. 12 | 13 | 14 | This tutorial will follow a mix of 15 | [Flattened Data Layout](../../explanations/structuring_your_project.md#flattened-data-layout-example) 16 | and [Hierarchy Data Layout](../../explanations/structuring_your_project.md#hierarchy-data-layout-example). 17 | It is recommended to use one or the other, but not both. Both are used in this tutorial 18 | to demonstrate the flexibility of storing folders within folders on Synapse. 19 | With this example layout: 20 | ``` 21 | . 22 | ├── experiment_notes 23 | │   ├── notes_2022 24 | │   │   ├── fileA.txt 25 | │   │   └── fileB.txt 26 | │   └── notes_2023 27 | │   ├── fileC.txt 28 | │   └── fileD.txt 29 | ├── biospecimen_experiment_1 30 | │   ├── fileA.txt 31 | │   └── fileB.txt 32 | ├── biospecimen_experiment_2 33 | │   ├── fileC.txt 34 | │   └── fileD.txt 35 | ├── single_cell_RNAseq_batch_1 36 | │   ├── SRR12345678_R1.fastq.gz 37 | │   └── SRR12345678_R2.fastq.gz 38 | └── single_cell_RNAseq_batch_2 39 | ├── SRR12345678_R1.fastq.gz 40 | └── SRR12345678_R2.fastq.gz 41 | ``` 42 | 43 | ## Tutorial Purpose 44 | In this tutorial you will: 45 | 46 | 1. Create 4 new folders 47 | 1. Print stored attributes about your folder 48 | 1. Create 2 sub-folders 49 | 50 | 51 | ## Prerequisites 52 | * Make sure that you have completed the [Project](./project.md) tutorial. 53 | 54 | 55 | ## 1. Create a new folder 56 | 57 | ```python 58 | {!docs/tutorials/python/tutorial_scripts/folder.py!lines=5-35} 59 | ``` 60 | 61 | ## 2. Print stored attributes about your folder 62 | 63 | ```python 64 | {!docs/tutorials/python/tutorial_scripts/folder.py!lines=35-49} 65 | ``` 66 | 67 |
68 | You'll notice the output looks like: 69 | ``` 70 | My folder ID is: syn53205629 71 | The parent ID of my folder is: syn53185532 72 | I created my folder on: 2023-12-28T20:52:50.193Z 73 | The ID of the user that created my folder is: 3481671 74 | My folder was last modified on: 2023-12-28T20:52:50.193Z 75 | ``` 76 |
77 | 78 | 79 | ## 3. Create 2 sub-folders 80 | 81 | ```python 82 | {!docs/tutorials/python/tutorial_scripts/folder.py!lines=52-59} 83 | ``` 84 | 85 | ## Results 86 | Now that you have created your folders you'll be able to inspect this on the Files tab of your project in the synapse web UI. It should look similar to: 87 | 88 | ![folder](./tutorial_screenshots/folder.png) 89 | 90 | 91 | ## Source code for this tutorial 92 | 93 |
94 | Click to show me 95 | 96 | ```python 97 | {!docs/tutorials/python/tutorial_scripts/folder.py!} 98 | ``` 99 |
100 | 101 | ## References used in this tutorial 102 | 103 | - [Folder][folder-reference-sync] 104 | - [syn.login][synapseclient.Synapse.login] 105 | - [syn.findEntityId][synapseclient.Synapse.findEntityId] 106 | - [syn.store][synapseclient.Synapse.store] 107 | -------------------------------------------------------------------------------- /docs/tutorials/python/migrate_data_to_other_storage_locations.md: -------------------------------------------------------------------------------- 1 | # Migrating data to other store locations 2 | ![Under Construction](../../assets/under_construction.png) 3 | -------------------------------------------------------------------------------- /docs/tutorials/python/move_files_and_folders.md: -------------------------------------------------------------------------------- 1 | # Moving files and folders 2 | ![Under Construction](../../assets/under_construction.png) 3 | -------------------------------------------------------------------------------- /docs/tutorials/python/project.md: -------------------------------------------------------------------------------- 1 | # Projects in Synapse 2 | Projects in Synapse are “containers” that group relevant content and people together. All data must be uploaded into a project. Projects can be private so only you can see the contents, they can be shared with your collaborators, or they can be made public so anyone on the web can view your research. 3 | 4 | [Read more about Projects](../../explanations/domain_models_of_synapse.md#projects) 5 | 6 | 7 | [Recommendations for Structuring Your Synapse Project](../../explanations/structuring_your_project.md) 8 | 9 | ## Tutorial Purpose 10 | In this tutorial you will: 11 | 12 | 1. Create a new project 13 | 1. Print stored attributes about your project 14 | 1. Get an existing project 15 | 16 | ## Prerequisites 17 | * Make sure that you have completed the 18 | [Installation](../installation.md) and [Authentication](../authentication.md) setup. 19 | 20 | ## 1. Create a new project 21 | 22 | ```python 23 | import synapseclient 24 | from synapseclient import Project 25 | 26 | syn = synapseclient.login() 27 | 28 | # Project names must be globally unique 29 | project = Project(name="My uniquely named project about Alzheimer's Disease") 30 | project = syn.store(obj=project) 31 | ``` 32 | 33 | Now that you have created your project you are able to inspect the project in the [synapse web UI](https://www.synapse.org/#!Profile:v/projects/created_by_me). 34 | 35 | 36 | 37 | ## 2. Print stored attributes about your project 38 | ```python 39 | my_synapse_project_id = project.id 40 | print(f"My project ID is: {my_synapse_project_id}") 41 | 42 | project_creation_date = project.createdOn 43 | print(f"I created my project on: {project.createdOn}") 44 | 45 | user_id_who_created_project = project.createdBy 46 | print(f"The ID of the user that created my project is: {user_id_who_created_project}") 47 | 48 | project_modified_on_date = project.modifiedOn 49 | print(f"My project was last modified on: {project_modified_on_date}") 50 | ``` 51 |
52 | You'll notice the output looks like: 53 | 54 | ``` 55 | My project ID is: syn12345678 56 | I created my project on: 2000-01-01T12:00:00.001Z 57 | The ID of the user that created my project is: 1234567 58 | My project was last modified on: 2000-01-01T12:00:00.001Z 59 | ``` 60 |
61 | 62 | 63 | Find all of the available attributes about your project in the 64 | [API Reference section of the documentation][synapseclient.entity.Project]. 65 | 66 | 67 | ## 3. Get an existing project 68 | Each Project only needs to be created once. Since you've already created it you can 69 | access it again by retrieving the synapse ID of the project and retrieving 70 | the existing project object. 71 | ```python 72 | my_project_id = syn.findEntityId( 73 | name="My uniquely named project about Alzheimer's Disease" 74 | ) 75 | my_project_object = syn.get(entity=my_project_id) 76 | print(f"I just got my project: {my_project_object.name}, id: {my_project_id}") 77 | ``` 78 | 79 |
80 | The result will look like: 81 | 82 | ``` 83 | I just got my project: My uniquely named project about Alzheimer's Disease, id: syn12345678 84 | ``` 85 |
86 | 87 | ## Source code for this tutorial 88 | 89 |
90 | Click to show me 91 | 92 | ```python 93 | {!docs/tutorials/python/tutorial_scripts/project.py!} 94 | ``` 95 |
96 | 97 | ## References used in this tutorial 98 | 99 | - [Project][project-reference-sync] 100 | - [syn.login][synapseclient.Synapse.login] 101 | - [syn.store][synapseclient.Synapse.store] 102 | - [syn.get][synapseclient.Synapse.get] 103 | - [syn.findEntityId][synapseclient.Synapse.findEntityId] 104 | -------------------------------------------------------------------------------- /docs/tutorials/python/sharing_settings.md: -------------------------------------------------------------------------------- 1 | # Sharing Settings 2 | ![Under Construction](../../assets/under_construction.png) 3 | -------------------------------------------------------------------------------- /docs/tutorials/python/table.md: -------------------------------------------------------------------------------- 1 | # Tables 2 | ![Under Construction](../../assets/under_construction.png) 3 | -------------------------------------------------------------------------------- /docs/tutorials/python/table_crud.md: -------------------------------------------------------------------------------- 1 | # Using a Table (Create, Read, Update, Delete) 2 | ![Under Construction](../../assets/under_construction.png) 3 | -------------------------------------------------------------------------------- /docs/tutorials/python/team.md: -------------------------------------------------------------------------------- 1 | # Teams 2 | ![Under Construction](../../assets/under_construction.png) 3 | Teams allow you to easily manage groups of users to control access to projects, 4 | communicate with colleagues, and participate in challenges. 5 | 6 | In this tutorial you will: 7 | 8 | - Create and manage a team 9 | - Invite users to join a team 10 | - Resend/Cancel an invitation to join a team 11 | - Remove users from a team 12 | -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_screenshots/annotation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/tutorials/python/tutorial_screenshots/annotation.png -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_screenshots/dataset_collection_default_schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/tutorials/python/tutorial_screenshots/dataset_collection_default_schema.png -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_screenshots/dataset_collection_with_datasets.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/tutorials/python/tutorial_screenshots/dataset_collection_with_datasets.png -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_screenshots/dataset_default_schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/tutorials/python/tutorial_screenshots/dataset_default_schema.png -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_screenshots/dataset_with_files.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/tutorials/python/tutorial_screenshots/dataset_with_files.png -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_screenshots/edit_provenance_button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/tutorials/python/tutorial_screenshots/edit_provenance_button.png -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_screenshots/edit_provenance_screen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/tutorials/python/tutorial_screenshots/edit_provenance_screen.png -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_screenshots/entityview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/tutorials/python/tutorial_screenshots/entityview.png -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_screenshots/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/tutorials/python/tutorial_screenshots/file.png -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_screenshots/folder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/docs/tutorials/python/tutorial_screenshots/folder.png -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_scripts/annotation.py: -------------------------------------------------------------------------------- 1 | """ 2 | Here is where you'll find the code for the Annotation tutorial. 3 | """ 4 | 5 | # Step 1: Add several annotations to stored files 6 | import os 7 | 8 | import synapseclient 9 | from synapseclient import File 10 | 11 | syn = synapseclient.login() 12 | 13 | # Retrieve the project ID 14 | my_project_id = syn.findEntityId( 15 | name="My uniquely named project about Alzheimer's Disease" 16 | ) 17 | 18 | # Retrieve the folders I want to annotate files in 19 | batch_1_folder_id = syn.findEntityId( 20 | name="single_cell_RNAseq_batch_1", parent=my_project_id 21 | ) 22 | 23 | print(f"Batch 1 Folder ID: {batch_1_folder_id}") 24 | 25 | 26 | # Define the annotations I want to set 27 | annotation_values = { 28 | "species": "Homo sapiens", 29 | "dataType": "geneExpression", 30 | "assay": "SCRNA-seq", 31 | "fileFormat": "fastq", 32 | } 33 | 34 | # Loop over all of the files and set their annotations 35 | for file_batch_1 in syn.getChildren(parent=batch_1_folder_id, includeTypes=["file"]): 36 | # Grab and print the existing annotations this File may already have 37 | existing_annotations_for_file = syn.get_annotations(entity=file_batch_1) 38 | 39 | print( 40 | f"Got the annotations for File: {file_batch_1['name']}, ID: {file_batch_1['id']}, Annotations: {existing_annotations_for_file}" 41 | ) 42 | 43 | # Merge the new annotations with anything existing 44 | existing_annotations_for_file.update(annotation_values) 45 | 46 | existing_annotations_for_file = syn.set_annotations( 47 | annotations=existing_annotations_for_file 48 | ) 49 | 50 | print( 51 | f"Set the annotations for File: {file_batch_1['name']}, ID: {file_batch_1['id']}, Annotations: {existing_annotations_for_file}" 52 | ) 53 | 54 | # Step 2: Upload 2 new files and set the annotations at the same time 55 | # In order for the following script to work please replace the files with ones that 56 | # already exist on your local machine. 57 | batch_1_scrnaseq_new_file_1 = File( 58 | path=os.path.expanduser( 59 | "~/my_ad_project/single_cell_RNAseq_batch_1/SRR92345678_R1.fastq.gz" 60 | ), 61 | parent=batch_1_folder_id, 62 | annotations=annotation_values, 63 | ) 64 | batch_1_scrnaseq_new_file_2 = File( 65 | path=os.path.expanduser( 66 | "~/my_ad_project/single_cell_RNAseq_batch_1/SRR92345678_R2.fastq.gz" 67 | ), 68 | parent=batch_1_folder_id, 69 | annotations=annotation_values, 70 | ) 71 | batch_1_scrnaseq_new_file_1 = syn.store(obj=batch_1_scrnaseq_new_file_1) 72 | batch_1_scrnaseq_new_file_2 = syn.store(obj=batch_1_scrnaseq_new_file_2) 73 | 74 | print( 75 | f"Stored file: {batch_1_scrnaseq_new_file_1['name']}, ID: {batch_1_scrnaseq_new_file_1['id']}, Annotations: {batch_1_scrnaseq_new_file_1['annotations']}" 76 | ) 77 | print( 78 | f"Stored file: {batch_1_scrnaseq_new_file_2['name']}, ID: {batch_1_scrnaseq_new_file_2['id']}, Annotations: {batch_1_scrnaseq_new_file_2['annotations']}" 79 | ) 80 | -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_scripts/dataset.py: -------------------------------------------------------------------------------- 1 | """Here is where you'll find the code for the dataset tutorial.""" 2 | 3 | import pandas as pd 4 | 5 | from synapseclient import Synapse 6 | from synapseclient.models import ( 7 | Column, 8 | ColumnType, 9 | Dataset, 10 | EntityRef, 11 | File, 12 | Folder, 13 | Project, 14 | ) 15 | 16 | # First, let's get the project that we want to create the dataset in 17 | syn = Synapse() 18 | syn.login() 19 | 20 | project = Project( 21 | name="My uniquely named project about Alzheimer's Disease" 22 | ).get() # Replace with your project name 23 | project_id = project.id 24 | print(f"My project ID is {project_id}") 25 | 26 | # Next, let's create the dataset. We'll use the project id as the parent id. 27 | # To begin, the dataset will be empty, but if you view the dataset's schema in the UI, 28 | # you will notice that datasets come with default columns. 29 | my_new_dataset = Dataset(parent_id=project_id, name="My New Dataset").store() 30 | print(f"My Dataset's ID is {my_new_dataset.id}") 31 | 32 | # Now, let's add some files to the dataset. There are three ways to add files to a dataset: 33 | # 1. Add an Entity Reference to a file with its ID and version 34 | my_new_dataset.add_item( 35 | EntityRef(id="syn51790029", version=1) 36 | ) # Replace with the ID of the file you want to add 37 | # 2. Add a File with its ID and version 38 | my_new_dataset.add_item( 39 | File(id="syn51790028", version_number=1) 40 | ) # Replace with the ID of the file you want to add 41 | # 3. Add a Folder. In this case, all child files of the folder are added to the dataset recursively. 42 | my_new_dataset.add_item( 43 | Folder(id="syn64893446") 44 | ) # Replace with the ID of the folder you want to add 45 | # Our changes won't be persisted to Synapse until we call the store() method. 46 | my_new_dataset.store() 47 | 48 | # Now that our Dataset with all of our files has been created, the next time 49 | # we want to use it, we can retrieve it from Synapse. 50 | my_retrieved_dataset = Dataset(id=my_new_dataset.id).get() 51 | print(f"My Dataset's ID is {my_retrieved_dataset.id}") 52 | print(len(my_retrieved_dataset.items)) 53 | 54 | # If you want to query your dataset for files that match certain criteria, you can do so 55 | # using the query method. 56 | rows = Dataset.query( 57 | query=f"SELECT * FROM {my_retrieved_dataset.id} WHERE name like '%test%'" 58 | ) 59 | print(rows) 60 | 61 | # In addition to the default columns, you may want to annotate items in your dataset using 62 | # custom columns. 63 | my_retrieved_dataset.add_column( 64 | column=Column( 65 | name="my_annotation", 66 | column_type=ColumnType.STRING, 67 | ) 68 | ) 69 | my_retrieved_dataset.store() 70 | 71 | # Now that our custom column has been added, we can update the dataset with new values. 72 | modified_data = pd.DataFrame( 73 | { 74 | "id": "syn51790028", # The ID of one of our Files 75 | "my_annotation": ["excellent data"], 76 | } 77 | ) 78 | my_retrieved_dataset.update_rows( 79 | values=modified_data, primary_keys=["id"], dry_run=False 80 | ) 81 | 82 | 83 | # Finally, let's save a snapshot of the dataset. 84 | snapshot_info = my_retrieved_dataset.snapshot( 85 | comment="My first snapshot", 86 | label="My first snapshot", 87 | ) 88 | print(snapshot_info) 89 | -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_scripts/dataset_collection.py: -------------------------------------------------------------------------------- 1 | """Here is where you'll find the code for the DatasetCollection tutorial.""" 2 | 3 | import pandas as pd 4 | 5 | from synapseclient import Synapse 6 | from synapseclient.models import Column, ColumnType, Dataset, DatasetCollection, Project 7 | 8 | # First, let's get the project that we want to create the DatasetCollection in 9 | syn = Synapse() 10 | syn.login() 11 | 12 | project = Project( 13 | name="My uniquely named project about Alzheimer's Disease" 14 | ).get() # Replace with your project name 15 | project_id = project.id 16 | print(f"My project ID is {project_id}") 17 | 18 | # This tutorial assumes that you have already created datasets that you would like to add to a DatasetCollection. 19 | # If you need help creating datasets, you can refer to the dataset tutorial. 20 | 21 | # For this example, we will be using datasets already created in the project. 22 | # Let's create the DatasetCollection. We'll use the project id as the parent id. 23 | # At first, the DatasetCollection will be empty, but if you view the DatasetCollection's schema in the UI, 24 | # you will notice that DatasetCollections come with default columns. 25 | DATASET_IDS = [ 26 | "syn65987017", 27 | "syn65987019", 28 | "syn65987020", 29 | ] # Replace with your dataset IDs 30 | test_dataset_collection = DatasetCollection( 31 | parent_id=project_id, name="test_dataset_collection" 32 | ).store() 33 | print(f"My DatasetCollection's ID is {test_dataset_collection.id}") 34 | 35 | # Now, let's add some datasets to the collection. We will loop through our dataset ids and add each dataset to the 36 | # collection using the `add_item` method. 37 | for dataset_id in DATASET_IDS: 38 | test_dataset_collection.add_item(Dataset(id=dataset_id).get()) 39 | # Our changes won't be persisted to Synapse until we call the `store` method on our DatasetCollection. 40 | test_dataset_collection.store() 41 | 42 | # Now that our DatasetCollection with all of our datasets has been created, the next time we want to use it, 43 | # we can retrieve it from Synapse. 44 | my_retrieved_dataset_collection = DatasetCollection(id=test_dataset_collection.id).get() 45 | print(f"My DatasetCollection's ID is still {my_retrieved_dataset_collection.id}") 46 | print(f"My DatasetCollection has {len(my_retrieved_dataset_collection.items)} items") 47 | 48 | # In addition to the default columns, you may want to annotate items in your DatasetCollection using 49 | # custom columns. 50 | my_retrieved_dataset_collection.add_column( 51 | column=Column( 52 | name="my_annotation", 53 | column_type=ColumnType.STRING, 54 | ) 55 | ) 56 | my_retrieved_dataset_collection.store() 57 | 58 | # Now that our custom column has been added, we can update the DatasetCollection with new annotations. 59 | modified_data = pd.DataFrame( 60 | { 61 | "id": DATASET_IDS, 62 | "my_annotation": ["good dataset"] * len(DATASET_IDS), 63 | } 64 | ) 65 | my_retrieved_dataset_collection.update_rows( 66 | values=modified_data, primary_keys=["id"], dry_run=False 67 | ) 68 | 69 | # If you want to query your DatasetCollection for items that match certain criteria, you can do so 70 | # using the `query` method. 71 | rows = my_retrieved_dataset_collection.query( 72 | query=f"SELECT id, my_annotation FROM {my_retrieved_dataset_collection.id} WHERE my_annotation = 'good dataset'" 73 | ) 74 | print(rows) 75 | 76 | # Create a snapshot of the DatasetCollection 77 | my_retrieved_dataset_collection.snapshot(comment="test snapshot") 78 | -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_scripts/download_data_in_bulk.py: -------------------------------------------------------------------------------- 1 | """ 2 | Here is where you'll find the code for the downloading data in bulk tutorial. 3 | """ 4 | 5 | import os 6 | 7 | import synapseclient 8 | from synapseclient.models import Folder, Project 9 | 10 | syn = synapseclient.Synapse() 11 | syn.login() 12 | 13 | # Create some constants to store the paths to the data 14 | DIRECTORY_TO_SYNC_PROJECT_TO = os.path.expanduser(os.path.join("~", "my_ad_project")) 15 | FOLDER_NAME_TO_SYNC = "biospecimen_experiment_1" 16 | DIRECTORY_TO_SYNC_FOLDER_TO = os.path.join( 17 | DIRECTORY_TO_SYNC_PROJECT_TO, FOLDER_NAME_TO_SYNC 18 | ) 19 | 20 | # Step 1: Create an instance of the container I want to sync the data from and sync 21 | project = Project(name="My uniquely named project about Alzheimer's Disease") 22 | 23 | # We'll set the `if_collision` to `keep.local` so that we don't overwrite any files 24 | project.sync_from_synapse(path=DIRECTORY_TO_SYNC_PROJECT_TO, if_collision="keep.local") 25 | 26 | # Print out the contents of the directory where the data was synced to 27 | # Explore the directory to see the contents have been recursively synced. 28 | print(os.listdir(DIRECTORY_TO_SYNC_PROJECT_TO)) 29 | 30 | # Step 2: The same as step 1, but for a single folder 31 | folder = Folder(name=FOLDER_NAME_TO_SYNC, parent_id=project.id) 32 | 33 | folder.sync_from_synapse(path=DIRECTORY_TO_SYNC_FOLDER_TO, if_collision="keep.local") 34 | 35 | print(os.listdir(os.path.expanduser(DIRECTORY_TO_SYNC_FOLDER_TO))) 36 | 37 | # Step 3: Loop over all files/folders on the project/folder object instances 38 | for folder_at_root in project.folders: 39 | print(f"Folder at root: {folder_at_root.name}") 40 | 41 | for file_in_root_folder in folder_at_root.files: 42 | print(f"File in {folder_at_root.name}: {file_in_root_folder.name}") 43 | 44 | for folder_in_folder in folder_at_root.folders: 45 | print(f"Folder in {folder_at_root.name}: {folder_in_folder.name}") 46 | for file_in_folder in folder_in_folder.files: 47 | print(f"File in {folder_in_folder.name}: {file_in_folder.name}") 48 | -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_scripts/entityview.py: -------------------------------------------------------------------------------- 1 | """ 2 | Here is where you'll find the code for the EntityView tutorial. 3 | """ 4 | 5 | import pandas as pd 6 | 7 | from synapseclient import Synapse 8 | from synapseclient.models import ( 9 | Column, 10 | ColumnType, 11 | EntityView, 12 | Project, 13 | ViewTypeMask, 14 | query, 15 | ) 16 | 17 | syn = Synapse() 18 | syn.login() 19 | 20 | # First let's get the project we want to create the EntityView in 21 | my_project = Project(name="My uniquely named project about Alzheimer's Disease").get() 22 | project_id = my_project.id 23 | 24 | # Next let's add some columns to the EntityView, the data in these columns will end up 25 | # being stored as annotations on the files 26 | columns = [ 27 | Column(name="species", column_type=ColumnType.STRING), 28 | Column(name="dataType", column_type=ColumnType.STRING), 29 | Column(name="assay", column_type=ColumnType.STRING), 30 | Column(name="fileFormat", column_type=ColumnType.STRING), 31 | ] 32 | 33 | # Then we will create a EntityView that is scoped to the project, and will contain a row 34 | # for each file in the project 35 | view = EntityView( 36 | name="My Entity View", 37 | parent_id=project_id, 38 | scope_ids=[project_id], 39 | view_type_mask=ViewTypeMask.FILE, 40 | columns=columns, 41 | ).store() 42 | 43 | print(f"My EntityView ID is: {view.id}") 44 | 45 | # When the columns are printed you'll notice that it contains a number of columns that 46 | # are automatically added by Synapse in addition to the ones we added 47 | print(view.columns.keys()) 48 | 49 | # Query the EntityView 50 | results_as_dataframe: pd.DataFrame = query( 51 | query=f"SELECT id, name, species, dataType, assay, fileFormat, path FROM {view.id} WHERE path like '%single_cell_RNAseq_batch_1%'", 52 | include_row_id_and_row_version=False, 53 | ) 54 | print(results_as_dataframe) 55 | 56 | # Finally let's update the annotations on the files in the project 57 | results_as_dataframe["species"] = ["Homo sapiens"] * len(results_as_dataframe) 58 | results_as_dataframe["dataType"] = ["geneExpression"] * len(results_as_dataframe) 59 | results_as_dataframe["assay"] = ["SCRNA-seq"] * len(results_as_dataframe) 60 | results_as_dataframe["fileFormat"] = ["fastq"] * len(results_as_dataframe) 61 | 62 | view.update_rows( 63 | values=results_as_dataframe, 64 | primary_keys=["id"], 65 | wait_for_eventually_consistent_view=True, 66 | ) 67 | 68 | 69 | # Over time you may have a need to add or remove scopes from the EntityView, you may 70 | # use `add` or `remove` along with the Synapse ID of the scope you wish to add/remove 71 | view.scope_ids.add("syn1234") 72 | # view.scope_ids.remove("syn1234") 73 | view.store() 74 | 75 | # You may also need to add or remove the types of Entities that may show up in your view 76 | # You will be able to specify multiple types using the bitwise OR operator, or a single value 77 | view.view_type_mask = ViewTypeMask.FILE | ViewTypeMask.FOLDER 78 | # view.view_type_mask = ViewTypeMask.FILE 79 | view.store() 80 | -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_scripts/folder.py: -------------------------------------------------------------------------------- 1 | """ 2 | Here is where you'll find the code for the Folder tutorial. 3 | """ 4 | 5 | # Step 1: Create a new folder 6 | import synapseclient 7 | from synapseclient import Folder 8 | 9 | syn = synapseclient.login() 10 | 11 | # Retrieve the project ID 12 | my_project_id = syn.findEntityId( 13 | name="My uniquely named project about Alzheimer's Disease" 14 | ) 15 | 16 | # Create a Folder object and store it 17 | my_scrnaseq_batch_1_folder = Folder( 18 | name="single_cell_RNAseq_batch_1", parent=my_project_id 19 | ) 20 | my_scrnaseq_batch_1_folder = syn.store(obj=my_scrnaseq_batch_1_folder) 21 | 22 | my_scrnaseq_batch_2_folder = Folder( 23 | name="single_cell_RNAseq_batch_2", parent=my_project_id 24 | ) 25 | my_scrnaseq_batch_2_folder = syn.store(obj=my_scrnaseq_batch_2_folder) 26 | 27 | biospecimen_experiment_1_folder = Folder( 28 | name="biospecimen_experiment_1", parent=my_project_id 29 | ) 30 | biospecimen_experiment_1_folder = syn.store(obj=biospecimen_experiment_1_folder) 31 | 32 | biospecimen_experiment_2_folder = Folder( 33 | name="biospecimen_experiment_2", parent=my_project_id 34 | ) 35 | biospecimen_experiment_2_folder = syn.store(obj=biospecimen_experiment_2_folder) 36 | 37 | # Step 2: Print stored attributes about your folder 38 | my_scrnaseq_batch_1_folder_id = my_scrnaseq_batch_1_folder.id 39 | print(f"My folder ID is: {my_scrnaseq_batch_1_folder_id}") 40 | 41 | print(f"The parent ID of my folder is: {my_scrnaseq_batch_1_folder.parentId}") 42 | 43 | print(f"I created my folder on: {my_scrnaseq_batch_1_folder.createdOn}") 44 | 45 | print( 46 | f"The ID of the user that created my folder is: {my_scrnaseq_batch_1_folder.createdBy}" 47 | ) 48 | 49 | print(f"My folder was last modified on: {my_scrnaseq_batch_1_folder.modifiedOn}") 50 | 51 | # Step 3: Create 2 sub-folders 52 | hierarchical_root_folder = Folder(name="experiment_notes", parent=my_project_id) 53 | hierarchical_root_folder = syn.store(obj=hierarchical_root_folder) 54 | 55 | folder_notes_2023 = Folder(name="notes_2023", parent=hierarchical_root_folder.id) 56 | folder_notes_2023 = syn.store(obj=folder_notes_2023) 57 | 58 | folder_notes_2022 = Folder(name="notes_2022", parent=hierarchical_root_folder.id) 59 | folder_notes_2022 = syn.store(obj=folder_notes_2022) 60 | -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_scripts/project.py: -------------------------------------------------------------------------------- 1 | """ 2 | Here is where you'll find the code for the Project tutorial. 3 | """ 4 | 5 | # Step 1: Create a new project 6 | import synapseclient 7 | from synapseclient import Project 8 | 9 | syn = synapseclient.login() 10 | 11 | # Project names must be globally unique 12 | project = Project(name="My uniquely named project about Alzheimer's Disease") 13 | project = syn.store(obj=project) 14 | 15 | # Step 2: Print stored attributes about your project 16 | print(f"My project ID is: {project.id}") 17 | 18 | print(f"I created my project on: {project.createdOn}") 19 | 20 | print(f"The ID of the user that created my project is: {project.createdBy}") 21 | 22 | print(f"My project was last modified on: {project.modifiedOn}") 23 | 24 | # Step 3: Get an existing project 25 | my_project_id = syn.findEntityId( 26 | name="My uniquely named project about Alzheimer's Disease" 27 | ) 28 | my_project_object = syn.get(entity=my_project_id) 29 | print(f"I just got my project: {my_project_object.name}, id: {my_project_id}") 30 | -------------------------------------------------------------------------------- /docs/tutorials/python/tutorial_scripts/upload_data_in_bulk.py: -------------------------------------------------------------------------------- 1 | """ 2 | Here is where you'll find the code for the uploading data in bulk tutorial. 3 | """ 4 | 5 | import os 6 | 7 | import synapseclient 8 | import synapseutils 9 | 10 | syn = synapseclient.Synapse() 11 | syn.login() 12 | 13 | # Create some constants to store the paths to the data 14 | DIRECTORY_FOR_MY_PROJECT = os.path.expanduser(os.path.join("~", "my_ad_project")) 15 | PATH_TO_MANIFEST_FILE = os.path.expanduser(os.path.join("~", "manifest-for-upload.tsv")) 16 | 17 | # Step 1: Let's find the synapse ID of our project: 18 | my_project_id = syn.findEntityId( 19 | name="My uniquely named project about Alzheimer's Disease" 20 | ) 21 | 22 | # Step 2: Create a manifest TSV file to upload data in bulk 23 | # Note: When this command is run it will re-create your directory structure within 24 | # Synapse. Be aware of this before running this command. 25 | # If folders with the exact names already exists in Synapse, those folders will be used. 26 | synapseutils.generate_sync_manifest( 27 | syn=syn, 28 | directory_path=DIRECTORY_FOR_MY_PROJECT, 29 | parent_id=my_project_id, 30 | manifest_path=PATH_TO_MANIFEST_FILE, 31 | ) 32 | 33 | # Step 3: After generating the manifest file, we can upload the data in bulk 34 | synapseutils.syncToSynapse( 35 | syn=syn, manifestFile=PATH_TO_MANIFEST_FILE, sendMessages=False 36 | ) 37 | 38 | # Step 4: Let's add an annotation to our manifest file 39 | # Pandas is a powerful data manipulation library in Python, although it is not required 40 | # for this tutorial, it is used here to demonstrate how you can manipulate the manifest 41 | # file before uploading it to Synapse. 42 | import pandas as pd 43 | 44 | # Read TSV file into a pandas DataFrame 45 | df = pd.read_csv(PATH_TO_MANIFEST_FILE, sep="\t") 46 | 47 | # Add a new column to the DataFrame 48 | df["species"] = "Homo sapiens" 49 | 50 | # Write the DataFrame back to the manifest file 51 | df.to_csv(PATH_TO_MANIFEST_FILE, sep="\t", index=False) 52 | 53 | synapseutils.syncToSynapse( 54 | syn=syn, 55 | manifestFile=PATH_TO_MANIFEST_FILE, 56 | sendMessages=False, 57 | ) 58 | 59 | # Step 5: Let's create an Activity/Provenance 60 | # First let's find the row in the TSV we want to update. This code finds the row number 61 | # that we would like to update. 62 | row_index = df[ 63 | df["path"] == f"{DIRECTORY_FOR_MY_PROJECT}/biospecimen_experiment_1/fileA.txt" 64 | ].index 65 | 66 | 67 | # After finding the row we want to update let's go ahead and add a relationship to 68 | # another file in our manifest. This allows us to say "We used 'this' file in some way". 69 | df.loc[ 70 | row_index, "used" 71 | ] = f"{DIRECTORY_FOR_MY_PROJECT}/single_cell_RNAseq_batch_1/SRR12345678_R1.fastq.gz" 72 | 73 | # Let's also link to the pipeline that we ran in order to produce these results. In a 74 | # real scenario you may want to link to a specific run of the tool where the results 75 | # were produced. 76 | df.loc[row_index, "executed"] = "https://nf-co.re/rnaseq/3.14.0" 77 | 78 | # Let's also add a description for this Activity/Provenance 79 | df.loc[ 80 | row_index, "activityDescription" 81 | ] = "Experiment results created as a result of the linked data while running the pipeline." 82 | 83 | # Write the DataFrame back to the manifest file 84 | df.to_csv(PATH_TO_MANIFEST_FILE, sep="\t", index=False) 85 | 86 | synapseutils.syncToSynapse( 87 | syn=syn, 88 | manifestFile=PATH_TO_MANIFEST_FILE, 89 | sendMessages=False, 90 | ) 91 | -------------------------------------------------------------------------------- /docs/tutorials/python/versions.md: -------------------------------------------------------------------------------- 1 | # Versions 2 | [See the current available tutorial](../python_client.md#versioning) 3 | 4 | ![Under Construction](../../assets/under_construction.png) 5 | -------------------------------------------------------------------------------- /docs/tutorials/python/wiki.md: -------------------------------------------------------------------------------- 1 | # Wikis on Projects 2 | ![Under Construction](../../assets/under_construction.png) 3 | -------------------------------------------------------------------------------- /docs/tutorials/sample_files/README: -------------------------------------------------------------------------------- 1 | These are some example files referenced in some tutorials. These are text files with 2 | specific file extensions used to outline some possible file types that might be used 3 | in Synapse. 4 | -------------------------------------------------------------------------------- /docs/tutorials/sample_files/my_ad_project/experiment_notes/notes_2022/fileA.txt: -------------------------------------------------------------------------------- 1 | a 2 | 1 3 | -------------------------------------------------------------------------------- /docs/tutorials/sample_files/my_ad_project/experiment_notes/notes_2022/fileB.txt: -------------------------------------------------------------------------------- 1 | a 2 | 2 3 | -------------------------------------------------------------------------------- /docs/tutorials/sample_files/my_ad_project/experiment_notes/notes_2023/fileC.txt: -------------------------------------------------------------------------------- 1 | a 2 | 3 3 | -------------------------------------------------------------------------------- /docs/tutorials/sample_files/my_ad_project/experiment_notes/notes_2023/fileD.txt: -------------------------------------------------------------------------------- 1 | a 2 | 4 3 | -------------------------------------------------------------------------------- /docs/tutorials/sample_files/my_ad_project/single_cell_RNAseq_batch_1/SRR12345678_R1.fastq.gz: -------------------------------------------------------------------------------- 1 | a 2 | 5 3 | 9 4 | -------------------------------------------------------------------------------- /docs/tutorials/sample_files/my_ad_project/single_cell_RNAseq_batch_1/SRR12345678_R2.fastq.gz: -------------------------------------------------------------------------------- 1 | a 2 | 6 3 | 10 4 | -------------------------------------------------------------------------------- /docs/tutorials/sample_files/my_ad_project/single_cell_RNAseq_batch_1/SRR92345678_R1.fastq.gz: -------------------------------------------------------------------------------- 1 | 12 2 | -------------------------------------------------------------------------------- /docs/tutorials/sample_files/my_ad_project/single_cell_RNAseq_batch_1/SRR92345678_R2.fastq.gz: -------------------------------------------------------------------------------- 1 | 11 2 | -------------------------------------------------------------------------------- /docs/tutorials/sample_files/my_ad_project/single_cell_RNAseq_batch_2/SRR12345678_R1.fastq.gz: -------------------------------------------------------------------------------- 1 | a 2 | 7 3 | -------------------------------------------------------------------------------- /docs/tutorials/sample_files/my_ad_project/single_cell_RNAseq_batch_2/SRR12345678_R2.fastq.gz: -------------------------------------------------------------------------------- 1 | a 2 | 8 3 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 88 3 | target-version = ['py39'] 4 | include = '\.pyi?$' 5 | 6 | [tool.ruff] 7 | extend-ignore = ["E501"] 8 | exclude = [ 9 | 'docs', 10 | 'tests' 11 | ] 12 | 13 | [tool.bandit] 14 | exclude_dirs = ["tests"] 15 | skips = ["B101", "B303", "B608", "B311", "B113", "B310", "B110"] 16 | 17 | [tool.isort] 18 | profile = "black" 19 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | python_files = test_*.py unit_test_*.py integration_test*.py 3 | asyncio_mode = auto 4 | asyncio_default_fixture_loop_scope = session 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Installation script for Synapse Client for Python 2 | ############################################################ 3 | import json 4 | import os 5 | 6 | from setuptools import setup 7 | 8 | # make sure not to overwrite existing .synapseConfig with our example one 9 | data_files = ( 10 | [(os.path.expanduser("~"), ["synapseclient/.synapseConfig"])] 11 | if not os.path.exists(os.path.expanduser("~/.synapseConfig")) 12 | else [] 13 | ) 14 | # figure out the version 15 | with open("synapseclient/synapsePythonClient") as config: 16 | __version__ = json.load(config)["latestVersion"] 17 | 18 | setup(data_files=data_files, version=__version__) 19 | -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | sonar.projectKey=Sage-Bionetworks_synapsePythonClient 2 | sonar.organization=sage-bionetworks 3 | sonar.python.coverage.reportPaths=coverage.xml 4 | # Reccommendation per: 5 | # https://community.sonarsource.com/t/6-new-rules-to-support-python-type-hints/89560/2 6 | sonar.sources=synapseclient,synapseutils 7 | sonar.tests=tests 8 | -------------------------------------------------------------------------------- /synapseclient/.synapseConfig: -------------------------------------------------------------------------------- 1 | ########################### 2 | # Login Credentials # 3 | ########################### 4 | 5 | ## Used for logging in to Synapse. See https://python-docs.synapse.org/tutorials/authentication/ 6 | ## for information on retrieving an auth token. 7 | 8 | #[default] 9 | #username = default_user 10 | #authtoken = default_auth_token 11 | 12 | #[profile user1] 13 | #username = user1 14 | #authtoken = user1_auth_token 15 | 16 | #[profile user2] 17 | #username = user2 18 | #authtoken = user2_auth_token 19 | 20 | ## If you have projects with file stored on SFTP servers, you can specify your credentials here 21 | ## You can specify multiple sftp credentials 22 | #[sftp://some.sftp.url.com] 23 | #username= 24 | #password= 25 | #[sftp://a.different.sftp.url.com] 26 | #username= 27 | #password= 28 | 29 | 30 | ## If you have projects that need to be stored in an S3-like (e.g. AWS S3, Openstack) storage but cannot allow Synapse 31 | ## to manage access your storage you may put your credentials here. 32 | ## To avoid duplicating credentials with that used by the AWS Command Line Client, 33 | ## simply put the profile name form your ~/.aws/credentials file 34 | ## more information about aws credentials can be found here http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html 35 | #[https://s3.amazonaws.com/bucket_name] # this is the bucket's endpoint 36 | #profile_name=local_credential_profile_name 37 | 38 | 39 | ########################### 40 | # Caching # 41 | ########################### 42 | 43 | ## your downloaded files are cached to avoid repeat downloads of the same file. change 'location' to use a different folder on your computer as the cache location 44 | #[cache] 45 | #location = ~/.synapseCache 46 | 47 | 48 | ########################### 49 | # Advanced Configurations # 50 | ########################### 51 | 52 | ## If this section is specified, then the synapseclient will print out debug information 53 | #[debug] 54 | 55 | 56 | ## Configuring these will cause the Python client to use these as Synapse service endpoints instead of the default prod endpoints. 57 | #[endpoints] 58 | #repoEndpoint= 59 | #authEndpoint= 60 | #fileHandleEndpoint= 61 | #portalEndpoint= 62 | 63 | ## Settings to configure how Synapse uploads/downloads data 64 | #[transfer] 65 | 66 | # use this to configure the default for how many threads/connections Synapse will use to perform file transfers. 67 | # Currently this applies only to files whose underlying storage is AWS S3. 68 | # max_threads=16 69 | -------------------------------------------------------------------------------- /synapseclient/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib.resources 2 | import json 3 | 4 | import requests # ensure user-agent is set to track Synapse Python client usage 5 | 6 | from .activity import Activity 7 | from .annotations import Annotations 8 | 9 | # public APIs 10 | from .client import AUTHENTICATED_USERS, PUBLIC, Synapse, login 11 | from .core.models.permission import Permissions 12 | from .core.version_check import check_for_updates, release_notes 13 | from .entity import DockerRepository, Entity, File, Folder, Link, Project 14 | from .evaluation import Evaluation, Submission, SubmissionStatus 15 | from .table import ( 16 | Column, 17 | Dataset, 18 | EntityViewSchema, 19 | EntityViewType, 20 | MaterializedViewSchema, 21 | PartialRowset, 22 | Row, 23 | RowSet, 24 | Schema, 25 | SubmissionViewSchema, 26 | Table, 27 | as_table_columns, 28 | build_table, 29 | delete_rows, 30 | ) 31 | from .team import Team, TeamMember, UserGroupHeader, UserProfile 32 | from .wiki import Wiki 33 | 34 | ref = importlib.resources.files(__name__).joinpath("synapsePythonClient") 35 | with ref.open("r") as fp: 36 | __version__ = json.load(fp)["latestVersion"] 37 | 38 | __all__ = [ 39 | # objects 40 | "Synapse", 41 | "Activity", 42 | "Entity", 43 | "Project", 44 | "Folder", 45 | "File", 46 | "Link", 47 | "DockerRepository", 48 | "Evaluation", 49 | "Submission", 50 | "SubmissionStatus", 51 | "Schema", 52 | "EntityViewSchema", 53 | "Column", 54 | "Row", 55 | "RowSet", 56 | "Table", 57 | "PartialRowset", 58 | "Team", 59 | "UserProfile", 60 | "UserGroupHeader", 61 | "TeamMember", 62 | "Wiki", 63 | "Annotations", 64 | "SubmissionViewSchema", 65 | "MaterializedViewSchema", 66 | "Dataset", 67 | "Permissions", 68 | # functions 69 | "login", 70 | "build_table", 71 | "delete_rows", 72 | "as_table_columns", 73 | "check_for_updates", 74 | "release_notes", 75 | # enum 76 | "EntityViewType", 77 | # constants 78 | "PUBLIC", 79 | "AUTHENTICATED_USERS", 80 | ] 81 | 82 | USER_AGENT = { 83 | "User-Agent": "synapseclient/%s %s" 84 | % (__version__, requests.utils.default_user_agent()) 85 | } 86 | 87 | USER_AGENT_COMMAND_LINE = { 88 | "User-Agent": "synapsecommandlineclient/%s %s" 89 | % (__version__, requests.utils.default_user_agent()) 90 | } 91 | 92 | # patch logging 93 | from .core import logging_setup # noqa 94 | 95 | # patch json 96 | from .core.models import custom_json # noqa 97 | -------------------------------------------------------------------------------- /synapseclient/api/__init__.py: -------------------------------------------------------------------------------- 1 | # These are all of the models that are used by the Synapse client. 2 | from .agent_services import ( 3 | get_agent, 4 | get_session, 5 | get_trace, 6 | register_agent, 7 | start_session, 8 | update_session, 9 | ) 10 | from .annotations import set_annotations, set_annotations_async 11 | from .configuration_services import ( 12 | get_client_authenticated_s3_profile, 13 | get_config_authentication, 14 | get_config_file, 15 | get_config_section_dict, 16 | get_transfer_config, 17 | ) 18 | from .entity_bundle_services_v2 import ( 19 | get_entity_id_bundle2, 20 | get_entity_id_version_bundle2, 21 | post_entity_bundle2_create, 22 | put_entity_id_bundle2, 23 | ) 24 | from .entity_factory import get_from_entity_factory 25 | from .entity_services import ( 26 | create_access_requirements_if_none, 27 | delete_entity, 28 | delete_entity_acl, 29 | delete_entity_generated_by, 30 | get_entities_by_md5, 31 | get_entity, 32 | get_entity_path, 33 | get_upload_destination, 34 | get_upload_destination_location, 35 | post_entity, 36 | put_entity, 37 | ) 38 | from .file_services import ( 39 | AddPartResponse, 40 | get_file_handle, 41 | get_file_handle_for_download, 42 | get_file_handle_for_download_async, 43 | post_external_filehandle, 44 | post_external_object_store_filehandle, 45 | post_external_s3_file_handle, 46 | post_file_multipart, 47 | post_file_multipart_presigned_urls, 48 | put_file_multipart_add, 49 | put_file_multipart_complete, 50 | ) 51 | from .table_services import ( 52 | ViewEntityType, 53 | ViewTypeMask, 54 | get_columns, 55 | get_default_columns, 56 | post_columns, 57 | ) 58 | 59 | __all__ = [ 60 | # annotations 61 | "set_annotations", 62 | "set_annotations_async", 63 | "get_entity_id_bundle2", 64 | "get_entity_id_version_bundle2", 65 | "post_entity_bundle2_create", 66 | "put_entity_id_bundle2", 67 | # file_services 68 | "post_file_multipart", 69 | "put_file_multipart_add", 70 | "put_file_multipart_complete", 71 | "post_external_object_store_filehandle", 72 | "post_external_s3_file_handle", 73 | "get_file_handle", 74 | "post_external_filehandle", 75 | "post_file_multipart_presigned_urls", 76 | "put_file_multipart_add", 77 | "AddPartResponse", 78 | "get_file_handle_for_download_async", 79 | "get_file_handle_for_download", 80 | # entity_services 81 | "get_entity", 82 | "put_entity", 83 | "post_entity", 84 | "delete_entity", 85 | "delete_entity_acl", 86 | "get_upload_destination", 87 | "get_upload_destination_location", 88 | "create_access_requirements_if_none", 89 | "delete_entity_generated_by", 90 | "get_entity_path", 91 | "get_entities_by_md5", 92 | # configuration_services 93 | "get_config_file", 94 | "get_config_section_dict", 95 | "get_config_authentication", 96 | "get_client_authenticated_s3_profile", 97 | "get_transfer_config", 98 | # entity_factory 99 | "get_from_entity_factory", 100 | # agent_services 101 | "register_agent", 102 | "get_agent", 103 | "start_session", 104 | "get_session", 105 | "update_session", 106 | "get_trace", 107 | # columns 108 | "get_columns", 109 | "post_columns", 110 | "get_default_columns", 111 | "ViewTypeMask", 112 | "ViewEntityType", 113 | ] 114 | -------------------------------------------------------------------------------- /synapseclient/api/annotations.py: -------------------------------------------------------------------------------- 1 | """ 2 | The purpose of this module is to provide any functions that are needed to interact with 3 | annotations that are not cleanly provided by the synapseclient library. 4 | """ 5 | 6 | import json 7 | from dataclasses import asdict 8 | from typing import TYPE_CHECKING, Optional 9 | 10 | from synapseclient.annotations import _convert_to_annotations_list 11 | 12 | if TYPE_CHECKING: 13 | from synapseclient import Synapse 14 | from synapseclient.models import Annotations 15 | 16 | 17 | def set_annotations( 18 | annotations: "Annotations", 19 | *, 20 | synapse_client: Optional["Synapse"] = None, 21 | ): 22 | """Call to synapse and set the annotations for the given input. 23 | 24 | Arguments: 25 | annotations: The annotations to set. This is expected to have the id, etag, 26 | and annotations filled in. 27 | synapse_client: If not passed in and caching was not disabled by 28 | `Synapse.allow_client_caching(False)` this will use the last created 29 | instance from the Synapse class constructor. 30 | 31 | Returns: The annotations set in Synapse. 32 | """ 33 | annotations_dict = asdict(annotations) 34 | 35 | synapse_annotations = _convert_to_annotations_list( 36 | annotations_dict["annotations"] or {} 37 | ) 38 | from synapseclient import Synapse 39 | 40 | return Synapse.get_client(synapse_client=synapse_client).restPUT( 41 | f"/entity/{annotations.id}/annotations2", 42 | body=json.dumps( 43 | { 44 | "id": annotations.id, 45 | "etag": annotations.etag, 46 | "annotations": synapse_annotations, 47 | } 48 | ), 49 | ) 50 | 51 | 52 | async def set_annotations_async( 53 | annotations: "Annotations", 54 | *, 55 | synapse_client: Optional["Synapse"] = None, 56 | ): 57 | """Call to synapse and set the annotations for the given input. 58 | 59 | Arguments: 60 | annotations: The annotations to set. This is expected to have the id, etag, 61 | and annotations filled in. 62 | synapse_client: If not passed in and caching was not disabled by 63 | `Synapse.allow_client_caching(False)` this will use the last created 64 | instance from the Synapse class constructor. 65 | 66 | Returns: The annotations set in Synapse. 67 | """ 68 | annotations_dict = asdict(annotations) 69 | 70 | synapse_annotations = _convert_to_annotations_list( 71 | annotations_dict["annotations"] or {} 72 | ) 73 | from synapseclient import Synapse 74 | 75 | return await Synapse.get_client(synapse_client=synapse_client).rest_put_async( 76 | f"/entity/{annotations.id}/annotations2", 77 | body=json.dumps( 78 | { 79 | "id": annotations.id, 80 | "etag": annotations.etag, 81 | "annotations": synapse_annotations, 82 | } 83 | ), 84 | ) 85 | -------------------------------------------------------------------------------- /synapseclient/core/__init__.py: -------------------------------------------------------------------------------- 1 | # These are exposed functions and objects from the `synapseclient.core` package. 2 | # However, these functions and objects are not public APIs for the Synapse Python client. 3 | # The Synapse Engineering team is free to change their signatures and implementations anytime. 4 | # Please use them at your own risk. 5 | -------------------------------------------------------------------------------- /synapseclient/core/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Store configuration for the synapseclient package. 3 | """ 4 | 5 | single_threaded = False 6 | -------------------------------------------------------------------------------- /synapseclient/core/constants/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/synapseclient/core/constants/__init__.py -------------------------------------------------------------------------------- /synapseclient/core/constants/concrete_types.py: -------------------------------------------------------------------------------- 1 | """ 2 | Constant variables for Synapse's concreteType 3 | """ 4 | 5 | # Concrete types for StorageLocationSettings 6 | SYNAPSE_S3_STORAGE_LOCATION_SETTING = ( 7 | "org.sagebionetworks.repo.model.project.S3StorageLocationSetting" 8 | ) 9 | EXTERNAL_S3_STORAGE_LOCATION_SETTING = ( 10 | "org.sagebionetworks.repo.model.project.ExternalS3StorageLocationSetting" 11 | ) 12 | # EXTERNAL_GCP_STORAGE_LOCATION_SETTING = 'org.sagebionetworks.repo.model.project.ExternalGoogleCloudStorageLocationSetting' # noqa: E501 13 | 14 | # Concrete types for UploadDestinations 15 | SYNAPSE_S3_UPLOAD_DESTINATION = ( 16 | "org.sagebionetworks.repo.model.file.S3UploadDestination" 17 | ) 18 | EXTERNAL_UPLOAD_DESTINATION = ( 19 | "org.sagebionetworks.repo.model.file.ExternalUploadDestination" 20 | ) 21 | EXTERNAL_S3_UPLOAD_DESTINATION = ( 22 | "org.sagebionetworks.repo.model.file.ExternalS3UploadDestination" 23 | ) 24 | EXTERNAL_GCP_UPLOAD_DESTINATION = ( 25 | "org.sagebionetworks.repo.model.file.ExternalGoogleCloudUploadDestination" 26 | ) 27 | EXTERNAL_OBJECT_STORE_UPLOAD_DESTINATION = ( 28 | "org.sagebionetworks.repo.model.file.ExternalObjectStoreUploadDestination" 29 | ) 30 | 31 | # Concrete types for FileHandles 32 | EXTERNAL_OBJECT_STORE_FILE_HANDLE = ( 33 | "org.sagebionetworks.repo.model.file.ExternalObjectStoreFileHandle" 34 | ) 35 | EXTERNAL_FILE_HANDLE = "org.sagebionetworks.repo.model.file.ExternalFileHandle" 36 | S3_FILE_HANDLE = "org.sagebionetworks.repo.model.file.S3FileHandle" 37 | 38 | # Concrete types for Tables 39 | ROW_REFERENCE_SET_RESULTS = ( 40 | "org.sagebionetworks.repo.model.table.RowReferenceSetResults" 41 | ) 42 | ENTITY_UPDATE_RESULTS = "org.sagebionetworks.repo.model.table.EntityUpdateResults" 43 | TABLE_SCHEMA_CHANGE_RESPONSE = ( 44 | "org.sagebionetworks.repo.model.table.TableSchemaChangeResponse" 45 | ) 46 | TABLE_SCHEMA_CHANGE_REQUEST = ( 47 | "org.sagebionetworks.repo.model.table.TableSchemaChangeRequest" 48 | ) 49 | TABLE_UPDATE_TRANSACTION_REQUEST = ( 50 | "org.sagebionetworks.repo.model.table.TableUpdateTransactionRequest" 51 | ) 52 | UPLOAD_TO_TABLE_REQUEST = "org.sagebionetworks.repo.model.table.UploadToTableRequest" 53 | UPLOAD_TO_TABLE_RESULT = "org.sagebionetworks.repo.model.table.UploadToTableResult" 54 | 55 | PARTIAL_ROW_SET = "org.sagebionetworks.repo.model.table.PartialRowSet" 56 | APPENDABLE_ROWSET_REQUEST = ( 57 | "org.sagebionetworks.repo.model.table.AppendableRowSetRequest" 58 | ) 59 | 60 | COLUMN_MODEL = "org.sagebionetworks.repo.model.table.ColumnModel" 61 | COLUMN_CHANGE = "org.sagebionetworks.repo.model.table.ColumnChange" 62 | 63 | # EntityTypes 64 | FILE_ENTITY = "org.sagebionetworks.repo.model.FileEntity" 65 | FOLDER_ENTITY = "org.sagebionetworks.repo.model.Folder" 66 | LINK_ENTITY = "org.sagebionetworks.repo.model.Link" 67 | PROJECT_ENTITY = "org.sagebionetworks.repo.model.Project" 68 | TABLE_ENTITY = "org.sagebionetworks.repo.model.table.TableEntity" 69 | DATASET_ENTITY = "org.sagebionetworks.repo.model.table.Dataset" 70 | DATASET_COLLECTION_ENTITY = "org.sagebionetworks.repo.model.table.DatasetCollection" 71 | ENTITY_VIEW = "org.sagebionetworks.repo.model.table.EntityView" 72 | MATERIALIZED_VIEW = "org.sagebionetworks.repo.model.table.MaterializedView" 73 | SUBMISSION_VIEW = "org.sagebionetworks.repo.model.table.SubmissionView" 74 | VIRTUAL_TABLE = "org.sagebionetworks.repo.model.table.VirtualTable" 75 | 76 | # upload requests 77 | MULTIPART_UPLOAD_REQUEST = "org.sagebionetworks.repo.model.file.MultipartUploadRequest" 78 | MULTIPART_UPLOAD_COPY_REQUEST = ( 79 | "org.sagebionetworks.repo.model.file.MultipartUploadCopyRequest" 80 | ) 81 | 82 | # Activity/Provenance 83 | USED_URL = "org.sagebionetworks.repo.model.provenance.UsedURL" 84 | USED_ENTITY = "org.sagebionetworks.repo.model.provenance.UsedEntity" 85 | 86 | # Agent 87 | AGENT_CHAT_REQUEST = "org.sagebionetworks.repo.model.agent.AgentChatRequest" 88 | -------------------------------------------------------------------------------- /synapseclient/core/constants/config_file_constants.py: -------------------------------------------------------------------------------- 1 | AUTHENTICATION_SECTION_NAME = "authentication" 2 | -------------------------------------------------------------------------------- /synapseclient/core/constants/limits.py: -------------------------------------------------------------------------------- 1 | MAX_FILE_HANDLE_PER_COPY_REQUEST = ( 2 | 100 # The maximum number of FilesHandles that can be copied in a single request 3 | ) 4 | -------------------------------------------------------------------------------- /synapseclient/core/constants/method_flags.py: -------------------------------------------------------------------------------- 1 | """Constants for any methods that have flags that are not expressed through other 2 | means like Enums or other constants.""" 3 | 4 | COLLISION_OVERWRITE_LOCAL = "overwrite.local" 5 | COLLISION_KEEP_LOCAL = "keep.local" 6 | COLLISION_KEEP_BOTH = "keep.both" 7 | -------------------------------------------------------------------------------- /synapseclient/core/credentials/__init__.py: -------------------------------------------------------------------------------- 1 | from .cred_data import UserLoginArgs # noqa 2 | from .credential_provider import get_default_credential_chain # noqa 3 | -------------------------------------------------------------------------------- /synapseclient/core/download/__init__.py: -------------------------------------------------------------------------------- 1 | """Functions related to downloading files from Synapse.""" 2 | 3 | from .download_async import ( 4 | SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE, 5 | DownloadRequest, 6 | PresignedUrlInfo, 7 | PresignedUrlProvider, 8 | _MultithreadedDownloader, 9 | _pre_signed_url_expiration_time, 10 | download_file, 11 | ) 12 | from .download_functions import ( 13 | download_by_file_handle, 14 | download_file_entity, 15 | download_file_entity_model, 16 | download_from_url, 17 | download_from_url_multi_threaded, 18 | ensure_download_location_is_directory, 19 | ) 20 | 21 | __all__ = [ 22 | # download_functions 23 | "download_file_entity", 24 | "download_file_entity_model", 25 | "ensure_download_location_is_directory", 26 | "download_by_file_handle", 27 | "download_from_url", 28 | "download_from_url_multi_threaded", 29 | # download_async 30 | "DownloadRequest", 31 | "download_file", 32 | "SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE", 33 | "PresignedUrlInfo", 34 | "PresignedUrlProvider", 35 | "_MultithreadedDownloader", 36 | "_pre_signed_url_expiration_time", 37 | ] 38 | -------------------------------------------------------------------------------- /synapseclient/core/dozer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Sep 21, 2017 3 | 4 | @author: bhoff 5 | 6 | sleep while checking registered _listeners 7 | """ 8 | import time 9 | 10 | from opentelemetry import trace 11 | 12 | tracer = trace.get_tracer("synapseclient") 13 | 14 | _listeners = [] 15 | 16 | 17 | def add_listener(listener): 18 | if not callable(listener): 19 | raise ValueError("listener is not callable") 20 | _listeners.append(listener) 21 | 22 | 23 | def clear_listeners(): 24 | del _listeners[:] 25 | 26 | 27 | def doze( 28 | secs: float, 29 | listener_check_interval_secs: float = 0.1, 30 | trace_span_name: str = "doze", 31 | ) -> None: 32 | """Sleep for a given number of seconds while checking registered listeners. 33 | 34 | Arguments: 35 | secs: the number of seconds to sleep 36 | listener_check_interval_secs: the interval at which to check the listeners 37 | trace_span_name: the name of the trace span 38 | """ 39 | with tracer.start_as_current_span(name=trace_span_name): 40 | end_time = time.time() + secs 41 | while time.time() < end_time: 42 | for listener in _listeners: 43 | listener() 44 | time.sleep(listener_check_interval_secs) 45 | -------------------------------------------------------------------------------- /synapseclient/core/logging_setup.py: -------------------------------------------------------------------------------- 1 | ################ 2 | # logging setup 3 | ################ 4 | 5 | 6 | import logging 7 | import logging.config as logging_config 8 | 9 | logging.captureWarnings(True) 10 | logging.getLogger("requests").setLevel(logging.WARNING) 11 | 12 | DEBUG_LOGGER_NAME = "synapseclient_debug" 13 | DEFAULT_LOGGER_NAME = "synapseclient_default" 14 | SILENT_LOGGER_NAME = "synapseclient_silent" 15 | 16 | 17 | class LoggingInfoOnlyFilter(logging.Filter): 18 | def filter(self, record): 19 | return record.levelno == logging.INFO 20 | 21 | 22 | class LoggingIgnoreInfoFilter(logging.Filter): 23 | def filter(self, record): 24 | return record.levelno != logging.INFO 25 | 26 | 27 | #################################################### 28 | # logging levels from high to low: 29 | # CRITICAL 30 | # ERROR 31 | # WARNING 32 | # INFO 33 | # DEBUG 34 | # NOTSET 35 | # 36 | # see https://docs.python.org/2/library/logging.html 37 | ###################################################### 38 | # TODO: debug file also or only write errors to log? 39 | logging_config.dictConfig( 40 | { 41 | "version": 1, 42 | "disable_existing_loggers": False, 43 | "formatters": { 44 | "debug_format": { 45 | "format": "%(asctime)s [%(module)s:%(lineno)d - %(levelname)s]: %(message)s" 46 | }, 47 | "brief_format": {"format": "%(message)s"}, 48 | "warning_format": {"format": "[%(levelname)s] %(message)s"}, 49 | }, 50 | "filters": { 51 | "info_only": {"()": LoggingInfoOnlyFilter}, 52 | "ignore_info": {"()": LoggingIgnoreInfoFilter}, 53 | }, 54 | "handlers": { 55 | "info_only_stdout": { 56 | "level": "INFO", 57 | "class": "logging.StreamHandler", 58 | "formatter": "brief_format", 59 | "stream": "ext://sys.stdout", 60 | "filters": ["info_only"], 61 | }, 62 | "debug_stderr": { 63 | "level": "DEBUG", 64 | "class": "logging.StreamHandler", 65 | "formatter": "debug_format", 66 | "stream": "ext://sys.stderr", 67 | }, 68 | "warning_stderr": { 69 | "level": "WARNING", 70 | "class": "logging.StreamHandler", 71 | "formatter": "warning_format", 72 | "stream": "ext://sys.stderr", 73 | }, 74 | # , 75 | # "error_to_file": { 76 | # "class": "logging.handlers.RotatingFileHandler", 77 | # "level": "ERROR", 78 | # "formatter": "debug_format", 79 | # "filename": _errlog_path, 80 | # "maxBytes": 10485760, #10 MB 81 | # "backupCount": 15, 82 | # "encoding": "utf8" 83 | # } 84 | }, 85 | "loggers": { 86 | DEFAULT_LOGGER_NAME: { 87 | # TODO: add 'error_to_file' back in after we find solution for multi process logging 88 | "handlers": ["info_only_stdout", "warning_stderr"], 89 | "level": "INFO", 90 | "propagate": True, 91 | }, 92 | DEBUG_LOGGER_NAME: { 93 | "handlers": ["info_only_stdout", "debug_stderr"], 94 | "level": "DEBUG", 95 | "propagate": True, 96 | }, 97 | SILENT_LOGGER_NAME: {"handlers": [], "level": "INFO", "propagate": False}, 98 | # "httpx": { 99 | # "handlers": ["debug_stderr"], 100 | # "level": "DEBUG", 101 | # "propagate": True, 102 | # }, 103 | # "httpcore": { 104 | # "handlers": ["debug_stderr"], 105 | # "level": "DEBUG", 106 | # "propagate": True, 107 | # }, 108 | }, 109 | } 110 | ) 111 | -------------------------------------------------------------------------------- /synapseclient/core/models/__init__.py: -------------------------------------------------------------------------------- 1 | # These are exposed functions and objects from the `synapseclient.core.models` package. 2 | # However, these functions and objects are not public APIs for the Synapse Python client. 3 | # The Synapse Engineering team is free to change their signatures and implementations anytime. 4 | # Please use them at your own risk. 5 | 6 | from .dict_object import DictObject # noqa 7 | -------------------------------------------------------------------------------- /synapseclient/core/models/custom_json.py: -------------------------------------------------------------------------------- 1 | """ 2 | When imported, monkey-patches the 'json' module's encoder with a custom json encoding function. 3 | """ 4 | import datetime 5 | import json 6 | 7 | from synapseclient.core.utils import datetime_to_iso 8 | 9 | 10 | # monkey-patching JSONEncoder from 11 | # https://stackoverflow.com/questions/18478287/making-object-json-serializable-with-regular-encoder 12 | def _json_encoder(self, obj): 13 | if isinstance(obj, datetime.datetime): 14 | # backend takes date string format of "yy-M-d H:m:s.SSS" with the time zone being UTC 15 | return datetime_to_iso(obj, sep=" ").replace("Z", "") 16 | 17 | else: 18 | return getattr(obj.__class__, "to_json", _json_encoder.default)(obj) 19 | 20 | 21 | _json_encoder.default = json.JSONEncoder().default # Save unmodified default. 22 | json.JSONEncoder.default = _json_encoder # replacement 23 | -------------------------------------------------------------------------------- /synapseclient/core/models/dict_object.py: -------------------------------------------------------------------------------- 1 | # 2 | # Represent user-defined annotations on a synapse entity 3 | # chris.bare@sagebase.org 4 | ############################################################ 5 | 6 | import collections.abc 7 | import json 8 | 9 | 10 | class DictObject(dict): 11 | @classmethod 12 | def getByNameURI(cls, name): 13 | print("%s can't be retrieved by name" % cls) 14 | raise ValueError 15 | 16 | def __init__(self, *args, **kwargs): 17 | self.__dict__ = self 18 | for arg in args: 19 | if isinstance(arg, collections.abc.Mapping): 20 | self.__dict__.update(arg) 21 | self.__dict__.update(kwargs) 22 | 23 | def __str__(self): 24 | return json.dumps(self, sort_keys=True, indent=2) 25 | 26 | def json(self, ensure_ascii=True): 27 | return json.dumps(self, sort_keys=True, indent=2, ensure_ascii=ensure_ascii) 28 | -------------------------------------------------------------------------------- /synapseclient/core/multithread_download/__init__.py: -------------------------------------------------------------------------------- 1 | from .download_threads import ( 2 | SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE, 3 | DownloadRequest, 4 | download_file, 5 | shared_executor, 6 | ) 7 | 8 | __all__ = [ 9 | "DownloadRequest", 10 | "download_file", 11 | "shared_executor", 12 | "SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE", 13 | ] 14 | -------------------------------------------------------------------------------- /synapseclient/core/pool_provider.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides access to thread pools to use in concurrent work. 3 | Both a multiprocessing style ThreadPool and an Executor based pool 4 | are available, Executors should be preferred for new work as it provides 5 | a more modern interface. 6 | 7 | To use these wrappers for single thread environment, set the following: 8 | 9 | synapseclient.config.single_threaded = True 10 | """ 11 | 12 | import multiprocessing 13 | import multiprocessing.dummy 14 | from concurrent.futures import Executor, Future, ThreadPoolExecutor 15 | 16 | from . import config 17 | 18 | # +4 matches ThreadPoolExecutor, at least 5 threads for I/O bound tasks 19 | # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor 20 | DEFAULT_NUM_THREADS = multiprocessing.cpu_count() + 4 21 | 22 | 23 | class SingleThreadPool: 24 | def map(self, func, iterable): 25 | for item in iterable: 26 | func(item) 27 | 28 | def terminate(self): 29 | pass 30 | 31 | 32 | class SingleThreadExecutor(Executor): 33 | def __init__(self): 34 | self._shutdown = False 35 | 36 | def submit(self, fn, *args, **kwargs): 37 | if self._shutdown: 38 | raise RuntimeError("cannot schedule new futures after shutdown") 39 | 40 | f = Future() 41 | try: 42 | result = fn(*args, **kwargs) 43 | except BaseException as e: 44 | f.set_exception(e) 45 | else: 46 | f.set_result(result) 47 | 48 | return f 49 | 50 | 51 | class FakeLock: 52 | def __enter__(self): 53 | pass 54 | 55 | def __exit__(self, type, value, traceback): 56 | pass 57 | 58 | 59 | class SingleValue: 60 | value = None 61 | 62 | def __init__(self, type, value): 63 | self.value = value 64 | 65 | def get_lock(self): 66 | return FakeLock() 67 | 68 | 69 | def get_pool(): 70 | if config.single_threaded: 71 | return SingleThreadPool() 72 | else: 73 | return multiprocessing.dummy.Pool(DEFAULT_NUM_THREADS) 74 | 75 | 76 | def get_executor(thread_count=DEFAULT_NUM_THREADS): 77 | """ 78 | Provides an Executor as defined by the client config suitable 79 | for running tasks work as defined by the client config. 80 | 81 | :param thread_count: number of concurrent threads 82 | 83 | :return: an Executor 84 | """ 85 | if config.single_threaded: 86 | return SingleThreadExecutor() 87 | else: 88 | return ThreadPoolExecutor(max_workers=thread_count) 89 | 90 | 91 | def get_value(type, value): 92 | if config.single_threaded: 93 | return SingleValue(type, value) 94 | else: 95 | return multiprocessing.Value(type, value) 96 | -------------------------------------------------------------------------------- /synapseclient/core/upload/__init__.py: -------------------------------------------------------------------------------- 1 | # These are exposed functions and objects from the `synapseclient.core.upload` package. 2 | # However, these functions and objects are not public APIs for the Synapse Python client. 3 | # The Synapse Engineering team is free to change their signatures and implementations anytime. 4 | # Please use them at your own risk. 5 | 6 | from .multipart_upload import ( 7 | multipart_copy, 8 | multipart_upload_file, 9 | multipart_upload_string, 10 | ) 11 | from .upload_functions import upload_file_handle, upload_synapse_s3 12 | 13 | __all__ = [ 14 | "multipart_copy", 15 | "multipart_upload_file", 16 | "multipart_upload_string", 17 | "upload_file_handle", 18 | "upload_synapse_s3", 19 | ] 20 | -------------------------------------------------------------------------------- /synapseclient/models/__init__.py: -------------------------------------------------------------------------------- 1 | # These are all of the models that are used by the Synapse client. 2 | from synapseclient.models.activity import Activity, UsedEntity, UsedURL 3 | from synapseclient.models.agent import ( 4 | Agent, 5 | AgentPrompt, 6 | AgentSession, 7 | AgentSessionAccessLevel, 8 | ) 9 | from synapseclient.models.annotations import Annotations 10 | from synapseclient.models.dataset import Dataset, DatasetCollection, EntityRef 11 | from synapseclient.models.entityview import EntityView, ViewTypeMask 12 | from synapseclient.models.file import File, FileHandle 13 | from synapseclient.models.folder import Folder 14 | from synapseclient.models.materializedview import MaterializedView 15 | from synapseclient.models.mixins.table_components import QueryMixin 16 | from synapseclient.models.project import Project 17 | from synapseclient.models.services import FailureStrategy 18 | from synapseclient.models.submissionview import SubmissionView 19 | from synapseclient.models.table import Table 20 | from synapseclient.models.table_components import ( 21 | AppendableRowSetRequest, 22 | Column, 23 | ColumnChange, 24 | ColumnExpansionStrategy, 25 | ColumnType, 26 | CsvTableDescriptor, 27 | FacetType, 28 | JsonSubColumn, 29 | PartialRow, 30 | PartialRowSet, 31 | QueryResultBundle, 32 | SchemaStorageStrategy, 33 | TableSchemaChangeRequest, 34 | TableUpdateTransaction, 35 | UploadToTableRequest, 36 | ) 37 | from synapseclient.models.team import Team, TeamMember 38 | from synapseclient.models.user import UserPreference, UserProfile 39 | from synapseclient.models.virtualtable import VirtualTable 40 | 41 | __all__ = [ 42 | "Activity", 43 | "UsedURL", 44 | "UsedEntity", 45 | "FailureStrategy", 46 | "File", 47 | "FileHandle", 48 | "Folder", 49 | "Project", 50 | "Annotations", 51 | "Team", 52 | "TeamMember", 53 | "UserProfile", 54 | "UserPreference", 55 | "Agent", 56 | "AgentSession", 57 | "AgentSessionAccessLevel", 58 | "AgentPrompt", 59 | # EntityView models 60 | "EntityView", 61 | "ViewTypeMask", 62 | # Table models 63 | "SchemaStorageStrategy", 64 | "ColumnExpansionStrategy", 65 | "Table", 66 | "Column", 67 | "ColumnType", 68 | "FacetType", 69 | "JsonSubColumn", 70 | "QueryResultBundle", 71 | "query_async", 72 | "query", 73 | "query_part_mask_async", 74 | "query_part_mask", 75 | "ColumnChange", 76 | "PartialRow", 77 | "PartialRowSet", 78 | "TableSchemaChangeRequest", 79 | "AppendableRowSetRequest", 80 | "UploadToTableRequest", 81 | "TableUpdateTransaction", 82 | "CsvTableDescriptor", 83 | "MaterializedView", 84 | "VirtualTable", 85 | # Dataset models 86 | "Dataset", 87 | "EntityRef", 88 | "DatasetCollection", 89 | # Submission models 90 | "SubmissionView", 91 | ] 92 | 93 | # Static methods to expose as functions 94 | query_async = QueryMixin.query_async 95 | query = QueryMixin.query 96 | query_part_mask_async = QueryMixin.query_part_mask_async 97 | query_part_mask = QueryMixin.query_part_mask 98 | -------------------------------------------------------------------------------- /synapseclient/models/evaluation.py: -------------------------------------------------------------------------------- 1 | # TODO 2 | -------------------------------------------------------------------------------- /synapseclient/models/mixins/__init__.py: -------------------------------------------------------------------------------- 1 | """References to the mixins that are used in the Synapse models.""" 2 | 3 | from synapseclient.models.mixins.access_control import AccessControllable 4 | from synapseclient.models.mixins.asynchronous_job import AsynchronousCommunicator 5 | from synapseclient.models.mixins.storable_container import StorableContainer 6 | 7 | __all__ = [ 8 | "AccessControllable", 9 | "StorableContainer", 10 | "AsynchronousCommunicator", 11 | ] 12 | -------------------------------------------------------------------------------- /synapseclient/models/protocols/__init__.py: -------------------------------------------------------------------------------- 1 | # These are all of the protocols that are used by the Synapse client. 2 | -------------------------------------------------------------------------------- /synapseclient/models/protocols/annotations_protocol.py: -------------------------------------------------------------------------------- 1 | """Protocol for the specific methods of this class that have synchronous counterparts 2 | generated at runtime.""" 3 | 4 | from typing import TYPE_CHECKING, Optional, Protocol 5 | 6 | from synapseclient import Synapse 7 | 8 | if TYPE_CHECKING: 9 | from synapseclient.models import Annotations 10 | 11 | 12 | class AnnotationsSynchronousProtocol(Protocol): 13 | """ 14 | The protocol for methods that are asynchronous but also 15 | have a synchronous counterpart that may also be called. 16 | """ 17 | 18 | def store( 19 | self, 20 | *, 21 | synapse_client: Optional[Synapse] = None, 22 | ) -> "Annotations": 23 | """Storing annotations to synapse. 24 | 25 | Arguments: 26 | synapse_client: If not passed in and caching was not disabled by 27 | `Synapse.allow_client_caching(False)` this will use the last created 28 | instance from the Synapse class constructor. 29 | 30 | Returns: 31 | The stored annotations. 32 | 33 | Raises: 34 | ValueError: If the id or etag are not set. 35 | """ 36 | return self 37 | -------------------------------------------------------------------------------- /synapseclient/models/protocols/user_protocol.py: -------------------------------------------------------------------------------- 1 | """Protocol for the specific methods of this class that have synchronous counterparts 2 | generated at runtime.""" 3 | 4 | from typing import TYPE_CHECKING, Optional, Protocol 5 | 6 | from synapseclient import Synapse 7 | 8 | if TYPE_CHECKING: 9 | from synapseclient.models import UserProfile 10 | 11 | 12 | class UserProfileSynchronousProtocol(Protocol): 13 | """ 14 | The protocol for methods that are asynchronous but also 15 | have a synchronous counterpart that may also be called. 16 | """ 17 | 18 | def get( 19 | self, 20 | *, 21 | synapse_client: Optional[Synapse] = None, 22 | ) -> "UserProfile": 23 | """ 24 | Gets a UserProfile object using its id or username in that order. If an id 25 | and username is not specified this will retrieve the current user's profile. 26 | 27 | Arguments: 28 | synapse_client: If not passed in and caching was not disabled by 29 | `Synapse.allow_client_caching(False)` this will use the last created 30 | instance from the Synapse class constructor. 31 | 32 | Returns: 33 | The UserProfile object. 34 | 35 | """ 36 | return self 37 | 38 | @classmethod 39 | def from_id( 40 | cls, user_id: int, *, synapse_client: Optional[Synapse] = None 41 | ) -> "UserProfile": 42 | """Gets UserProfile object using its integer id. Wrapper for the 43 | [get][synapseclient.models.UserProfile.get] method. 44 | 45 | Arguments: 46 | user_id: The id of the user. 47 | synapse_client: If not passed in and caching was not disabled by 48 | `Synapse.allow_client_caching(False)` this will use the last created 49 | instance from the Synapse class constructor. 50 | 51 | Returns: 52 | The UserProfile object. 53 | """ 54 | from synapseclient.models import UserProfile 55 | 56 | return UserProfile() 57 | 58 | @classmethod 59 | def from_username( 60 | cls, username: str, *, synapse_client: Optional[Synapse] = None 61 | ) -> "UserProfile": 62 | """ 63 | Gets UserProfile object using its string name. Wrapper for the 64 | [get][synapseclient.models.UserProfile.get] method. 65 | 66 | Arguments: 67 | username: A name chosen by the user that uniquely identifies them. 68 | synapse_client: If not passed in and caching was not disabled by 69 | `Synapse.allow_client_caching(False)` this will use the last created 70 | instance from the Synapse class constructor. 71 | 72 | Returns: 73 | The UserProfile object. 74 | """ 75 | from synapseclient.models import UserProfile 76 | 77 | return UserProfile() 78 | 79 | def is_certified( 80 | self, 81 | *, 82 | synapse_client: Optional[Synapse] = None, 83 | ) -> "bool": 84 | """ 85 | Determine whether a user is certified. 86 | 87 | Arguments: 88 | synapse_client: If not passed in and caching was not disabled by 89 | `Synapse.allow_client_caching(False)` this will use the last created 90 | instance from the Synapse class constructor. 91 | 92 | Returns: 93 | True if the user is certified, False otherwise. 94 | 95 | Raises: 96 | ValueError: If id nor username is specified. 97 | """ 98 | return bool() 99 | -------------------------------------------------------------------------------- /synapseclient/models/services/__init__.py: -------------------------------------------------------------------------------- 1 | from synapseclient.models.services.search import get_id 2 | from synapseclient.models.services.storable_entity import store_entity 3 | from synapseclient.models.services.storable_entity_components import ( 4 | FailureStrategy, 5 | store_entity_components, 6 | ) 7 | 8 | __all__ = ["store_entity_components", "store_entity", "FailureStrategy", "get_id"] 9 | -------------------------------------------------------------------------------- /synapseclient/models/services/search.py: -------------------------------------------------------------------------------- 1 | """Functional interface for searching for entities in Synapse.""" 2 | 3 | import asyncio 4 | from typing import TYPE_CHECKING, Optional, Union 5 | 6 | from synapseclient import Synapse 7 | from synapseclient.core.exceptions import SynapseNotFoundError 8 | from synapseclient.models.services.storable_entity_components import FailureStrategy 9 | 10 | if TYPE_CHECKING: 11 | from synapseclient.models import File, Folder, Project 12 | 13 | 14 | async def get_id( 15 | entity: Union["Project", "Folder", "File"], 16 | failure_strategy: Optional[FailureStrategy] = FailureStrategy.RAISE_EXCEPTION, 17 | *, 18 | synapse_client: Optional[Synapse] = None, 19 | ) -> Union[str, None]: 20 | """ 21 | Get the ID of the entity from either the ID field or the name/parent of the entity. 22 | This is a wrapper for the [synapseclient.Synapse.findEntityId][] method that is 23 | used in order to search by name/parent. 24 | 25 | Arguments: 26 | failure_strategy: Determines how to handle failures when getting the entity 27 | from Synapse and an exception occurs. Only RAISE_EXCEPTION and None are 28 | supported. 29 | synapse_client: If not passed in and caching was not disabled by 30 | `Synapse.allow_client_caching(False)` this will use the last created 31 | instance from the Synapse class constructor. 32 | 33 | Returns: 34 | The ID of the entity. 35 | 36 | Raises: 37 | ValueError: If the entity ID or Name and parent is not set. 38 | SynapseNotFoundError: If the entity is not found in Synapse. 39 | """ 40 | can_search = ( 41 | entity.id 42 | or ( 43 | entity.name and (entity.__class__.__name__ == "Project" or entity.parent_id) 44 | ) 45 | ) is not None 46 | if not can_search: 47 | if failure_strategy is None: 48 | return None 49 | raise ValueError("Entity ID or Name/Parent is required") 50 | 51 | loop = asyncio.get_event_loop() 52 | entity_id = entity.id or await loop.run_in_executor( 53 | None, 54 | lambda: Synapse.get_client(synapse_client=synapse_client).findEntityId( 55 | name=entity.name, 56 | parent=entity.parent_id, 57 | ), 58 | ) 59 | 60 | if not entity_id: 61 | if failure_strategy is None: 62 | return None 63 | raise SynapseNotFoundError( 64 | f"{entity.__class__.__name__} [Id: {entity.id}, Name: {entity.name}, " 65 | f"Parent: {entity.parent_id}] not found in Synapse." 66 | ) 67 | entity.id = entity_id 68 | return entity_id 69 | -------------------------------------------------------------------------------- /synapseclient/models/wiki.py: -------------------------------------------------------------------------------- 1 | # TODO 2 | -------------------------------------------------------------------------------- /synapseclient/services/__init__.py: -------------------------------------------------------------------------------- 1 | # These are exposed functions and objects from the `synapseclient.services` package. 2 | 3 | from .json_schema import JsonSchemaService 4 | 5 | __all__ = ["JsonSchemaService"] 6 | -------------------------------------------------------------------------------- /synapseclient/synapsePythonClient: -------------------------------------------------------------------------------- 1 | { 2 | "client": "synapsePythonClient", 3 | "latestVersion": "4.7.0", 4 | "blacklist": [ 5 | "0.0.0", 6 | "0.4.1", 7 | "0.4.0", 8 | "0.3.0", 9 | "0.2.1", 10 | "0.2.0", 11 | "0.1.4" 12 | ], 13 | "releaseNotes": "https://python-docs.synapse.org/news/" 14 | } 15 | -------------------------------------------------------------------------------- /synapseutils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | The ``synapseutils`` package provides both higher level beta functions as well as utilities for interacting with 3 | [Synapse](http://www.synapse.org). The behavior of these functions are subject to change. 4 | 5 | """ 6 | 7 | # flake8: noqa F401 unclear who is using these 8 | from .copy_functions import changeFileMetaData, copy, copyFileHandles, copyWiki 9 | from .describe_functions import describe 10 | from .migrate_functions import index_files_for_migration, migrate_indexed_files 11 | from .monitor import notify_me_async, notifyMe, with_progress_bar 12 | from .sync import generate_sync_manifest, syncFromSynapse, syncToSynapse 13 | from .walk_functions import walk 14 | -------------------------------------------------------------------------------- /test.synapseConfig.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/test.synapseConfig.enc -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/__init__.py -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | QUERY_TIMEOUT_SEC = 25 2 | -------------------------------------------------------------------------------- /tests/integration/synapse_creds.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Inputs 4 | SC_ENDPOINT=$1 # i.e. https://sc.sageit.org 5 | SYNAPSE_PAT=$2 # The Synapse Personal Access Token 6 | 7 | # Endpoints 8 | STS_TOKEN_ENDPOINT="${SC_ENDPOINT}/ststoken" 9 | 10 | # Get Credentials 11 | AWS_STS_CREDS=$(curl --location-trusted --silent -H "Authorization:Bearer ${SYNAPSE_PAT}" ${STS_TOKEN_ENDPOINT}) 12 | 13 | echo ${AWS_STS_CREDS} 14 | -------------------------------------------------------------------------------- /tests/integration/synapseclient/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/integration/synapseclient/__init__.py -------------------------------------------------------------------------------- /tests/integration/synapseclient/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/integration/synapseclient/core/__init__.py -------------------------------------------------------------------------------- /tests/integration/synapseclient/core/test_REST_request_helpers.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains integration tests for methods in the :py:class:`Synapse` class that act as helpers for making REST 3 | requests to the Synapse backend 4 | """ 5 | 6 | from synapseclient import Column 7 | 8 | 9 | async def test_createColumns(syn): 10 | columns_to_create = [ 11 | Column(name="FirstTestColumn", columnType="INTEGER"), 12 | Column(name="SecondTestColumn", columnType="DOUBLE"), 13 | ] 14 | created_columns = syn.createColumns(columns_to_create) 15 | assert len(columns_to_create) == len(created_columns) 16 | for col_to_create, created_col in zip(columns_to_create, created_columns): 17 | assert "id" in created_col 18 | assert set(col_to_create.items()).issubset(set(created_col.items())) 19 | -------------------------------------------------------------------------------- /tests/integration/synapseclient/core/test_version_check.py: -------------------------------------------------------------------------------- 1 | """Integration tests for version checking""" 2 | 3 | import httpx 4 | from pytest_mock import MockerFixture 5 | 6 | from synapseclient.core.version_check import ( 7 | _PYPI_JSON_URL, 8 | _get_version_info_from_pypi, 9 | version_check, 10 | ) 11 | 12 | 13 | async def test_version_check(mocker: MockerFixture): 14 | """Integration checks for version_check""" 15 | # Should be higher than current version and return true 16 | assert version_check(current_version="999.999.999") 17 | 18 | # Test out of date version 19 | assert not version_check(current_version="0.0.1") 20 | 21 | # Assert httpx.get called when running version_check 22 | spy = mocker.spy(httpx, "get") 23 | version_check() 24 | spy.assert_called_once_with(_PYPI_JSON_URL) 25 | 26 | 27 | def test_get_version_info_from_pypi(): 28 | """Integration test for _get_version_info_from_pypi""" 29 | assert _get_version_info_from_pypi() 30 | -------------------------------------------------------------------------------- /tests/integration/synapseclient/core/upload/Dockerfile_sftp: -------------------------------------------------------------------------------- 1 | # a Docker image with a running SSH service that can be used with the SFTP integration tests. 2 | # A username/password test/test user is available to test with. 3 | 4 | FROM ubuntu:latest 5 | RUN apt update && apt install openssh-server -y 6 | RUN useradd -m -s /bin/sh test 7 | RUN echo 'test:test' | chpasswd 8 | RUN service ssh start 9 | EXPOSE 22 10 | CMD ["/usr/sbin/sshd","-D"] 11 | -------------------------------------------------------------------------------- /tests/integration/synapseclient/core/upload/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/integration/synapseclient/core/upload/__init__.py -------------------------------------------------------------------------------- /tests/integration/synapseclient/models/async/test_user_async.py: -------------------------------------------------------------------------------- 1 | """Integration tests for UserProfile.""" 2 | 3 | from typing import Callable 4 | 5 | import pytest 6 | 7 | from synapseclient import Synapse 8 | from synapseclient.models import UserProfile 9 | 10 | 11 | class TestUser: 12 | """Integration tests for UserProfile.""" 13 | 14 | @pytest.fixture(autouse=True, scope="function") 15 | def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: 16 | self.syn = syn 17 | self.schedule_for_cleanup = schedule_for_cleanup 18 | 19 | async def test_from_id(self) -> None: 20 | # GIVEN our test profile 21 | integration_test_profile = await UserProfile().get_async( 22 | synapse_client=self.syn 23 | ) 24 | 25 | # WHEN we get the profile by ID 26 | profile = await UserProfile.from_id_async(integration_test_profile.id) 27 | 28 | # THEN we expect the profile to be the same as the one we got from the fixture 29 | assert profile == integration_test_profile 30 | 31 | async def test_from_username(self) -> None: 32 | # GIVEN our test profile 33 | integration_test_profile = await UserProfile().get_async( 34 | synapse_client=self.syn 35 | ) 36 | 37 | # WHEN we get the profile by username 38 | profile = await UserProfile.from_username_async( 39 | integration_test_profile.username 40 | ) 41 | 42 | # THEN we expect the profile to be the same as the one we got from the fixture 43 | assert profile == integration_test_profile 44 | 45 | async def test_is_certified_id(self) -> None: 46 | # GIVEN out test profile 47 | integration_test_profile = await UserProfile().get_async( 48 | synapse_client=self.syn 49 | ) 50 | 51 | # AND a copy of the profile 52 | profile_copy = UserProfile(id=integration_test_profile.id) 53 | 54 | # WHEN we check if the profile is certified 55 | is_certified = await profile_copy.is_certified_async() 56 | 57 | # THEN we expect the profile to not be certified 58 | assert is_certified is False 59 | 60 | async def test_is_certified_username(self) -> None: 61 | # GIVEN out test profile 62 | integration_test_profile = await UserProfile().get_async( 63 | synapse_client=self.syn 64 | ) 65 | 66 | # AND a copy of the profile 67 | profile_copy = UserProfile(username=integration_test_profile.username) 68 | 69 | # WHEN we check if the profile is certified 70 | is_certified = await profile_copy.is_certified_async() 71 | 72 | # THEN we expect the profile to not be certified 73 | assert is_certified is False 74 | -------------------------------------------------------------------------------- /tests/integration/synapseclient/models/synchronous/test_user.py: -------------------------------------------------------------------------------- 1 | """Integration tests for UserProfile.""" 2 | 3 | from typing import Callable 4 | 5 | import pytest 6 | 7 | from synapseclient import Synapse 8 | from synapseclient.models import UserProfile 9 | 10 | 11 | class TestUser: 12 | """Integration tests for UserProfile.""" 13 | 14 | @pytest.fixture(autouse=True, scope="function") 15 | def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: 16 | self.syn = syn 17 | self.schedule_for_cleanup = schedule_for_cleanup 18 | 19 | async def test_from_id(self) -> None: 20 | # GIVEN our test profile 21 | integration_test_profile = UserProfile().get(synapse_client=self.syn) 22 | 23 | # WHEN we get the profile by ID 24 | profile = UserProfile.from_id(integration_test_profile.id) 25 | 26 | # THEN we expect the profile to be the same as the one we got from the fixture 27 | assert profile == integration_test_profile 28 | 29 | async def test_from_username(self) -> None: 30 | # GIVEN our test profile 31 | integration_test_profile = UserProfile().get(synapse_client=self.syn) 32 | 33 | # WHEN we get the profile by username 34 | profile = UserProfile.from_username(integration_test_profile.username) 35 | 36 | # THEN we expect the profile to be the same as the one we got from the fixture 37 | assert profile == integration_test_profile 38 | 39 | async def test_is_certified_id(self) -> None: 40 | # GIVEN out test profile 41 | integration_test_profile = UserProfile().get(synapse_client=self.syn) 42 | 43 | # AND a copy of the profile 44 | profile_copy = UserProfile(id=integration_test_profile.id) 45 | 46 | # WHEN we check if the profile is certified 47 | is_certified = profile_copy.is_certified() 48 | 49 | # THEN we expect the profile to not be certified 50 | assert is_certified is False 51 | 52 | async def test_is_certified_username(self) -> None: 53 | # GIVEN out test profile 54 | integration_test_profile = UserProfile().get(synapse_client=self.syn) 55 | 56 | # AND a copy of the profile 57 | profile_copy = UserProfile(username=integration_test_profile.username) 58 | 59 | # WHEN we check if the profile is certified 60 | is_certified = profile_copy.is_certified() 61 | 62 | # THEN we expect the profile to not be certified 63 | assert is_certified is False 64 | -------------------------------------------------------------------------------- /tests/integration/synapseutils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/integration/synapseutils/__init__.py -------------------------------------------------------------------------------- /tests/integration/synapseutils/test_synapseutils_walk.py: -------------------------------------------------------------------------------- 1 | import os 2 | import uuid 3 | 4 | import pytest 5 | from func_timeout import FunctionTimedOut, func_set_timeout 6 | 7 | import synapseclient.core.utils as utils 8 | import synapseutils 9 | from synapseclient import File, Folder, Project 10 | 11 | 12 | async def test_walk(syn, schedule_for_cleanup): 13 | try: 14 | execute_test_walk(syn, schedule_for_cleanup) 15 | except FunctionTimedOut: 16 | syn.logger.warning("test_walk timed out") 17 | pytest.skip("test_walk timed out, skipping test") 18 | 19 | 20 | # When running with multiple threads it can lock up and do nothing until pipeline is killed at 6hrs 21 | @func_set_timeout(120) 22 | def execute_test_walk(syn, schedule_for_cleanup): 23 | walked = [] 24 | firstfile = utils.make_bogus_data_file() 25 | schedule_for_cleanup(firstfile) 26 | project_entity = syn.store(Project(name=str(uuid.uuid4()))) 27 | schedule_for_cleanup(project_entity.id) 28 | folder_entity = syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity)) 29 | schedule_for_cleanup(folder_entity.id) 30 | second_folder = syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity)) 31 | schedule_for_cleanup(second_folder.id) 32 | file_entity = syn.store(File(firstfile, parent=project_entity)) 33 | schedule_for_cleanup(file_entity.id) 34 | 35 | walked.append( 36 | ( 37 | (project_entity.name, project_entity.id), 38 | [ 39 | (folder_entity.name, folder_entity.id), 40 | (second_folder.name, second_folder.id), 41 | ], 42 | [(file_entity.name, file_entity.id)], 43 | ) 44 | ) 45 | 46 | nested_folder = syn.store(Folder(name=str(uuid.uuid4()), parent=folder_entity)) 47 | schedule_for_cleanup(nested_folder.id) 48 | secondfile = utils.make_bogus_data_file() 49 | schedule_for_cleanup(secondfile) 50 | second_file = syn.store(File(secondfile, parent=nested_folder)) 51 | schedule_for_cleanup(second_file.id) 52 | thirdfile = utils.make_bogus_data_file() 53 | schedule_for_cleanup(thirdfile) 54 | third_file = syn.store(File(thirdfile, parent=second_folder)) 55 | schedule_for_cleanup(third_file.id) 56 | 57 | walked.append( 58 | ( 59 | (os.path.join(project_entity.name, folder_entity.name), folder_entity.id), 60 | [(nested_folder.name, nested_folder.id)], 61 | [], 62 | ) 63 | ) 64 | walked.append( 65 | ( 66 | ( 67 | os.path.join( 68 | os.path.join(project_entity.name, folder_entity.name), 69 | nested_folder.name, 70 | ), 71 | nested_folder.id, 72 | ), 73 | [], 74 | [(second_file.name, second_file.id)], 75 | ) 76 | ) 77 | walked.append( 78 | ( 79 | (os.path.join(project_entity.name, second_folder.name), second_folder.id), 80 | [], 81 | [(third_file.name, third_file.id)], 82 | ) 83 | ) 84 | 85 | temp = synapseutils.walk(syn, project_entity.id) 86 | temp = list(temp) 87 | # Must sort the tuples returned, because order matters for the assert 88 | # Folders are returned in a different ordering depending on the name 89 | for i in walked: 90 | for x in i: 91 | if type(x) == list: 92 | x.sort() 93 | for i in temp: 94 | for x in i: 95 | if type(x) == list: 96 | x.sort() 97 | assert i in walked 98 | 99 | temp = synapseutils.walk(syn, second_file.id) 100 | assert list(temp) == [] 101 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | """Utility functions for unit/ingration tests""" 2 | 3 | from typing import Any, Callable, Coroutine 4 | 5 | 6 | def spy_for_async_function( 7 | original_func: Callable[..., Any] 8 | ) -> Callable[..., Coroutine[Any, Any, Any]]: 9 | """This function is used to create a spy for async functions.""" 10 | 11 | async def wrapper(*args, **kwargs): 12 | return await original_func(*args, **kwargs) # Call the original function 13 | 14 | return wrapper 15 | 16 | 17 | def spy_for_function(original_func: Callable[..., Any]) -> Callable[..., Any]: 18 | """This function is used to create a spy for functions.""" 19 | 20 | def wrapper(*args, **kwargs): 21 | return original_func(*args, **kwargs) # Call the original function 22 | 23 | return wrapper 24 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | pytest unit test session level fixtures 3 | """ 4 | 5 | import logging 6 | import os 7 | import platform 8 | import time 9 | import urllib.request 10 | 11 | import pytest 12 | from pytest_socket import SocketBlockedError, disable_socket 13 | 14 | from synapseclient import Synapse 15 | from synapseclient.core.logging_setup import SILENT_LOGGER_NAME 16 | 17 | Synapse.allow_client_caching = False 18 | 19 | 20 | def pytest_runtest_setup(): 21 | """Disable socket connections during unit tests. 22 | 23 | This uses the https://pypi.org/project/pytest-socket/ library for this functionality. 24 | 25 | allow_unix_socket=True is required for async to work. 26 | """ 27 | # This is a work-around because of https://github.com/python/cpython/issues/77589 28 | if platform.system() != "Windows": 29 | disable_socket(allow_unix_socket=True) 30 | 31 | 32 | def test_confirm_connections_blocked(): 33 | """Confirm that socket connections are blocked during unit tests.""" 34 | if platform.system() != "Windows": 35 | with pytest.raises(SocketBlockedError) as cm_ex: 36 | urllib.request.urlopen("http://example.com") 37 | assert "A test tried to use socket.socket." == str(cm_ex.value) 38 | 39 | 40 | @pytest.fixture(autouse=True) 41 | def set_timezone(): 42 | os.environ["TZ"] = "UTC" 43 | if platform.system() != "Windows": 44 | time.tzset() 45 | 46 | 47 | @pytest.fixture(scope="session") 48 | def syn(): 49 | """ 50 | Create a Synapse instance that can be shared by all tests in the session. 51 | """ 52 | syn = Synapse(debug=False, skip_checks=True) 53 | syn.logger = logging.getLogger(SILENT_LOGGER_NAME) 54 | Synapse.set_client(syn) 55 | return syn 56 | -------------------------------------------------------------------------------- /tests/unit/synapseclient/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/unit/synapseclient/__init__.py -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/unit/synapseclient/core/__init__.py -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/credentials/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/unit/synapseclient/core/credentials/__init__.py -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/download/unit_test_download_functions.py: -------------------------------------------------------------------------------- 1 | """Unit tests for synapseclient.core.download.download_functions""" 2 | 3 | import os 4 | from unittest.mock import patch 5 | 6 | import pytest 7 | 8 | from synapseclient import File, Synapse 9 | from synapseclient.core import utils 10 | from synapseclient.core.download import ( 11 | download_file_entity, 12 | ensure_download_location_is_directory, 13 | ) 14 | 15 | 16 | def test_ensure_download_location_is_directory() -> None: 17 | download_location = "/foo/bar/baz" 18 | with patch("synapseclient.core.download.download_functions.os") as mock_os: 19 | mock_os.path.isfile.return_value = False 20 | ensure_download_location_is_directory(download_location) 21 | 22 | mock_os.path.isfile.return_value = True 23 | with pytest.raises(ValueError): 24 | ensure_download_location_is_directory(download_location) 25 | 26 | 27 | async def test_download_file_entity_correct_local_state(syn: Synapse) -> None: 28 | mock_cache_path = utils.normalize_path("/i/will/show/you/the/path/yi.txt") 29 | file_entity = File(parentId="syn123") 30 | file_entity.dataFileHandleId = 123 31 | with patch.object(syn.cache, "get", return_value=mock_cache_path): 32 | await download_file_entity( 33 | download_location=None, 34 | entity=file_entity, 35 | if_collision="overwrite.local", 36 | submission=None, 37 | synapse_client=syn, 38 | ) 39 | assert mock_cache_path == utils.normalize_path(file_entity.path) 40 | assert os.path.dirname(mock_cache_path) == file_entity.cacheDir 41 | assert 1 == len(file_entity.files) 42 | assert os.path.basename(mock_cache_path) == file_entity.files[0] 43 | -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/unit/synapseclient/core/models/__init__.py -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/models/unit_test_DictObject.py: -------------------------------------------------------------------------------- 1 | from synapseclient.core.models.dict_object import DictObject 2 | 3 | 4 | def test_DictObject(): 5 | """Test creation and property access on DictObjects""" 6 | d = DictObject( 7 | {"args_working?": "yes"}, a=123, b="foobar", nerds=["chris", "jen", "janey"] 8 | ) 9 | assert d.a == 123 10 | assert d["a"] == 123 11 | assert d.b == "foobar" 12 | assert d["b"] == "foobar" 13 | assert d.nerds == ["chris", "jen", "janey"] 14 | assert hasattr(d, "nerds") 15 | assert d["nerds"] == ["chris", "jen", "janey"] 16 | assert not hasattr(d, "qwerqwer") 17 | 18 | assert all([key in d.keys() for key in ["args_working?", "a", "b", "nerds"]]) 19 | d.new_key = "new value!" 20 | assert d["new_key"] == "new value!" 21 | -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/models/unit_test_custom_json.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | import time 4 | 5 | 6 | def test_to_json(): 7 | json_str = "{some json str}" 8 | 9 | class JsonTest: 10 | def to_json(self): 11 | return json_str 12 | 13 | assert json_str == json.loads(json.dumps(JsonTest())) 14 | 15 | 16 | def test_datetime_json(): 17 | datetime_obj = datetime.datetime.fromtimestamp(round(time.time(), 3)) 18 | datetime_json_str = json.dumps(datetime_obj) 19 | datetime_from_json = datetime.datetime.strptime( 20 | json.loads(datetime_json_str), "%Y-%m-%d %H:%M:%S.%f" 21 | ) 22 | assert datetime_obj == datetime_from_json 23 | -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/multithread_download/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/unit/synapseclient/core/multithread_download/__init__.py -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/unit_test_cumulative_transfer_progress.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from unittest import mock 3 | 4 | from pytest_mock import MockerFixture 5 | from tqdm import tqdm 6 | 7 | from synapseclient.core import cumulative_transfer_progress, utils 8 | 9 | 10 | @mock.patch.object(utils, "printTransferProgress") 11 | def test_not_configured(mock_utils_print_transfer_progress): 12 | """Verify that if no thread local state is configured that printTransferProgress falls through to utils""" 13 | 14 | args = [5, 10] 15 | kwargs = { 16 | "prefix": "prefix", 17 | "postfix": "postfix", 18 | "isBytes": True, 19 | "dt": 100, 20 | "previouslyTransferred": 200, 21 | } 22 | 23 | cumulative_transfer_progress.printTransferProgress(*args, **kwargs) 24 | mock_utils_print_transfer_progress.assert_called_once_with(*args, **kwargs) 25 | 26 | 27 | @mock.patch.object(cumulative_transfer_progress, "time") 28 | def test_progress(mock_time, mocker: MockerFixture) -> None: 29 | """Verify writing progress via a CumulativeProgress""" 30 | 31 | # mock time for predictability 32 | # first call is in init constructor below, remaining three calls are in print calls to calculate transfer rate 33 | mock_time.time.side_effect = [0, 100, 200, 300] 34 | progress = cumulative_transfer_progress.CumulativeTransferProgress("Testing") 35 | 36 | # two prints for file 1 in a separate thread 37 | # the first halfway through the second print indicates the file is done 38 | 39 | # one print for a second file in this thread showing a file 3/4 done 40 | 41 | args1a = [100, 200] 42 | kwargs1a = { 43 | "prefix": "prefix1", 44 | "postfix": "postfix1", 45 | "isBytes": True, 46 | "dt": 300, 47 | "previouslyTransferred": 0, 48 | } 49 | 50 | args1b = [200, 200] 51 | kwargs1b = { 52 | "prefix": "prefix1", 53 | "postfix": "postfix1", 54 | "isBytes": True, 55 | "dt": 400, 56 | "previouslyTransferred": 0, 57 | } 58 | 59 | args2 = [150, 200] 60 | kwargs2 = { 61 | "prefix": "prefix2", 62 | "postfix": "postfix2", 63 | "isBytes": True, 64 | "dt": 300, 65 | "previouslyTransferred": 0, 66 | } 67 | 68 | def print_completed(): 69 | with progress.accumulate_progress(): 70 | cumulative_transfer_progress.printTransferProgress(*args1a, **kwargs1a) 71 | cumulative_transfer_progress.printTransferProgress(*args1b, **kwargs1b) 72 | 73 | thread = threading.Thread(target=print_completed) 74 | thread.start() 75 | 76 | thread.join() 77 | 78 | spy_tqdm_update = mocker.spy(tqdm, "update") 79 | 80 | with progress.accumulate_progress(): 81 | assert progress is getattr( 82 | cumulative_transfer_progress._thread_local, "cumulative_transfer_progress" 83 | ) 84 | cumulative_transfer_progress.printTransferProgress(*args2, **kwargs2) 85 | 86 | # 150 bytes transferred by this thread 87 | assert 150 == cumulative_transfer_progress._thread_local.thread_transferred 88 | 89 | # total transferred is 350, 200 from the first file in the separate thread, 90 | # 150 from the transfer above from this thread 91 | assert 350 == progress._total_transferred 92 | 93 | assert spy_tqdm_update.called 94 | 95 | # test thread local props cleaned up 96 | assert not hasattr(cumulative_transfer_progress._thread_local, "thread_transferred") 97 | assert not hasattr( 98 | cumulative_transfer_progress._thread_local, "cumulative_transfer_progress" 99 | ) 100 | -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/unit_test_doze.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Sep 21, 2017 3 | 4 | @author: bhoff 5 | """ 6 | import synapseclient.core.dozer as doze 7 | 8 | 9 | def teardown(): 10 | doze.clear_listeners() 11 | 12 | 13 | def test_doze(): 14 | class CounterClass(object): 15 | def __init__(self): 16 | self.val = 0 17 | 18 | def __call__(self): 19 | self.val = self.val + 1 20 | 21 | counter = CounterClass() 22 | 23 | # register Listener 24 | doze.add_listener(counter) 25 | doze.doze(1) # should call counter_inc() about 10 times 26 | assert counter.val > 0 27 | -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/unit_test_lock.py: -------------------------------------------------------------------------------- 1 | import random 2 | import time 3 | from datetime import timedelta 4 | from threading import Thread 5 | 6 | from synapseclient.core.lock import Lock 7 | 8 | 9 | def test_lock(): 10 | user1_lock = Lock("foo", max_age=timedelta(seconds=5)) 11 | user2_lock = Lock("foo", max_age=timedelta(seconds=5)) 12 | 13 | assert user1_lock.acquire() 14 | assert user1_lock.get_age() < 5 15 | assert not user2_lock.acquire() 16 | 17 | user1_lock.release() 18 | 19 | assert user2_lock.acquire() 20 | assert not user1_lock.acquire() 21 | 22 | user2_lock.release() 23 | 24 | 25 | def test_with_lock(): 26 | user1_lock = Lock("foo", max_age=timedelta(seconds=5)) 27 | user2_lock = Lock("foo", max_age=timedelta(seconds=5)) 28 | 29 | with user1_lock: 30 | assert user1_lock.get_age() < 5 31 | assert not user2_lock.acquire() 32 | 33 | with user2_lock: 34 | assert user2_lock.acquire() 35 | assert not user1_lock.acquire() 36 | 37 | 38 | def test_lock_timeout(): 39 | user1_lock = Lock("foo", max_age=timedelta(seconds=1)) 40 | user2_lock = Lock("foo", max_age=timedelta(seconds=1)) 41 | 42 | with user1_lock: 43 | assert user1_lock.held 44 | assert user1_lock.get_age() < 1.0 45 | assert not user2_lock.acquire(break_old_locks=True) 46 | time.sleep(1.1) 47 | assert user1_lock.get_age() > 1.0 48 | assert user2_lock.acquire(break_old_locks=True) 49 | 50 | 51 | # Try to hammer away at the locking mechanism from multiple threads 52 | NUMBER_OF_TIMES_PER_THREAD = 3 53 | 54 | 55 | def do_stuff_with_a_locked_resource(name, event_log): 56 | lock = Lock("foo", max_age=timedelta(seconds=5)) 57 | for i in range(NUMBER_OF_TIMES_PER_THREAD): 58 | with lock: 59 | event_log.append((name, i)) 60 | time.sleep(random.betavariate(2, 5)) 61 | 62 | 63 | def test_multithreaded(): 64 | event_log = [] 65 | 66 | threads = [ 67 | Thread( 68 | target=do_stuff_with_a_locked_resource, args=("thread %d" % i, event_log) 69 | ) 70 | for i in range(4) 71 | ] 72 | 73 | for thread in threads: 74 | thread.start() 75 | 76 | for thread in threads: 77 | thread.join() 78 | 79 | counts = {} 80 | for event in event_log: 81 | counts.setdefault(event[0], set()) 82 | counts[event[0]].add(event[1]) 83 | 84 | for key in counts: 85 | assert counts[key] == set(range(NUMBER_OF_TIMES_PER_THREAD)) 86 | -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/unit_test_pool_provider.py: -------------------------------------------------------------------------------- 1 | from concurrent.futures import ThreadPoolExecutor 2 | from multiprocessing.pool import ThreadPool 3 | from multiprocessing.sharedctypes import Synchronized 4 | from unittest.mock import MagicMock, PropertyMock, call, patch 5 | 6 | import synapseclient 7 | from synapseclient.core.pool_provider import ( 8 | SingleThreadExecutor, 9 | SingleThreadPool, 10 | SingleValue, 11 | get_executor, 12 | get_pool, 13 | get_value, 14 | ) 15 | 16 | 17 | class TestSingleThreadPool: 18 | def setup_method(self) -> None: 19 | pass 20 | 21 | def test_map(self) -> None: 22 | test_func = MagicMock() 23 | pool = SingleThreadPool() 24 | pool.map(test_func, range(0, 10)) 25 | test_func.assert_has_calls(call(x) for x in range(0, 10)) 26 | 27 | 28 | class TestSingleThreadExecutor: 29 | def test_execution(self) -> None: 30 | executor = SingleThreadExecutor() 31 | for i in range(10): 32 | result = executor.submit(lambda: i) 33 | 34 | assert result.done 35 | assert i == result.result() 36 | 37 | 38 | def _patch_config(single_threaded: bool): 39 | return patch.object( 40 | synapseclient.core.config, 41 | "single_threaded", 42 | single_threaded, 43 | ) 44 | 45 | 46 | class TestExecutorProvider: 47 | def test_get_executor_for_single_thread(self) -> None: 48 | with _patch_config(True): 49 | assert isinstance(get_executor(), SingleThreadExecutor) 50 | 51 | def test_get_executor_for_multiple_thread(self) -> None: 52 | with _patch_config(False): 53 | assert isinstance(get_executor(), ThreadPoolExecutor) 54 | 55 | 56 | class TestPoolProvider: 57 | def test_get_pool_for_single_thread(self) -> None: 58 | with _patch_config(True): 59 | assert isinstance(get_pool(), SingleThreadPool) 60 | 61 | def test_get_pool_for_multiple_thread(self) -> None: 62 | with _patch_config(False): 63 | assert isinstance(get_pool(), ThreadPool) 64 | 65 | 66 | class TestGetValue: 67 | @patch.object( 68 | synapseclient.core.config, "single_threaded", PropertyMock(return_value=False) 69 | ) 70 | def test_get_value_for_multiple_thread(self) -> None: 71 | synapseclient.core.config.single_threaded = False 72 | test_value = get_value("d", 500) 73 | type(test_value) 74 | assert isinstance(test_value, Synchronized) 75 | assert test_value.value == 500 76 | 77 | test_value.get_lock() 78 | test_value.value = 900 79 | assert test_value.value == 900 80 | 81 | @patch.object( 82 | synapseclient.core.config, "single_threaded", PropertyMock(return_value=True) 83 | ) 84 | def test_get_value_for_single_thread(self) -> None: 85 | synapseclient.core.config.single_threaded = True 86 | test_value = get_value("d", 500) 87 | assert isinstance(test_value, SingleValue) 88 | test_value.value 89 | assert test_value.value == 500 90 | 91 | test_value.get_lock() 92 | test_value.value = 900 93 | assert test_value.value == 900 94 | -------------------------------------------------------------------------------- /tests/unit/synapseclient/core/upload/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/unit/synapseclient/core/upload/__init__.py -------------------------------------------------------------------------------- /tests/unit/synapseclient/services/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/unit/synapseclient/services/__init__.py -------------------------------------------------------------------------------- /tests/unit/synapseclient/services/unit_test_json_schema.py: -------------------------------------------------------------------------------- 1 | """TODO: Add more tests""" 2 | import pytest 3 | 4 | from synapseclient.services import json_schema 5 | 6 | 7 | def test_json_schema_organization(): 8 | org = json_schema.JsonSchemaOrganization( 9 | name="foo123", 10 | ) 11 | assert org.name == "foo123" 12 | assert org.id is None 13 | assert org.created_on is None 14 | assert org.created_by is None 15 | assert org._json_schemas == dict() 16 | assert org._raw_json_schemas == dict() 17 | 18 | 19 | @pytest.mark.parametrize( 20 | "org_name", 21 | ["foo", "123foo"], 22 | ids=["name too short", "name can't start with number"], 23 | ) 24 | def test_json_schema_organization_bad_name(org_name): 25 | with pytest.raises(ValueError): 26 | json_schema.JsonSchemaOrganization( 27 | name=org_name, 28 | ) 29 | -------------------------------------------------------------------------------- /tests/unit/synapseclient/unit_test_Wiki.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import mock_open, patch 2 | 3 | import pytest 4 | 5 | from synapseclient import Synapse, Wiki 6 | 7 | 8 | def test_Wiki(): 9 | """Test the construction and accessors of Wiki objects.""" 10 | 11 | # Wiki contstuctor only takes certain values 12 | pytest.raises(ValueError, Wiki, title="foo") 13 | 14 | # Construct a wiki and test uri's 15 | Wiki(title="foobar2", markdown="bar", owner={"id": "5"}) 16 | 17 | 18 | if __name__ == "__main__": 19 | test_Wiki() 20 | 21 | 22 | def test_Wiki__with_markdown_file(): 23 | markdown_data = """ 24 | MARK DOWN MARK DOWN MARK DOWN MARK DOWN 25 | AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 26 | adkflajsl;kfjasd;lfkjsal;kfajslkfjasdlkfj 27 | """ 28 | markdown_path = "/somewhere/over/the/rainbow.txt" 29 | with ( 30 | patch( 31 | "synapseclient.wiki.open", mock_open(read_data=markdown_data), create=True 32 | ) as mocked_open, 33 | patch("os.path.isfile", return_value=True), 34 | ): 35 | # method under test 36 | wiki = Wiki(owner="doesn't matter", markdownFile=markdown_path) 37 | 38 | mocked_open.assert_called_once_with(markdown_path, "r") 39 | mocked_open().read.assert_called_once_with() 40 | assert markdown_data == wiki.markdown 41 | 42 | 43 | def test_Wiki__markdown_and_markdownFile_both_defined(): 44 | with pytest.raises(ValueError): 45 | Wiki(owner="doesn't matter", markdown="asdf", markdownFile="~/fakeFile.txt") 46 | 47 | 48 | def test_Wiki__markdown_is_None_markdownFile_defined(): 49 | markdown_path = "/somewhere/over/the/rainbow.txt" 50 | with ( 51 | patch("synapseclient.wiki.open", mock_open(), create=True) as mocked_open, 52 | patch("os.path.isfile", return_value=True), 53 | ): 54 | # method under test 55 | Wiki(owner="doesn't matter", markdownFile=markdown_path) 56 | 57 | mocked_open.assert_called_once_with(markdown_path, "r") 58 | mocked_open().read.assert_called_once_with() 59 | 60 | 61 | def test_Wiki__markdown_defined_markdownFile_is_None(): 62 | markdown = "Somebody once told me the OS was gonna delete me. I'm not the largest file on the disk." 63 | 64 | # method under test 65 | wiki = Wiki(owner="doesn't matter", markdown=markdown) 66 | 67 | assert markdown == wiki.markdown 68 | 69 | 70 | def test_Wiki__markdownFile_path_not_exist(): 71 | # method under test 72 | with pytest.raises(ValueError): 73 | Wiki( 74 | owner="doesn't matter", 75 | markdownFile="/this/is/not/the/file/you/are/looking.for", 76 | ) 77 | 78 | 79 | def test_wiki_with_none_attachments(syn: Synapse): 80 | with patch.object(syn, "restPOST"): 81 | w = Wiki(owner="syn1", markdown="markdown", attachments=None) 82 | syn.store(w) 83 | 84 | 85 | def test_wiki_with_empty_string_parent_wiki_id(syn: Synapse): 86 | with patch.object(syn, "restPOST") as mock_restPOST: 87 | # WHEN a wiki is created with an empty string parentWikiId 88 | w = Wiki(owner="syn1", markdown="markdown", parentWikiId="") 89 | # THEN the parentWikiId is set to None 90 | assert w.parentWikiId is None 91 | # WHEN the wiki is stored 92 | syn.store(w) 93 | # THEN the parentWikiId is set to None in the request 94 | mock_restPOST.assert_called_once_with( 95 | "/entity/syn1/wiki", 96 | '{"markdown": "markdown", "parentWikiId": null, "attachmentFileHandleIds": []}', 97 | ) 98 | -------------------------------------------------------------------------------- /tests/unit/synapseclient/unit_test_activity.py: -------------------------------------------------------------------------------- 1 | from synapseclient.activity import Activity 2 | 3 | 4 | # SYNPY-744 5 | def test_private_getStringList(): 6 | act = Activity() 7 | url_string = "https://github.com/Sage-Bionetworks/ampAdScripts/blob/master/Broad-Rush/migrateROSMAPGenotypesFeb2015.R" 8 | act.used( 9 | [ 10 | { 11 | "wasExecuted": True, 12 | "concreteType": "org.sagebionetworks.repo.model.provenance.UsedURL", 13 | "url": url_string, 14 | } 15 | ] 16 | ) 17 | assert [url_string] == act._getStringList() 18 | -------------------------------------------------------------------------------- /tests/unit/synapseutils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/unit/synapseutils/__init__.py -------------------------------------------------------------------------------- /tests/unit/synapseutils/unit_test_synapseutils_monitor.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, call, patch 2 | 3 | import synapseutils 4 | from synapseutils import notifyMe, with_progress_bar 5 | 6 | 7 | def test_notifyMe__successful_call(syn): 8 | subject = "some message subject" 9 | owner_id = "12434" 10 | user_profile = {"ownerId": owner_id} 11 | with patch.object(syn, "sendMessage") as mocked_send_message, patch.object( 12 | syn, "getUserProfile", return_value=user_profile 13 | ) as mocked_get_user_profile: 14 | mocked_func = MagicMock() 15 | 16 | @notifyMe(syn, messageSubject=subject) 17 | def test_function(): 18 | mocked_func() 19 | 20 | test_function() 21 | mocked_get_user_profile.assert_called_once() 22 | mocked_send_message.assert_called_once_with( 23 | [owner_id], 24 | subject, 25 | messageBody="Call to test_function completed successfully!", 26 | ) 27 | 28 | 29 | def test_notifyMe__exception_thrown_and_retry_fail(syn): 30 | subject = "some message subject" 31 | owner_id = "12434" 32 | user_profile = {"ownerId": owner_id} 33 | with patch.object(syn, "sendMessage") as mocked_send_message, patch.object( 34 | syn, "getUserProfile", return_value=user_profile 35 | ): 36 | mocked_func = MagicMock( 37 | side_effect=[Exception("first time fails"), "second time is Fine"] 38 | ) 39 | 40 | @notifyMe(syn, messageSubject=subject, retries=1) 41 | def test_function(): 42 | mocked_func() 43 | 44 | test_function() 45 | assert 2 == mocked_send_message.call_count 46 | 47 | # call_args_list is a list of tuples, each tuple in the form (args,kwargs) 48 | first_call_args = mocked_send_message.call_args_list[0][0] 49 | first_call_kwargs = mocked_send_message.call_args_list[0][1] 50 | 51 | second_call_args = mocked_send_message.call_args_list[1][0] 52 | second_call_kwargs = mocked_send_message.call_args_list[1][1] 53 | 54 | assert ([owner_id], subject) == first_call_args 55 | assert ( 56 | "Encountered a temporary Failure during upload" 57 | in first_call_kwargs["messageBody"] 58 | ) 59 | 60 | assert ([owner_id], subject) == first_call_args 61 | assert 1 == len(first_call_kwargs) 62 | assert ( 63 | "Encountered a temporary Failure during upload" 64 | in first_call_kwargs["messageBody"] 65 | ) 66 | 67 | assert ([owner_id], subject) == second_call_args 68 | assert 1 == len(second_call_kwargs) 69 | assert ( 70 | "Call to test_function completed successfully!" 71 | == second_call_kwargs["messageBody"] 72 | ) 73 | 74 | 75 | def test_with_progress_bar(): 76 | num_calls = 5 77 | mocked_function = MagicMock() 78 | 79 | progress_func = with_progress_bar(mocked_function, num_calls) 80 | 81 | with patch.object( 82 | synapseutils.monitor, "printTransferProgress" 83 | ) as mocked_progress_print: 84 | for i in range(num_calls): 85 | progress_func(i) 86 | mocked_progress_print.assert_has_calls( 87 | [call(float(i + 1), num_calls, "", "", False) for i in range(num_calls)] 88 | ) 89 | mocked_function.assert_has_calls([call(i) for i in range(num_calls)]) 90 | -------------------------------------------------------------------------------- /tests/unit/test_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/6a4e8581eec36cd69f7c4eafe3b19bfaa8b011bf/tests/unit/test_utils/__init__.py -------------------------------------------------------------------------------- /tests/unit/test_utils/unit_utils.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | 3 | 4 | class StringIOContextManager(StringIO): 5 | """ 6 | A StringIO that can be used as a context manager 7 | """ 8 | 9 | def __enter__(self): 10 | return self 11 | 12 | def __exit__(self, *args): 13 | # reset the pointer to the beginning of the string, so that we can read it again 14 | self.seek(0) 15 | pass 16 | --------------------------------------------------------------------------------