├── recipes ├── __init__.py ├── requirements.txt ├── tests │ ├── requirements-dev.txt │ ├── test_inputs │ │ ├── valid.yml │ │ └── valid2.yml │ └── test_recipes_sync.py ├── sync_recipe_utils.py └── sync_recipes.py ├── solvebio ├── cli │ ├── __init__.py │ ├── tutorial.py │ ├── ipython_init.py │ ├── auth.py │ ├── ipython.py │ ├── credentials.py │ └── tutorial.md ├── contrib │ ├── __init__.py │ ├── streamlit │ │ ├── __init__.py │ │ ├── dev-requirements.txt │ │ ├── streamlit-secure-app-demo.gif │ │ ├── .env.template │ │ ├── README.md │ │ ├── solvebio_auth.py │ │ ├── app_example.py │ │ └── solvebio_streamlit.py │ └── dash │ │ ├── tests │ │ ├── __init__.py │ │ ├── .gitignore │ │ ├── credentials.py │ │ ├── utils.py │ │ ├── test_solvebio_auth_integration.py │ │ └── IntegrationTests.py │ │ ├── js │ │ ├── .babelrc │ │ ├── .jshintrc │ │ ├── README.md │ │ ├── .eslintrc │ │ ├── webpack.config.js │ │ ├── package.json │ │ ├── styles │ │ │ └── application.scss │ │ └── src │ │ │ ├── oauth-redirect-index.react.js │ │ │ └── login-index.react.js │ │ ├── __init__.py │ │ ├── dev-requirements.txt │ │ ├── README.md │ │ ├── tox.ini │ │ ├── .gitignore │ │ ├── usage.py │ │ └── solvebio_dash.py ├── test │ ├── __init__.py │ ├── data │ │ ├── .gitignore │ │ ├── test_export.csv │ │ ├── test_export.json │ │ ├── sample.vcf.gz │ │ ├── test_export.xlsx │ │ ├── some_export.json.gz │ │ ├── test_creds │ │ ├── sample2.vcf │ │ └── template.json │ ├── test_conversion.py │ ├── test_annotate.py │ ├── test_client.py │ ├── test_errors.py │ ├── test_query_batch.py │ ├── test_exports.py │ ├── test_beacon.py │ ├── test_lookup.py │ ├── test_ratelimit.py │ ├── test_login.py │ ├── test_utils.py │ ├── helper.py │ ├── test_apiresource.py │ ├── test_dataset.py │ ├── test_credentials.py │ ├── test_vault.py │ ├── test_tabulate.py │ └── test_dataset_migrations.py ├── utils │ ├── __init__.py │ ├── md5sum.py │ ├── files.py │ ├── humanize.py │ └── printing.py ├── __main__.py ├── resource │ ├── user.py │ ├── util.py │ ├── application.py │ ├── datasetfield.py │ ├── datasettemplate.py │ ├── beacon.py │ ├── task.py │ ├── vault_sync_task.py │ ├── beaconset.py │ ├── savedquery.py │ ├── __init__.py │ ├── object_copy_task.py │ ├── dataset_restore_task.py │ ├── dataset_snapshot_task.py │ ├── datasetexport.py │ ├── group.py │ ├── manifest.py │ ├── datasetmigration.py │ ├── datasetimport.py │ ├── datasetcommit.py │ └── solveobject.py ├── version.py ├── help.py ├── errors.py ├── annotate.py └── auth.py ├── setup.cfg ├── AUTHORS ├── .bumpversion.cfg ├── INSTALL ├── examples ├── import │ ├── example_template.json │ ├── example_input.json │ └── README.md ├── README.md ├── import_data.py ├── download_vault_folder.py └── template_2to3.py ├── MANIFEST.in ├── .gitignore ├── requirements-dev.txt ├── LICENSE ├── .github └── workflows │ └── python-package.yml ├── tox.ini ├── Makefile ├── setup.py ├── README.md └── installer /recipes/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /solvebio/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /solvebio/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /solvebio/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /solvebio/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /solvebio/contrib/streamlit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /solvebio/test/data/.gitignore: -------------------------------------------------------------------------------- 1 | .solvebio 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description_file = README.md 3 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/tests/.gitignore: -------------------------------------------------------------------------------- 1 | credentials_local.py 2 | -------------------------------------------------------------------------------- /solvebio/test/data/test_export.csv: -------------------------------------------------------------------------------- 1 | rgd_id 2 | RGD:2645 3 | -------------------------------------------------------------------------------- /solvebio/test/data/test_export.json: -------------------------------------------------------------------------------- 1 | {"rgd_id": ["RGD:2645"]} 2 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | David Caplan 2 | Paul George 3 | Rocky Bernstein 4 | -------------------------------------------------------------------------------- /recipes/requirements.txt: -------------------------------------------------------------------------------- 1 | pyyaml==5.3.1 2 | click==7.1.2 3 | ruamel.yaml==0.16.12 4 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/js/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["es2015", "stage-0", "react"] 3 | } 4 | -------------------------------------------------------------------------------- /solvebio/__main__.py: -------------------------------------------------------------------------------- 1 | from .cli.main import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /recipes/tests/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pyyaml==5.3.1 2 | click==7.1.2 3 | ruamel.yaml==0.16.12 4 | mock==4.0.2 5 | pytest -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 2.31.2 3 | files = solvebio/version.py 4 | commit = True 5 | tag = True 6 | -------------------------------------------------------------------------------- /solvebio/test/data/sample.vcf.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/solvebio/solvebio-python/HEAD/solvebio/test/data/sample.vcf.gz -------------------------------------------------------------------------------- /solvebio/contrib/dash/js/.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "node": true, 3 | "browser": true, 4 | "esnext": true, 5 | "newcap": false 6 | } 7 | -------------------------------------------------------------------------------- /solvebio/contrib/streamlit/dev-requirements.txt: -------------------------------------------------------------------------------- 1 | httpx-oauth==0.3.7 2 | streamlit==1.1.0 3 | pandas==1.3.4 4 | python-dotenv==0.19.1 -------------------------------------------------------------------------------- /solvebio/test/data/test_export.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/solvebio/solvebio-python/HEAD/solvebio/test/data/test_export.xlsx -------------------------------------------------------------------------------- /solvebio/test/data/some_export.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/solvebio/solvebio-python/HEAD/solvebio/test/data/some_export.json.gz -------------------------------------------------------------------------------- /solvebio/test/data/test_creds: -------------------------------------------------------------------------------- 1 | machine api.solvebio.com 2 | login testing@solvebio.com 3 | password badcafebabe0feeddeadbeef1ee0012345678900 4 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/__init__.py: -------------------------------------------------------------------------------- 1 | from .solvebio_auth import SolveBioAuth # noqa: F401 2 | from .solvebio_dash import SolveBioDash # noqa: F401 3 | -------------------------------------------------------------------------------- /solvebio/resource/user.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from .apiresource import SingletonAPIResource 3 | 4 | 5 | class User(SingletonAPIResource): 6 | pass 7 | -------------------------------------------------------------------------------- /solvebio/contrib/streamlit/streamlit-secure-app-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/solvebio/solvebio-python/HEAD/solvebio/contrib/streamlit/streamlit-secure-app-demo.gif -------------------------------------------------------------------------------- /solvebio/contrib/dash/js/README.md: -------------------------------------------------------------------------------- 1 | Dash Auth for SolveBio: JS Components 2 | ===================================== 3 | 4 | To build: 5 | 6 | yarn 7 | yarn run build 8 | -------------------------------------------------------------------------------- /solvebio/contrib/streamlit/.env.template: -------------------------------------------------------------------------------- 1 | SOLVEBIO_CLIENT_ID=YOUR_APP_ID 2 | SOLVEBIO_SECRET=YOUR_APP_SECRET 3 | 4 | REDIRECT_URI=http://localhost:8501 5 | DEFAULT_SOLVEBIO_URL=https://my.solvebio.com 6 | -------------------------------------------------------------------------------- /solvebio/cli/tutorial.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pydoc import pager 3 | 4 | TUTORIAL = os.path.abspath(os.path.dirname(__file__)) + '/tutorial.md' 5 | 6 | 7 | def print_tutorial(args): 8 | pager(open(TUTORIAL, 'rb').read()) 9 | -------------------------------------------------------------------------------- /solvebio/version.py: -------------------------------------------------------------------------------- 1 | # Note that this file is multi-lingual and can be used in both Python 2 | # and POSIX shell. 3 | 4 | # This file should define a variable VERSION which we use as the 5 | # debugger version number. 6 | VERSION = '2.33.0' 7 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/dev-requirements.txt: -------------------------------------------------------------------------------- 1 | dash_core_components 2 | dash_html_components 3 | dash_renderer 4 | dash>=0.18.3 5 | dash_auth 6 | flask 7 | percy 8 | selenium 9 | mock 10 | tox 11 | tox-pyenv 12 | six 13 | requests[security] 14 | flake8 15 | solvebio>=2.2.1 16 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Welcome to SolveBio! 2 | 3 | To install it, make sure you have Python 2.7 or greater installed. Then run 4 | this command from the command prompt: 5 | 6 | python setup.py install 7 | 8 | 9 | If you're upgrading from a previous version, you might need to remove it first. 10 | -------------------------------------------------------------------------------- /solvebio/test/data/sample2.vcf: -------------------------------------------------------------------------------- 1 | ##fileformat=VCFv4.1 2 | ##fileDate=140926 3 | ##source=dandan 4 | ##FORMAT= 5 | #CHROM POS ID REF ALT QUAL FILTER INFO FORMAT GENOTYPE 6 | 1 82154 rs4477212 a . . . . GT 0/0 7 | 1 798959 rs11240777 g . . . . GT 0/0 8 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/README.md: -------------------------------------------------------------------------------- 1 | ## SolveBio Login for Dash Apps 2 | 3 | This module is based off the [dash_auth](https://github.com/plotly/dash-auth/) package and provides OAuth2-based login support for Dash apps. 4 | 5 | About Dash: [https://plot.ly/dash/](https://plot.ly/dash/) 6 | 7 | License: MIT 8 | -------------------------------------------------------------------------------- /examples/import/example_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Example SolveBio import template", 3 | "fields": [ 4 | { 5 | "name": "gene_id", 6 | "entity_type": "gene" 7 | }, 8 | { 9 | "name": "solvebio_variant_id", 10 | "entity_type": "variant" 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include AUTHORS 3 | include README.md 4 | include MANIFEST.in 5 | recursive-include solvebio *.md 6 | recursive-include solvebio *.py 7 | recursive-include solvebio/test/data * 8 | include solvebio/contrib/dash/login.js 9 | include solvebio/contrib/dash/oauth-redirect.js 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.py[co] 3 | *#* 4 | *~ 5 | *.swp 6 | *.cache 7 | ChangeLog 8 | MANIFEST 9 | Makefile 10 | dist/ 11 | tmp/ 12 | build/ 13 | docs/_build/ 14 | docs/locale/ 15 | tests/coverage_html/ 16 | tests/.coverage 17 | *.DS_Store 18 | .idea 19 | .tox 20 | .eggs/ 21 | venv* 22 | settings.json 23 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | flake8 2 | mock 3 | requests[security] 4 | urllib3>=1.26.0 5 | flask 6 | percy 7 | selenium 8 | dash>=2.14.0 9 | dash_auth>=2.0.0 10 | dash_core_components>=2.0.0 11 | dash_html_components>=2.0.0 12 | dash_renderer>=1.9.1 13 | Werkzeug<=2.0.3 14 | solvebio==2.12.0 15 | pyyaml==5.3.1 16 | click==7.1.2 17 | ruamel.yaml==0.16.12 18 | pytest 19 | brotli<=1.0.9 20 | -------------------------------------------------------------------------------- /recipes/tests/test_inputs/valid.yml: -------------------------------------------------------------------------------- 1 | recipes: 2 | - name: cDNA Change 3 | version: 1.0.2 4 | description: Gets the cDNA change from the variant field 5 | is_public: true 6 | fields: 7 | name: cdna_change 8 | data_type: string 9 | expression: | 10 | get(translate_variant(record.variant),'cdna_change') 11 | if record.variant else None -------------------------------------------------------------------------------- /solvebio/help.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urljoin 2 | 3 | try: 4 | import webbrowser 5 | except ImportError: 6 | webbrowser = None 7 | 8 | 9 | def open_help(path): 10 | url = urljoin('https://www.solvebio.com/', path) 11 | try: 12 | webbrowser.open(url) 13 | except webbrowser.Error: 14 | print('The SolveBio Python client was unable to open the following ' 15 | 'URL: %s' % url) 16 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/tests/credentials.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | OAUTH_CLIENT_ID = os.environ.get('SOVEBIO_OAUTH_CLIENT_ID') 5 | OAUTH_TOKEN = os.environ.get('SOVEBIO_OAUTH_TOKEN') 6 | OAUTH_USERNAME = os.environ.get('SOVEBIO_OAUTH_USERNAME') 7 | OAUTH_PASSWORD = os.environ.get('SOVEBIO_OAUTH_PASSWORD') 8 | OAUTH_DOMAIN = os.environ.get('SOLVEBIO_OAUTH_DOMAIN', 'test') 9 | 10 | try: 11 | from .credentials_local import * # noqa 12 | except: 13 | pass 14 | -------------------------------------------------------------------------------- /solvebio/test/test_conversion.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from solvebio.resource.util import class_to_api_name 4 | 5 | 6 | class ConversionTest(unittest.TestCase): 7 | def test_class_to_api_name(self): 8 | for class_name, expect in [ 9 | ('Annotation', 'annotations'), 10 | ('DataField', 'data_fields'), 11 | ('Depository', 'depositories')]: 12 | 13 | self.assertEqual(class_to_api_name(class_name), expect) 14 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/js/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "parserOptions": { 3 | "sourceType": "module", 4 | "ecmaFeatures": { 5 | "jsx": true 6 | } 7 | }, 8 | "env": { 9 | "browser": true, 10 | "node": true 11 | }, 12 | "rules": { 13 | "quotes": [2, "single"], 14 | "strict": [2, "never"], 15 | "react/jsx-uses-react": 2, 16 | "react/jsx-uses-vars": 2, 17 | "react/react-in-jsx-scope": 2 18 | }, 19 | "plugins": [ 20 | "react" 21 | ], 22 | "extends": ["eslint:recommended", "plugin:react/recommended"] 23 | } 24 | -------------------------------------------------------------------------------- /solvebio/test/data/template.json: -------------------------------------------------------------------------------- 1 | {"fields": [{"is_list": true, "name": "variants", "entity_type": "variant"}, {"name": "entrez_id"}, {"name": "name", "data_type": "string", "entity_type": "gene"}, {"name": "aliases", "data_type": "string"}], "description": "This is an automatically generated template. It was created by the solvebio-transform utility for the dataset \"sample\" on 2017-02-14 10:06:57", "name": "Auto-generated template: sample 2017-02-14 10:06:57", "tags": ["auto-generated"], "reader_params": {"reader":"json"}, "annotator_params": {"block_size": 100}, "validation_params": {"disabled": true}, "entity_params": {"disabled":true}} 2 | -------------------------------------------------------------------------------- /solvebio/resource/util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import re 3 | 4 | try: 5 | import json 6 | except ImportError: 7 | json = None 8 | 9 | # test for compatible json module 10 | if not (json and hasattr(json, 'loads')): 11 | import simplejson as json # noqa 12 | 13 | 14 | def camelcase_to_underscore(name): 15 | s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) 16 | return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() 17 | 18 | 19 | def class_to_api_name(name, pluralize=True): 20 | if pluralize: 21 | if name.endswith('y'): 22 | name = name[:-1] + 'ie' 23 | name = name + "s" 24 | 25 | return camelcase_to_underscore(name) 26 | -------------------------------------------------------------------------------- /recipes/tests/test_inputs/valid2.yml: -------------------------------------------------------------------------------- 1 | recipes: 2 | - name: Gene 3 | version: 1.0.3 4 | description: Gets the gene symbol from the variant field 5 | is_public: true 6 | fields: 7 | name: gene 8 | data_type: string 9 | expression: | 10 | get(translate_variant(record.variant),'gene') 11 | if record.variant else None 12 | - name: Protein Change 13 | version: 1.0.4 14 | description: Gets the protein change from the variant field 15 | is_public: true 16 | fields: 17 | name: protein_change 18 | data_type: string 19 | expression: | 20 | get(translate_variant(record.variant),'protein_change') 21 | if record.variant else None 22 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/js/webpack.config.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var webpack = require('webpack'); 3 | 4 | module.exports = { 5 | devtool: 'eval', 6 | entry: { 7 | 'login': './src/login-index.react.js', 8 | 'oauth-redirect': './src/oauth-redirect-index.react.js' 9 | }, 10 | output: { 11 | path: path.join(__dirname, '..'), 12 | filename: '[name].js', 13 | }, 14 | module: { 15 | loaders: [ 16 | { 17 | test: /react\.jsx?$/, 18 | loaders: ['babel-loader'], 19 | include: path.join(__dirname, 'src') 20 | }, 21 | { 22 | test: /\.scss$/, 23 | loaders: ["style-loader", "css-loader", "sass-loader"] 24 | } 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py36 3 | 4 | [testenv] 5 | deps = -rdev-requirements.txt 6 | 7 | passenv = * 8 | 9 | [testenv:py27] 10 | basepython={env:TOX_PYTHON_27} 11 | setenv = 12 | PERCY_PARALLEL_TOTAL=4 13 | commands = 14 | python --version 15 | flake8 solvebio_auth setup.py 16 | python -m unittest -v tests.test_solvebio_auth 17 | python -m unittest -v tests.test_solvebio_auth_integration 18 | 19 | [testenv:py36] 20 | basepython={env:TOX_PYTHON_36} 21 | setenv = 22 | PERCY_PARALLEL_TOTAL=4 23 | commands = 24 | python --version 25 | flake8 solvebio_auth setup.py 26 | python -m unittest -v tests.test_solvebio_auth 27 | python -m unittest -v tests.test_solvebio_auth_integration 28 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | vv 3 | *egg-info 4 | *pyc 5 | 6 | node_modules 7 | npm-debug.log 8 | .DS_Store 9 | dist 10 | 11 | # Vim 12 | *.swp 13 | 14 | ### PhpStorm ### 15 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 16 | 17 | *.iml 18 | 19 | ## Directory-based project format: 20 | .idea/ 21 | 22 | ## File-based project format: 23 | *.ipr 24 | *.iws 25 | 26 | ## Plugin-specific files: 27 | 28 | # IntelliJ 29 | /out/ 30 | 31 | # mpeltonen/sbt-idea plugin 32 | .idea_modules/ 33 | 34 | # JIRA plugin 35 | atlassian-ide-plugin.xml 36 | 37 | # Crashlytics plugin (for Android Studio and IntelliJ) 38 | com_crashlytics_export_strings.xml 39 | crashlytics.properties 40 | crashlytics-build.properties 41 | fabric.properties 42 | -------------------------------------------------------------------------------- /solvebio/resource/application.py: -------------------------------------------------------------------------------- 1 | """Solvebio Application API Resource""" 2 | from .apiresource import CreateableAPIResource 3 | from .apiresource import ListableAPIResource 4 | from .apiresource import SearchableAPIResource 5 | from .apiresource import UpdateableAPIResource 6 | from .apiresource import DeletableAPIResource 7 | 8 | 9 | class Application(CreateableAPIResource, 10 | ListableAPIResource, 11 | DeletableAPIResource, 12 | SearchableAPIResource, 13 | UpdateableAPIResource): 14 | ID_ATTR = 'client_id' 15 | RESOURCE_VERSION = 2 16 | 17 | LIST_FIELDS = ( 18 | ('client_id', 'Client ID'), 19 | ('name', 'Name'), 20 | ('description', 'Description'), 21 | ('web_url', 'web_url'), 22 | ) 23 | -------------------------------------------------------------------------------- /solvebio/test/test_annotate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .helper import SolveBioTestCase 4 | 5 | 6 | class TestAnnotator(SolveBioTestCase): 7 | 8 | def test_annotator(self): 9 | fields = [{'name': 'test', 'expression': '"hello world"'}] 10 | records = [{'i': i} for i in range(100)] 11 | a = self.client.Annotator(fields) 12 | 13 | for i, result in enumerate(a.annotate(records)): 14 | self.assertEqual( 15 | result, {'test': 'hello world', 'i': i}) 16 | 17 | 18 | class TestExpression(SolveBioTestCase): 19 | 20 | def test_expression(self): 21 | answer = self.client.Expression("1+1").evaluate(data_type="integer") 22 | self.assertEqual(answer, 2) 23 | 24 | def test_expression_with_context(self): 25 | answer = self.client.Expression("record").evaluate( 26 | data={'record': 123}, data_type="integer") 27 | self.assertEqual(answer, 123) 28 | -------------------------------------------------------------------------------- /solvebio/resource/datasetfield.py: -------------------------------------------------------------------------------- 1 | """Solvebio DatasetField API Resource""" 2 | from .solveobject import convert_to_solve_object 3 | from .apiresource import CreateableAPIResource 4 | from .apiresource import ListableAPIResource 5 | from .apiresource import UpdateableAPIResource 6 | from .apiresource import DeletableAPIResource 7 | 8 | 9 | class DatasetField(CreateableAPIResource, 10 | ListableAPIResource, 11 | DeletableAPIResource, 12 | UpdateableAPIResource): 13 | """ 14 | Each SolveBio dataset has a different set of fields, some of 15 | which can be used as filters. Dataset field resources provide 16 | users with documentation about each field. 17 | """ 18 | RESOURCE_VERSION = 2 19 | 20 | def facets(self, **params): 21 | response = self._client.get(self.facets_url, params) 22 | return convert_to_solve_object(response, client=self._client) 23 | 24 | def help(self): 25 | return self.facets() 26 | -------------------------------------------------------------------------------- /solvebio/test/test_client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .helper import SolveBioTestCase 4 | 5 | 6 | class TestClient(SolveBioTestCase): 7 | 8 | def test_client_resources(self): 9 | resources = [ 10 | 'Annotator', 11 | 'BatchQuery', 12 | 'Beacon', 13 | 'BeaconSet', 14 | 'Dataset', 15 | 'DatasetCommit', 16 | 'DatasetExport', 17 | 'DatasetField', 18 | 'DatasetImport', 19 | 'DatasetMigration', 20 | 'DatasetTemplate', 21 | 'Expression', 22 | 'Filter', 23 | 'GenomicFilter', 24 | 'Manifest', 25 | 'Object', 26 | 'ObjectCopyTask', 27 | 'Query', 28 | 'Task', 29 | 'User', 30 | 'Vault', 31 | 'VaultSyncTask', 32 | ] 33 | for r in resources: 34 | cls = getattr(self.client, r, None) 35 | self.assertTrue(cls) 36 | self.assertEqual(self.client, cls._client) 37 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "solvebio-contrib-dash", 3 | "version": "0.0.1", 4 | "description": "", 5 | "scripts": { 6 | "test": "eslint src", 7 | "build": "./node_modules/.bin/webpack" 8 | }, 9 | "repository": "", 10 | "keywords": [], 11 | "author": "", 12 | "license": "MIT", 13 | "devDependencies": { 14 | "babel-core": "^6.24.1", 15 | "babel-loader": "^7.0.0", 16 | "babel-preset-es2015": "^6.24.1", 17 | "babel-preset-react": "^6.24.1", 18 | "babel-preset-stage-0": "^6.24.1", 19 | "css-loader": "^0.28.0", 20 | "eslint": "^3.19.0", 21 | "eslint-plugin-react": "^6.10.3", 22 | "node-sass": "^4.5.2", 23 | "ramda": "^0.24.1", 24 | "sass-loader": "^6.0.3", 25 | "style-loader": "^0.16.1", 26 | "webpack": "^2.4.1", 27 | "webpack-dev-server": "^2.4.4" 28 | }, 29 | "dependencies": { 30 | "babel-loader": "^7.0.0", 31 | "react": "^15.5.4", 32 | "react-dom": "^15.5.4" 33 | }, 34 | "resolutions": { 35 | "form-data": "2.5.5" 36 | }, 37 | "main": "server.js" 38 | } 39 | -------------------------------------------------------------------------------- /examples/import/example_input.json: -------------------------------------------------------------------------------- 1 | {"aa": "p.T235M", "classification": "missense", "solvebio_variant_id": "GRCh37-1-155932911-155932911-A", "transcript": "ENST00000313695", "genomic_coordinates": {"start": 155932911, "stop": 155932911, "build": "GRCh37", "chromosome": "1"}, "gene_id": "ARHGEF2", "cosmic_id": "998184", "cdna": "c.704C>T","hgvs_g": "NC_000001.10:g.155932911G>A", "hgvs_c": "ENST00000313695:c.704C>T"} 2 | {"aa": "p.A185T", "classification": "missense", "solvebio_variant_id": "GRCh37-1-155934866-155934866-T", "transcript": "ENST00000313695", "genomic_coordinates": {"start": 155934866, "stop": 155934866, "build": "GRCh37", "chromosome": "1"}, "gene_id": "ARHGEF2", "cosmic_id": "998184", "cdna": "c.553G>A","hgvs_g": "NC_000001.10:g.155934866C>T", "hgvs_c": "ENST00000313695:c.553G>A"} 3 | {"aa": "p.A77T", "classification": "missense", "solvebio_variant_id": "GRCh37-1-155936237-155936237-T", "transcript": "ENST00000313695", "genomic_coordinates": {"start": 155936237, "stop": 155936237, "build": "GRCh37", "chromosome": "1"}, "gene_id": "ARHGEF2", "cosmic_id": "998184", "cdna": "c.229G>A","hgvs_g": "NC_000001.10:g.155936237C>T", "hgvs_c": "ENST00000313695:c.229G>A"} -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2023 QuartzBio (https://www.quartz.bio) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /solvebio/resource/datasettemplate.py: -------------------------------------------------------------------------------- 1 | from .apiresource import CreateableAPIResource 2 | from .apiresource import ListableAPIResource 3 | from .apiresource import UpdateableAPIResource 4 | from .apiresource import DeletableAPIResource 5 | 6 | 7 | class DatasetTemplate(CreateableAPIResource, ListableAPIResource, 8 | UpdateableAPIResource, DeletableAPIResource): 9 | """ 10 | DatasetTemplates contain the schema of a Dataset, including some 11 | properties and all the fields. 12 | """ 13 | RESOURCE_VERSION = 2 14 | 15 | LIST_FIELDS = ( 16 | ('id', 'ID'), 17 | ('name', 'Name'), 18 | ('description', 'Description'), 19 | ) 20 | 21 | @property 22 | def import_params(self): 23 | """ 24 | Get DatasetImport parameters from a template 25 | and format them correctly. 26 | """ 27 | return { 28 | 'target_fields': self.fields, 29 | 'reader_params': self.reader_params, 30 | 'entity_params': self.entity_params, 31 | 'annotator_params': self.annotator_params, 32 | 'validation_params': self.validation_params 33 | } 34 | -------------------------------------------------------------------------------- /solvebio/test/test_errors.py: -------------------------------------------------------------------------------- 1 | from .helper import SolveBioTestCase 2 | from solvebio.errors import SolveError 3 | from solvebio.resource import DatasetImport 4 | from solvebio.client import client 5 | 6 | 7 | class ErrorTests(SolveBioTestCase): 8 | 9 | def test_solve_error(self): 10 | try: 11 | # two errors get raised 12 | DatasetImport.create( 13 | dataset_id='510113719950913753', 14 | manifest=dict(files=[dict(filename='soemthing.md')]) 15 | ) 16 | except SolveError as e: 17 | self.assertTrue('Error (dataset_id):' in str(e), e) 18 | self.assertTrue('Invalid dataset' in str(e), e) 19 | self.assertTrue('Error (manifest):' in str(e), e) 20 | self.assertTrue('Each file must' in str(e), e) 21 | 22 | 23 | class ErrorTestsAuth(SolveBioTestCase): 24 | 25 | def test_no_body(self): 26 | # Remove auth 27 | auth = client._auth 28 | client._auth = None 29 | try: 30 | client.whoami() 31 | except SolveError as e: 32 | self.assertTrue('Error: Authentication credentials were not' 33 | in str(e), e) 34 | 35 | client._auth = auth 36 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | SolveBio Python Examples 2 | ====================== 3 | 4 | Complete beginner? Please start with the [basic Python tutorial](https://docs.solvebio.com/docs/tutorial). 5 | 6 | ## Importing data 7 | 8 | There are many ways to import data into SolveBio. 9 | 10 | **Web**: 11 | You can import VCF & JSON files directly via the [SolveBio web interface](https://my.solvebio.com/files) ([web import documentation](https://support.solvebio.com/hc/en-us/articles/218889718-Importing-VCF-and-JSON-Data 12 | )). 13 | 14 | **SolveBio Python client shortcut**: 15 | [Handy dandy command line shortcut](https://github.com/solvebio/solvebio-python/blob/master/examples/import/README.md) built into the SolveBio Python client. 16 | 17 | **Python script**: 18 | Or use an 19 | [example simple Python script](https://github.com/solvebio/solvebio-python/blob/master/examples/import_data.py). 20 | 21 | 22 | ## Advanced Examples 23 | 24 | IPython/Jupyter notebooks do not load embedded graphs in the GitHub viewer. 25 | For optimal experience, please use these links: 26 | 27 | [Generating ICGC survival curves](http://nbviewer.jupyter.org/github/solvebio/solvebio-python/blob/fef6b7987c718519da5ede17f47b1601768987a4/examples/generating_icgc_survival_curves.ipynb) 28 | 29 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/js/styles/application.scss: -------------------------------------------------------------------------------- 1 | html { 2 | font-family: Open Sans, Helvetica, sans-serif; 3 | font-weight: 400; 4 | color: #2A3F5F; 5 | } 6 | 7 | h2 { 8 | font-family: Dosis, Helvetica, sans-serif; 9 | font-weight: 600; 10 | font-size: 28px; 11 | margin-top: 14px; 12 | margin-bottom: 14px; 13 | } 14 | 15 | h4 { 16 | font-size: 18px; 17 | margin-top: 9px; 18 | margin-bottom: 18px; 19 | } 20 | 21 | button { 22 | border: 1px solid #119DFF; 23 | font-size: 14px; 24 | color: #ffffff; 25 | background-color: #119DFF; 26 | padding: 9px 18px; 27 | border-radius: 5px; 28 | text-align: center; 29 | text-transform: capitalize; 30 | letter-spacing: 0.5px; 31 | line-height: 1; 32 | cursor: pointer; 33 | outline: none; 34 | margin: 0px; 35 | } 36 | 37 | a { 38 | color: #119DFF; 39 | text-decoration: none; 40 | cursor: pointer; 41 | } 42 | 43 | .caption { 44 | font-size: 13px; 45 | margin-top: 20px; 46 | color: #A2B1C6; 47 | } 48 | 49 | .container { 50 | margin-left: auto; 51 | margin-right: auto; 52 | width: 90%; 53 | max-width: 300px; 54 | background-color: white; 55 | padding: 20px; 56 | border-radius: 4px; 57 | } 58 | -------------------------------------------------------------------------------- /solvebio/test/test_query_batch.py: -------------------------------------------------------------------------------- 1 | from .helper import SolveBioTestCase 2 | 3 | 4 | class BatchQueryTest(SolveBioTestCase): 5 | def setUp(self): 6 | super(BatchQueryTest, self).setUp() 7 | self.dataset = self.client.Object.get_by_full_path( 8 | self.TEST_DATASET_FULL_PATH) 9 | 10 | def test_invalid_batch_query(self): 11 | queries = [ 12 | self.dataset.query(limit=1, fields=['bogus_field']), 13 | self.dataset.query(limit=10).filter(bogus_id__gt=100000) 14 | ] 15 | 16 | results = self.client.BatchQuery(queries).execute() 17 | self.assertEqual(len(results), 2) 18 | self.assertEqual(results[0]['status_code'], 400) 19 | self.assertEqual(results[1]['status_code'], 400) 20 | 21 | def test_batch_query(self): 22 | queries = [ 23 | self.dataset.query(limit=1), 24 | self.dataset.query(limit=10).filter(mamit_trnadb__gt=1), 25 | self.dataset.query(limit=100), 26 | ] 27 | results = self.client.BatchQuery(queries).execute() 28 | self.assertEqual(len(results), 3) 29 | self.assertEqual(len(results[0]['results']), 1) 30 | self.assertEqual(len(results[1]['results']), 10) 31 | self.assertEqual(len(results[2]['results']), 100) 32 | -------------------------------------------------------------------------------- /examples/import/README.md: -------------------------------------------------------------------------------- 1 | SolveBio Import Shortcut Example 2 | ====================== 3 | 4 | Make sure you're using the newest version of SolveBio with `pip install solvebio --upgrade` 5 | 6 | The SolveBio Python client provides a simple command line shortcut to import data. 7 | 8 | To run the example commands, download the [example_input.json](https://github.com/solvebio/solvebio-python/blob/master/examples/import/example_input.json) and [example_template.json](https://github.com/solvebio/solvebio-python/blob/master/examples/import/example_template.json) files. 9 | 10 | In your command line: 11 | ```bash 12 | solvebio import --create-dataset test-dataset example_input.json 13 | ``` 14 | 15 | This dataset will be placed at the root of your personal vault. You can specify values for the `--vault` and `--path` flags 16 | to place the dataset into a different vault, or into a particular folder within the target vault. 17 | 18 | If you wish to add SolveBio entities to your data, you can do so by using a template file (you can also do it later on the SolveBio web interface). 19 | ```bash 20 | solvebio import --create-dataset test-dataset example_input.json --template-file example_template.json 21 | ``` 22 | 23 | To get a full list of the arguments options available, try 24 | ```bash 25 | solvebio import -h 26 | ``` 27 | -------------------------------------------------------------------------------- /solvebio/utils/md5sum.py: -------------------------------------------------------------------------------- 1 | import os 2 | import hashlib 3 | import binascii 4 | 5 | # Default thresholds for multipart S3 files 6 | MULTIPART_THRESHOLD = 64 * 1024 * 1024 7 | MULTIPART_CHUNKSIZE = 64 * 1024 * 1024 8 | 9 | 10 | def md5sum(path, multipart_threshold=MULTIPART_THRESHOLD, 11 | multipart_chunksize=MULTIPART_CHUNKSIZE): 12 | 13 | def _read_chunks(f, chunk_size): 14 | chunk = f.read(chunk_size) 15 | while chunk: 16 | yield chunk 17 | chunk = f.read(chunk_size) 18 | 19 | filesize = os.path.getsize(path) 20 | 21 | with open(path, "rb") as f: 22 | if multipart_threshold and filesize > multipart_threshold: 23 | block_count = 0 24 | md5string = "" 25 | for block in _read_chunks(f, multipart_chunksize): 26 | md5 = hashlib.md5() 27 | md5.update(block) 28 | md5string += md5.hexdigest() 29 | block_count += 1 30 | 31 | md5 = hashlib.md5() 32 | md5.update(binascii.unhexlify(md5string)) 33 | else: 34 | block_count = None 35 | md5 = hashlib.md5() 36 | for block in _read_chunks(f, multipart_chunksize): 37 | md5.update(block) 38 | 39 | return md5.hexdigest(), block_count 40 | -------------------------------------------------------------------------------- /examples/import_data.py: -------------------------------------------------------------------------------- 1 | import solvebio 2 | 3 | solvebio.login() 4 | 5 | vault = solvebio.Vault.get_personal_vault() 6 | 7 | # The folders that will contain your dataset 8 | path = '/SampleImport/1.0.0' 9 | 10 | # The name of your dataset 11 | dataset_name = 'SampleDataset' 12 | 13 | # Create a dataset 14 | dataset = solvebio.Object.get_or_create_by_full_path( 15 | '{0}:/{1}/{2}'.format(vault.name, path, dataset_name), 16 | ) 17 | 18 | # Create a manifest object and a file to it 19 | manifest = solvebio.Manifest() 20 | manifest.add_file('path/to/file.vcf.gz') 21 | 22 | # Create the import 23 | imp = solvebio.DatasetImport.create( 24 | dataset_id=dataset.id, 25 | manifest=manifest.manifest 26 | ) 27 | 28 | # Prints updates as the data is processed 29 | # and indexed into SolveBio 30 | dataset.activity(follow=True) 31 | 32 | # 33 | # You now have data! 34 | # 35 | 36 | # Let's add some more records that include a new field 37 | new_records = [ 38 | { 39 | 'gene_symbol': 'BRCA2', 40 | 'some_new_field': 'a new string field' 41 | }, 42 | { 43 | 'gene_symbol': 'CFTR', 44 | 'some_new_field': 'that new field' 45 | } 46 | ] 47 | 48 | imp = solvebio.DatasetImport.create( 49 | dataset_id=dataset.id, 50 | data_records=new_records 51 | ) 52 | dataset.activity(follow=True) 53 | -------------------------------------------------------------------------------- /solvebio/test/test_exports.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from solvebio.test.client_mocks import fake_export_create 4 | 5 | from .helper import SolveBioTestCase 6 | 7 | 8 | class TestDatasetExports(SolveBioTestCase): 9 | 10 | def _validate_export(self, export, dataset, **kwargs): 11 | self.assertEqual(export.dataset_id, dataset.id) 12 | 13 | @mock.patch('solvebio.resource.DatasetExport.create') 14 | def test_export_from_query(self, Create): 15 | Create.side_effect = fake_export_create 16 | 17 | # Test with params 18 | params = { 19 | 'fields': ['my_field'], 20 | 'limit': 100, 21 | } 22 | target_fields = [dict(name='test')] 23 | 24 | dataset = self.client.Dataset(1) 25 | export = dataset.export( 26 | params=params, 27 | dataset=dataset, 28 | target_fields=target_fields, 29 | target_full_path='~/hello', 30 | format='tsv.gz', 31 | follow=False 32 | ) 33 | self.assertEqual(export.dataset_id, dataset.id) 34 | self.assertEqual(export.target_fields, target_fields) 35 | self.assertEqual(export.target_full_path, '~/hello') 36 | self.assertEqual(export.format, 'tsv.gz') 37 | for k in ['fields', 'limit']: 38 | self.assertEqual(export.params[k], params[k]) 39 | -------------------------------------------------------------------------------- /solvebio/contrib/streamlit/README.md: -------------------------------------------------------------------------------- 1 | ## SolveBio Login for Streamlit Apps 2 | 3 | This module provides OAuth2-based login support for Streamlit apps. 4 | 5 | About Streamlit: [https://streamlit.io/](https://streamlit.io/) 6 | 7 | 8 | ### Securing Streamlit app 9 | 10 | Create a new app in SolveBio RUO and copy app's client id and secret to .env file. 11 | 12 | `SolveBioStreamlit` class is used to wrap Streamlit apps with SolveBio OAuth2. Once the user is successfully authenticated, OAuth2 `token` and the initialised `SolveClient` are saved to the Streamlit's session state. You can access them: 13 | ```python 14 | st.session_state.solvebio_client 15 | st.session_state.token 16 | ``` 17 | 18 | Wrapping Streamlit app: 19 | 20 | ```python 21 | def streamlit_demo_app(): 22 | # Getting the sovle client from the Streamlit session state 23 | solvebio_client = st.session_state.solvebio_client 24 | user = solvebio_client.User.retrieve() 25 | 26 | st.title("Solvebio app") 27 | st.header(f"Welcome back {user['first_name']}!") 28 | 29 | # Wrapping Streamlit app with SolveBio OAuth2 30 | secure_app = SolveBioStreamlit() 31 | secure_app.wrap(streamlit_app=streamlit_demo_app) 32 | ``` 33 | 34 | 35 | ### SolveBio secure Streamlit app demo 36 | 37 | To run streamlit demo app: 38 | ```bash 39 | streamlit run app.py 40 | ``` 41 | ![streamlit-secure-app-demo](streamlit-secure-app-demo.gif) 42 | -------------------------------------------------------------------------------- /solvebio/resource/beacon.py: -------------------------------------------------------------------------------- 1 | from .apiresource import CreateableAPIResource 2 | from .apiresource import ListableAPIResource 3 | from .apiresource import UpdateableAPIResource 4 | from .apiresource import DeletableAPIResource 5 | 6 | 7 | class Beacon(CreateableAPIResource, 8 | ListableAPIResource, 9 | DeletableAPIResource, 10 | UpdateableAPIResource): 11 | """ 12 | Beacons provide entity-based search endpoints for datasets. Beacons 13 | must be created within Beacon Sets. 14 | """ 15 | RESOURCE_VERSION = 2 16 | 17 | LIST_FIELDS = ( 18 | ('id', 'ID'), 19 | ('title', 'Title'), 20 | ('description', 'Description'), 21 | ('vault_object_id', 'Object ID'), 22 | ) 23 | 24 | def _query_url(self): 25 | if 'query_url' not in self: 26 | if 'id' not in self or not self['id']: 27 | raise Exception( 28 | 'No Beacon ID was provided. ' 29 | 'Please instantiate the Beacon' 30 | 'object with an ID.') 31 | return self.instance_url() + '/query' 32 | return self['query_url'] 33 | 34 | def query(self, query, entity_type=None): 35 | data = { 36 | 'query': query, 37 | 'entity_type': entity_type 38 | } 39 | return self._client.post(self._query_url(), data) 40 | -------------------------------------------------------------------------------- /solvebio/test/test_beacon.py: -------------------------------------------------------------------------------- 1 | from .helper import SolveBioTestCase 2 | 3 | 4 | class BeaconTests(SolveBioTestCase): 5 | 6 | # TEST_DATASET_FULL_PATH = 'solvebio:public:/ClinVar/3.7.4-2017-01-30/Variants-GRCh37' # noqa 7 | TEST_DATASET_FULL_PATH = 'quartzbio:Public:/ClinVar/5.2.0-20210110/Variants-GRCH38' 8 | 9 | def test_beacon_request(self): 10 | """ 11 | Check that current Clinvar/Variants returns correct 12 | fields for beacon 13 | """ 14 | dataset = self.client.Object.get_by_full_path( 15 | self.TEST_DATASET_FULL_PATH) 16 | beacon = dataset.beacon(chromosome='6', 17 | coordinate=51612854, # staging 18 | allele='G') 19 | 20 | check_fields = ['query', 'exist', 'total'] 21 | 22 | for f in check_fields: 23 | self.assertTrue(f in beacon) 24 | 25 | # Check that Clinvar/Variants version 3.7.0-2015-12-06 26 | # returns true for specific case 27 | 28 | dataset = self.client.Object.get_by_full_path( 29 | self.TEST_DATASET_FULL_PATH) 30 | beacontwo = dataset.beacon(chromosome='13', 31 | coordinate=113803460, 32 | allele='T') 33 | 34 | self.assertTrue(beacontwo['exist']) 35 | self.assertEqual(beacontwo['total'], 1) 36 | -------------------------------------------------------------------------------- /solvebio/test/test_lookup.py: -------------------------------------------------------------------------------- 1 | from .helper import SolveBioTestCase 2 | 3 | 4 | class LookupTests(SolveBioTestCase): 5 | 6 | def setUp(self): 7 | super(LookupTests, self).setUp() 8 | self.dataset = self.client.Object.get_by_full_path( 9 | self.TEST_DATASET_FULL_PATH) 10 | 11 | def test_lookup_error(self): 12 | # Check that incorrect lookup results in empty list. 13 | lookup_one = self.dataset.lookup('test') 14 | self.assertEqual(lookup_one, []) 15 | 16 | lookup_two = self.dataset.lookup('test', 'nothing') 17 | self.assertEqual(lookup_two, []) 18 | 19 | def test_lookup_correct(self): 20 | # Check that lookup with specific sbid is correct. 21 | records = list(self.dataset.query(limit=2)) 22 | record_one = records[0] 23 | record_two = records[1] 24 | sbid_one = record_one['_id'] 25 | sbid_two = record_two['_id'] 26 | 27 | lookup_one = self.dataset.lookup(sbid_one) 28 | self.assertEqual(lookup_one[0], record_one) 29 | 30 | lookup_two = self.dataset.lookup(sbid_two) 31 | self.assertEqual(lookup_two[0], record_two) 32 | 33 | # Check that combining sbids returns list of correct results. 34 | joint_lookup = self.dataset.lookup(sbid_one, sbid_two) 35 | self.assertEqual(joint_lookup[0], record_one) 36 | self.assertEqual(joint_lookup[1], record_two) 37 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | name: SolveBio Python Package 2 | 3 | # on: [push, pull_request] 4 | on: [push] 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-22.04 9 | strategy: 10 | matrix: 11 | python-version: ['3.8', '3.9','3.10', '3.11', '3.12'] 12 | env: 13 | SOLVEBIO_API_HOST: ${{ secrets.QUARTZBIO_API_HOST }} 14 | SOLVEBIO_API_KEY: ${{ secrets.QUARTZBIO_API_KEY }} 15 | steps: 16 | - name: Checkout repository 17 | uses: actions/checkout@v4 18 | - name: Setup Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v5 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | - name: Display Python version 23 | run: python -c "import sys; print(sys.version)" 24 | - name: Export pythonpath 25 | run: | 26 | export PYTHONPATH=$PYTHONPATH:$(pwd) 27 | - name: Install Tox and any other packages 28 | run: | 29 | pip install -U wheel --user 30 | pip install setuptools 31 | pip install flake8 pytest 32 | - name: Install dependencies 33 | run: | 34 | pip install -r requirements-dev.txt 35 | pip install XlsxWriter===0.9.3 36 | - name: Scripts 37 | run: | 38 | python -m pytest recipes/tests/test_recipes_sync.py 39 | python -m pytest solvebio/test/test_object.py 40 | python -m flake8 solvebio 41 | -------------------------------------------------------------------------------- /solvebio/contrib/streamlit/solvebio_auth.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Any 3 | from typing import Dict 4 | from urllib.parse import urlencode 5 | from urllib.parse import urljoin 6 | 7 | import solvebio 8 | 9 | from httpx_oauth.oauth2 import BaseOAuth2 10 | 11 | 12 | class SolveBioOAuth2(BaseOAuth2[Dict[str, Any]]): 13 | """Class implementing OAuth2 for SolveBio API""" 14 | 15 | # SolveBio API OAuth2 endpoints 16 | SOLVEBIO_URL = os.environ.get('SOLVEBIO_URL', 'https://my.solvebio.com') 17 | OAUTH2_TOKEN_URL = "/v1/oauth2/token" 18 | OAUTH2_REVOKE_TOKEN_URL = "/v1/oauth2/revoke_token" 19 | 20 | def __init__(self, client_id, client_secret, name="solvebio"): 21 | super().__init__( 22 | client_id, 23 | client_secret, 24 | self.SOLVEBIO_URL, 25 | urljoin(solvebio.get_api_host(), self.OAUTH2_TOKEN_URL), 26 | revoke_token_endpoint=urljoin( 27 | solvebio.get_api_host(), self.OAUTH2_REVOKE_TOKEN_URL 28 | ), 29 | name=name, 30 | ) 31 | 32 | def get_authorization_url(self, redirect_uri): 33 | """Creates authorization url for OAuth2""" 34 | 35 | params = { 36 | "response_type": "code", 37 | "client_id": self.client_id, 38 | "redirect_uri": redirect_uri, 39 | } 40 | 41 | return "{}/authorize?{}".format(self.authorize_endpoint, urlencode(params)) 42 | -------------------------------------------------------------------------------- /solvebio/resource/task.py: -------------------------------------------------------------------------------- 1 | """Solvebio Task API Resource""" 2 | from .apiresource import ListableAPIResource 3 | from .apiresource import UpdateableAPIResource 4 | 5 | 6 | class Task(ListableAPIResource, UpdateableAPIResource): 7 | """ 8 | Tasks are operations on datasets or vaults. 9 | """ 10 | RESOURCE_VERSION = 2 11 | 12 | LIST_FIELDS = ( 13 | ('id', 'ID'), 14 | ('task_display_name', 'Task Type'), 15 | ('task_id', 'Task ID'), 16 | ('description', 'Description'), 17 | ('status', 'Status'), 18 | ('created_at', 'Created'), 19 | ) 20 | 21 | SLEEP_WAIT_DEFAULT = 5.0 22 | 23 | @property 24 | def child_object(self): 25 | """ Get Task child object class """ 26 | from . import types 27 | child_klass = types.get(self.task_type.split('.')[1]) 28 | return child_klass.retrieve(self.task_id, client=self._client) 29 | 30 | def follow(self, sleep_seconds=SLEEP_WAIT_DEFAULT): 31 | """ Follow the child object but do not loop """ 32 | self.child_object.follow(loop=False, sleep_seconds=sleep_seconds) 33 | 34 | def cancel(self): 35 | """ Cancel a task """ 36 | _status = self.status 37 | self.status = "canceled" 38 | try: 39 | self.save() 40 | except: 41 | # Reset status to what it was before 42 | # status update failure 43 | self.status = _status 44 | raise 45 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | ; Settings file for flake8: 2 | ; http://flake8.readthedocs.org/en/latest/config.html#settings 3 | [flake8] 4 | exclude = *migrations/*,.tox,./tmp,./build 5 | filename = *.py 6 | ; E127 continuation line over-indented for visual indent 7 | ; For example in: 8 | ; if 'SOLVEBIO_API_KEY' in os.environ and \ 9 | ; os.environ['SOLVEBIO_API_KEY'].startswith('0cedb161d'): 10 | ; self.assertRaises(SolveError, lambda: Annotation.retrieve(1)) 11 | ; proper indentation would make it line up with the next ident which is 12 | ; E125 13 | ignore = E127,E402,N801,N802,E722,W504 14 | max-line-length = 120 15 | 16 | [gh-actions] 17 | python = 18 | 2.7: py27 19 | 3.6: py36 20 | 3.7: py37 21 | 3.8: py38 22 | 23 | [tox] 24 | envlist = py27, py34, py37, pypy 25 | 26 | [testenv] 27 | deps = 28 | requests>=2.0.0 29 | mock>=1.0.1 30 | unittest2 31 | commands = python -W always setup.py test {posargs} 32 | 33 | [testenv:py27] 34 | deps = 35 | flake8 36 | requests>=2.0.0 37 | mock>=1.0.1 38 | commands = 39 | python -W always setup.py test {posargs} 40 | flake8 solvebio 41 | 42 | [testenv:py34] 43 | deps = 44 | flake8 45 | requests>=2.0.0 46 | mock>=1.0.1 47 | commands = 48 | python -W always setup.py test {posargs} 49 | flake8 solvebio 50 | 51 | [testenv:py37] 52 | deps = 53 | flake8 54 | requests>=2.0.0 55 | mock>=1.0.1 56 | commands = 57 | python -W always setup.py test {posargs} 58 | flake8 solvebio 59 | -------------------------------------------------------------------------------- /solvebio/resource/vault_sync_task.py: -------------------------------------------------------------------------------- 1 | """Solvebio VaultSyncTask API Resource""" 2 | import time 3 | from .apiresource import ListableAPIResource 4 | from .apiresource import CreateableAPIResource 5 | from .apiresource import UpdateableAPIResource 6 | from .task import Task 7 | 8 | 9 | class VaultSyncTask(CreateableAPIResource, 10 | ListableAPIResource, 11 | UpdateableAPIResource): 12 | RESOURCE_VERSION = 2 13 | 14 | LIST_FIELDS = ( 15 | ('id', 'ID'), 16 | ('status', 'Status'), 17 | ('vault_id', 'Vault'), 18 | ('created_at', 'Created'), 19 | ) 20 | 21 | def follow(self, loop=True, sleep_seconds=Task.SLEEP_WAIT_DEFAULT): 22 | if self.status == 'queued': 23 | print("Waiting for Vault sync (id = {0}) to start..." 24 | .format(self.id)) 25 | 26 | _status = self.status 27 | while self.status in ['queued', 'running']: 28 | if self.status != _status: 29 | print("Vault sync is now {0} (was {1})" 30 | .format(self.status, _status)) 31 | _status = self.status 32 | 33 | if self.status == 'running': 34 | print("Vault sync '{0}' is {1}" 35 | .format(self.id, self.status)) 36 | 37 | if not loop: 38 | return 39 | 40 | time.sleep(sleep_seconds) 41 | self.refresh() 42 | 43 | if self.status == 'completed': 44 | print("Vault complete!") 45 | -------------------------------------------------------------------------------- /solvebio/test/test_ratelimit.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import time 4 | from mock import patch 5 | 6 | import solvebio.client 7 | from .helper import SolveBioTestCase 8 | 9 | 10 | class FakeResponse(): 11 | def __init__(self, headers, status_code): 12 | self.headers = headers 13 | self.status_code = status_code 14 | 15 | def json(self): 16 | return {} 17 | 18 | 19 | class ClientRateLimit(SolveBioTestCase): 20 | """Test of rate-limiting an API request""" 21 | 22 | def setUp(self): 23 | super(ClientRateLimit, self).setUp() 24 | self.call_count = 0 25 | 26 | def fake_response(self): 27 | if self.call_count == 0: 28 | self.call_count += 1 29 | return FakeResponse({'retry-after': '0'}, 429) 30 | else: 31 | return FakeResponse({}, 200) 32 | 33 | def test_rate_limit(self): 34 | with patch('solvebio.client.Session.request') as mock_request: 35 | mock_request.side_effect = lambda *args, **kw: self.fake_response() 36 | start_time = time.time() 37 | dataset = solvebio.Dataset.retrieve(1) 38 | elapsed_time = time.time() - start_time 39 | self.assertTrue(isinstance(dataset, solvebio.Dataset), 40 | "Got a dataset back (eventually)") 41 | self.assertTrue(elapsed_time > 1.0, 42 | "Should have delayed for over a second; " 43 | "(was %s)" % elapsed_time) 44 | self.assertEqual(self.call_count, 1) 45 | -------------------------------------------------------------------------------- /solvebio/resource/beaconset.py: -------------------------------------------------------------------------------- 1 | from .apiresource import CreateableAPIResource 2 | from .apiresource import ListableAPIResource 3 | from .apiresource import UpdateableAPIResource 4 | from .apiresource import DeletableAPIResource 5 | 6 | 7 | class BeaconSet(CreateableAPIResource, 8 | ListableAPIResource, 9 | DeletableAPIResource, 10 | UpdateableAPIResource): 11 | """ 12 | A beacon set is an arbitrary group of beacons, which provide 13 | entity-based search endpoints for datasets. Beacon sets can be 14 | used to query a group of related datasets in a single API request. 15 | """ 16 | RESOURCE_VERSION = 2 17 | 18 | LIST_FIELDS = ( 19 | ('id', 'ID'), 20 | ('title', 'Title'), 21 | ('description', 'Description'), 22 | ('is_shared', 'Shared?'), 23 | ('is_public', 'Public?'), 24 | ('created_at', 'Created'), 25 | ('updated_at', 'Last Updated'), 26 | ) 27 | 28 | def _query_url(self): 29 | if 'query_url' not in self: 30 | if 'id' not in self or not self['id']: 31 | raise Exception( 32 | 'No BeaconSet ID was provided. ' 33 | 'Please instantiate the BeaconSet' 34 | 'object with an ID.') 35 | return self.instance_url() + '/query' 36 | return self['query_url'] 37 | 38 | def query(self, query, entity_type=None): 39 | data = { 40 | 'query': query, 41 | 'entity_type': entity_type 42 | } 43 | return self._client.post(self._query_url(), data) 44 | -------------------------------------------------------------------------------- /solvebio/resource/savedquery.py: -------------------------------------------------------------------------------- 1 | from ..query import Query 2 | 3 | from .apiresource import CreateableAPIResource 4 | from .apiresource import ListableAPIResource 5 | from .apiresource import UpdateableAPIResource 6 | from .apiresource import DeletableAPIResource 7 | 8 | 9 | class SavedQuery(CreateableAPIResource, ListableAPIResource, 10 | UpdateableAPIResource, DeletableAPIResource): 11 | """ 12 | A saved query is a set of query parameters that persists, 13 | giving users the ability to apply them to compatible datasets 14 | with ease. A dataset is said to be compatible with a saved query 15 | if it contains all the fields found in said saved query. 16 | """ 17 | RESOURCE_VERSION = 2 18 | 19 | LIST_FIELDS = ( 20 | ('id', 'ID'), 21 | ('name', 'Name'), 22 | ('is_shared', 'Is Shared'), 23 | ('description', 'Description'), 24 | ) 25 | 26 | def query(self, dataset=None): 27 | if 'id' not in self or not self['id']: 28 | raise Exception( 29 | 'No SavedQuery ID was provided. ' 30 | 'Please instantiate the SavedQuery ' 31 | 'object with an ID.') 32 | if not dataset: 33 | raise Exception( 34 | 'No Dataset was specified. ' 35 | 'Please provide either the Dataset object or ' 36 | 'the Dataset ID.') 37 | 38 | # Can be provided as an object or as an ID. 39 | try: 40 | dataset_id = dataset.id 41 | except AttributeError: 42 | dataset_id = dataset 43 | 44 | return Query(dataset_id, client=self._client, **self.params) 45 | -------------------------------------------------------------------------------- /solvebio/resource/__init__.py: -------------------------------------------------------------------------------- 1 | from .apiresource import ListObject 2 | from .user import User 3 | from .dataset import Dataset 4 | from .datasetfield import DatasetField 5 | from .datasetimport import DatasetImport 6 | from .datasetexport import DatasetExport 7 | from .datasetcommit import DatasetCommit 8 | from .datasetmigration import DatasetMigration 9 | from .datasettemplate import DatasetTemplate 10 | from .vault_sync_task import VaultSyncTask 11 | from .object_copy_task import ObjectCopyTask 12 | from .dataset_restore_task import DatasetRestoreTask 13 | from .dataset_snapshot_task import DatasetSnapshotTask 14 | from .manifest import Manifest 15 | from .object import Object 16 | from .vault import Vault 17 | from .task import Task 18 | from .beacon import Beacon 19 | from .beaconset import BeaconSet 20 | from .application import Application 21 | from .group import Group 22 | from .savedquery import SavedQuery 23 | 24 | 25 | types = { 26 | 'Application': Application, 27 | 'Beacon': Beacon, 28 | 'BeaconSet': BeaconSet, 29 | 'Dataset': Dataset, 30 | 'DatasetImport': DatasetImport, 31 | 'DatasetExport': DatasetExport, 32 | 'DatasetCommit': DatasetCommit, 33 | 'DatasetMigration': DatasetMigration, 34 | 'DatasetTemplate': DatasetTemplate, 35 | 'DatasetField': DatasetField, 36 | 'DatasetRestoreTask': DatasetRestoreTask, 37 | 'DatasetSnapshotTask': DatasetSnapshotTask, 38 | 'Group': Group, 39 | 'Manifest': Manifest, 40 | 'Object': Object, 41 | 'ObjectCopyTask': ObjectCopyTask, 42 | 'ECSTask': Task, 43 | 'VaultSyncTask': VaultSyncTask, 44 | 'User': User, 45 | 'Vault': Vault, 46 | 'list': ListObject, 47 | 'SavedQuery': SavedQuery, 48 | } 49 | -------------------------------------------------------------------------------- /solvebio/test/test_login.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import contextlib 4 | import sys 5 | import unittest 6 | import solvebio 7 | import solvebio.cli.auth as auth 8 | 9 | 10 | @contextlib.contextmanager 11 | def nostdout(): 12 | savestderr = sys.stdout 13 | 14 | class Devnull(object): 15 | def write(self, _): 16 | pass 17 | sys.stdout = Devnull() 18 | try: 19 | yield 20 | finally: 21 | sys.stdout = savestderr 22 | 23 | 24 | class TestLogin(unittest.TestCase): 25 | def setUp(self): 26 | self.api_host = solvebio.get_api_host() 27 | # temporarily replace with dummy methods for testing 28 | self.delete_credentials = auth.delete_credentials 29 | auth.delete_credentials = lambda: None 30 | 31 | def tearDown(self): 32 | solvebio.client._host = self.api_host 33 | auth.delete_credentials = self.delete_credentials 34 | 35 | def test_bad_login(self): 36 | with nostdout(): 37 | self.assertEqual(auth.login_and_save_credentials(), None, 38 | 'Invalid login') 39 | 40 | # Test invalid host 41 | solvebio.client._host = 'https://some.fake.domain.foobar' 42 | self.assertEqual(auth.login_and_save_credentials(), None, 43 | 'Invalid login') 44 | 45 | def test_init_login(self): 46 | from solvebio import login 47 | from solvebio.client import client 48 | _auth = client._auth 49 | 50 | client._auth = None 51 | login(api_key="TEST_KEY") 52 | self.assertEqual(client._auth.token, "TEST_KEY") 53 | 54 | # Reset the key 55 | client._auth = _auth 56 | -------------------------------------------------------------------------------- /solvebio/resource/object_copy_task.py: -------------------------------------------------------------------------------- 1 | """Solvebio ObjectCopyTask API Resource""" 2 | import time 3 | from .apiresource import ListableAPIResource 4 | from .apiresource import CreateableAPIResource 5 | from .apiresource import UpdateableAPIResource 6 | from .task import Task 7 | 8 | 9 | class ObjectCopyTask(CreateableAPIResource, 10 | ListableAPIResource, 11 | UpdateableAPIResource): 12 | RESOURCE_VERSION = 2 13 | 14 | LIST_FIELDS = ( 15 | ('id', 'ID'), 16 | ('status', 'Status'), 17 | ('source_vault_id', 'Source Vault'), 18 | ('target_vault_id', 'Target Vault'), 19 | ('source_object_id', 'Source'), 20 | ('target_object_id', 'Target'), 21 | ('created_at', 'Created'), 22 | ) 23 | 24 | def follow(self, loop=True, sleep_seconds=Task.SLEEP_WAIT_DEFAULT): 25 | if self.status == 'queued': 26 | print("Waiting for Object Copy task (id = {0}) to start..." 27 | .format(self.id)) 28 | 29 | _status = self.status 30 | while self.status in ['queued', 'running']: 31 | if self.status != _status: 32 | print("Object Copy task is now {0} (was {1})" 33 | .format(self.status, _status)) 34 | _status = self.status 35 | 36 | if self.status == 'running': 37 | print("Object Copy task '{0}' is {1}" 38 | .format(self.id, self.status)) 39 | 40 | if not loop: 41 | return 42 | 43 | time.sleep(sleep_seconds) 44 | self.refresh() 45 | 46 | if self.status == 'completed': 47 | print("Object Copy task complete!") 48 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Note: This makefile include remake-style target comments. 2 | # These comments before the targets start with #: 3 | # remake --tasks to shows the targets and the comments 4 | 5 | PHONY=all check clean dist distclean test clean_pyc lint install changelog release 6 | GIT2CL ?= git2cl 7 | PYTHON ?= python 8 | PYTHON3 ?= python3 9 | RM ?= rm 10 | LINT = flake8 11 | 12 | #: the default target - same as running "check" 13 | all: check 14 | 15 | #: Run all tests 16 | check: 17 | $(PYTHON) ./setup.py test 18 | # $(PYTHON3) ./setup.py test 19 | 20 | #: Clean up temporary files and .pyc files 21 | clean: distclean clean_pyc 22 | $(PYTHON) ./setup.py clean 23 | $(RM) -rf dist/* 24 | 25 | #: Create source (tarball) and wheel distribution 26 | dist: 27 | $(PYTHON) ./setup.py sdist bdist_wheel 28 | 29 | #: Remove .pyc files 30 | clean_pyc: 31 | $(RM) -f */*.pyc */*/*.pyc 32 | 33 | #: Style check. Set env var LINT to pyflakes, flake, or flake8 34 | lint: 35 | $(LINT) 36 | 37 | # It is too much work to figure out how to add a new command to distutils 38 | # to do the following. I'm sure distutils will someday get there. 39 | DISTCLEAN_FILES = build dist *.egg-info *.pyc *.so py*.py 40 | 41 | #: Remove ALL derived files 42 | distclean: clean 43 | -rm -fr $(DISTCLEAN_FILES) || true 44 | 45 | #: Install package locally 46 | install: 47 | $(PYTHON) ./setup.py install 48 | 49 | #: Same as 'check' target 50 | test: check 51 | 52 | #: Run a specific unit test, eg test-sample runs solvebio.test.test_sample 53 | test-%: 54 | python -m unittest solvebio.test.$(subst test-,test_,$@) 55 | 56 | changelog: 57 | github_changelog_generator --user solvebio --project solvebio-python 58 | 59 | release: clean dist 60 | twine upload dist/* 61 | 62 | 63 | .PHONY: $(PHONY) 64 | -------------------------------------------------------------------------------- /solvebio/test/test_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from .helper import SolveBioTestCase 4 | from unittest import TestCase 5 | from solvebio.utils.files import check_gzip_path, separate_filename_extension 6 | 7 | FILENAME_PARAMS = [ 8 | { 9 | "filename": "test.txt", 10 | "base": "test", 11 | "ext": ".txt", 12 | "compression": "" 13 | }, 14 | { 15 | "filename": "/path/to/test.txt", 16 | "base": "/path/to/test", 17 | "ext": ".txt", 18 | "compression": "" 19 | }, 20 | { 21 | "filename": "test.txt.gz", 22 | "base": "test", 23 | "ext": ".txt", 24 | "compression": ".gz" 25 | }, 26 | ] 27 | 28 | 29 | class FilenameTests(SolveBioTestCase, TestCase): 30 | 31 | def test_extract_filename(self): 32 | for params in FILENAME_PARAMS: 33 | base, ext, compression = separate_filename_extension(params['filename']) 34 | self.assertEqual(base, params['base']) 35 | self.assertEqual(ext, params['ext']) 36 | self.assertEqual(compression, params['compression']) 37 | 38 | 39 | class GzipTest(SolveBioTestCase): 40 | 41 | def test_gzip_file(self): 42 | path = os.path.join(os.path.dirname(__file__), "data") 43 | for yes_gzip in ['some_export.json.gz', 44 | 'sample.vcf.gz']: 45 | path = os.path.join(path, yes_gzip) 46 | self.assertTrue(check_gzip_path(path), path) 47 | 48 | for non_gzip in ['sample2.vcf', 49 | 'test_export.json', 50 | 'test_export.csv', 51 | 'test_export.xlsx']: 52 | path = os.path.join(path, non_gzip) 53 | self.assertFalse(check_gzip_path(path), path) 54 | -------------------------------------------------------------------------------- /solvebio/test/helper.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import unittest 5 | 6 | import solvebio 7 | 8 | 9 | class SolveBioTestCase(unittest.TestCase): 10 | # 'solvebio:public:/HGNC/3.3.0-2020-10-29/HGNC' 11 | TEST_DATASET_FULL_PATH = 'quartzbio:Public:/HGNC/3.3.1-2021-08-25/HGNC' 12 | # 'solvebio:public:/ClinVar/5.1.0-20200720/Variants-GRCH38' 13 | TEST_DATASET_FULL_PATH_2 = 'quartzbio:Public:/ClinVar/5.2.0-20210110/Variants-GRCH38' 14 | # 'solvebio:public:/HGNC/3.3.0-2020-10-29/hgnc_1000_rows.txt' 15 | TEST_FILE_FULL_PATH = 'quartzbio:Public:/HGNC/3.3.1-2021-08-25/HGNC-3-3-1-2021-08-25-HGNC-1904014068027535892-20230418174248.json.gz' # noqa 16 | TEST_LARGE_TSV_FULL_PATH = '' 17 | 18 | def setUp(self): 19 | super(SolveBioTestCase, self).setUp() 20 | api_key = os.environ.get('SOLVEBIO_API_KEY', None) 21 | api_host = os.environ.get('SOLVEBIO_API_HOST', None) 22 | self.client = solvebio.SolveClient(host=api_host, token=api_key) 23 | 24 | def check_response(self, response, expect, msg): 25 | subset = [(key, response[key]) for 26 | key in [x[0] for x in expect]] 27 | self.assertEqual(subset, expect) 28 | 29 | # Python < 2.7 compatibility 30 | def assertRaisesRegexp(self, exception, regexp, callable, *args, **kwargs): 31 | try: 32 | callable(*args, **kwargs) 33 | except exception as err: 34 | if regexp is None: 35 | return True 36 | 37 | if isinstance(regexp, str): 38 | regexp = re.compile(regexp) 39 | if not regexp.search(str(err)): 40 | raise self.failureException('\'%s\' does not match \'%s\'' % 41 | (regexp.pattern, str(err))) 42 | else: 43 | raise self.failureException( 44 | '%s was not raised' % (exception.__name__,)) 45 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from setuptools import setup, find_packages 3 | 4 | import sys 5 | import warnings 6 | 7 | VERSION = 'undefined' 8 | install_requires = ['pyprind', 'requests>=2.0.0', 'urllib3>=1.26.0'] 9 | extra = {} 10 | 11 | with open('solvebio/version.py') as f: 12 | for row in f.readlines(): 13 | if row.startswith('VERSION'): 14 | exec(row) 15 | 16 | # solvebio-recipes requires additional packages 17 | recipes_requires = [ 18 | 'pyyaml==5.3.1', 19 | 'click==7.1.2', 20 | 'ruamel.yaml==0.16.12' 21 | ] 22 | extras_requires = { 23 | "recipes": recipes_requires 24 | } 25 | 26 | with open('README.md') as f: 27 | long_description = f.read() 28 | 29 | setup( 30 | name='solvebio', 31 | version=VERSION, 32 | description='The SolveBio Python client (DEPRECATED)', 33 | long_description=long_description, 34 | long_description_content_type='text/markdown', 35 | python_requires='>=3.8', 36 | author='Solve, Inc.', 37 | author_email='contact@solvebio.com', 38 | url='https://github.com/solvebio/solvebio-python', 39 | packages=find_packages(), 40 | package_dir={'solvebio': 'solvebio', 'recipes': 'recipes'}, 41 | test_suite='solvebio.test', 42 | include_package_data=True, 43 | install_requires=install_requires, 44 | platforms='any', 45 | extras_require=extras_requires, 46 | entry_points={ 47 | 'console_scripts': ['solvebio = solvebio.cli.main:main', 48 | 'solvebio-recipes = recipes.sync_recipes:sync_recipes'] 49 | }, 50 | classifiers=[ 51 | 'Development Status :: 7 - Inactive', 52 | 'Intended Audience :: Science/Research', 53 | 'Operating System :: OS Independent', 54 | 'Programming Language :: Python', 55 | 'Topic :: Software Development :: Libraries :: Python Modules', 56 | 'Topic :: Scientific/Engineering :: Bio-Informatics' 57 | ], 58 | **extra 59 | ) 60 | -------------------------------------------------------------------------------- /solvebio/utils/files.py: -------------------------------------------------------------------------------- 1 | import mimetypes 2 | import os 3 | import pathlib 4 | 5 | COMPRESSIONS = ('.gz', '.gzip', '.bz2', '.z', '.zip', '.bgz') 6 | 7 | 8 | def separate_filename_extension(filename): 9 | """Separates filename into base name 10 | and extension while handling compressed 11 | filename extensions. 12 | 13 | Args: 14 | filename (str): A filename, can also 15 | be a full filepath. 16 | Returns: 17 | base name (str): Base filename (or path) 18 | without extension. 19 | extension (str): File extension beginning 20 | with leading period (e.g. '.txt'). 21 | compression (str): Compression extension 22 | with leading period (e.x. '.gz'). 23 | """ 24 | base_filename, file_extension = os.path.splitext(filename) 25 | if file_extension in COMPRESSIONS and "." in base_filename: 26 | compression = file_extension 27 | base_filename, file_extension = os.path.splitext(base_filename) 28 | else: 29 | compression = '' 30 | return base_filename, file_extension, compression 31 | 32 | 33 | def check_gzip_path(file_path): 34 | """Check if we have a gzipped file path""" 35 | _, ftype = mimetypes.guess_type(file_path) 36 | return ftype == 'gzip' 37 | 38 | 39 | def get_home_dir(): 40 | try: 41 | # Python 3.5+ 42 | from pathlib import Path 43 | return str(Path.home()) 44 | except: 45 | from os.path import expanduser 46 | return expanduser("~") 47 | 48 | 49 | def edp_path_join(*edp_paths): 50 | p = str(pathlib.PurePosixPath(*edp_paths)) 51 | 52 | if not p.endswith("/"): 53 | p += "/" 54 | 55 | return p 56 | 57 | 58 | def edp_path(path: str): 59 | """normalize OS path to EDP remote path""" 60 | 61 | win_path = pathlib.PureWindowsPath(path) 62 | posix_path = pathlib.PurePosixPath('/', *win_path.parts) 63 | return posix_path.as_posix().removeprefix("/") 64 | 65 | -------------------------------------------------------------------------------- /solvebio/cli/ipython_init.py: -------------------------------------------------------------------------------- 1 | # Add common solvebio classes and methods our namespace here so that 2 | # inside the ipython shell users don't have run imports 3 | import solvebio # noqa 4 | from solvebio import login # noqa 5 | from solvebio import Annotator # noqa 6 | from solvebio import Application # noqa 7 | from solvebio import Beacon # noqa 8 | from solvebio import BeaconSet # noqa 9 | from solvebio import BatchQuery # noqa 10 | from solvebio import Dataset # noqa 11 | from solvebio import DatasetCommit # noqa 12 | from solvebio import DatasetExport # noqa 13 | from solvebio import DatasetField # noqa 14 | from solvebio import DatasetImport # noqa 15 | from solvebio import DatasetMigration # noqa 16 | from solvebio import DatasetTemplate # noqa 17 | from solvebio import Expression # noqa 18 | from solvebio import Filter # noqa 19 | from solvebio import GenomicFilter # noqa 20 | from solvebio import Manifest # noqa 21 | from solvebio import Object # noqa 22 | from solvebio import Query # noqa 23 | from solvebio import SolveClient # noqa 24 | from solvebio import SolveError # noqa 25 | from solvebio import User # noqa 26 | from solvebio import Vault # noqa 27 | from solvebio import Task # noqa 28 | from solvebio import VaultSyncTask # noqa 29 | from solvebio import ObjectCopyTask # noqa 30 | from solvebio import SavedQuery # noqa 31 | from solvebio import DatasetRestoreTask # noqa 32 | from solvebio import DatasetSnapshotTask # noqa 33 | from solvebio import GlobalSearch # noqa 34 | from solvebio import Group # noqa 35 | from solvebio import print_deprecation_notice # noqa 36 | from solvebio.utils.printing import pager # noqa 37 | 38 | # Add some convenience functions to the interactive shell 39 | from solvebio.cli.auth import logout # noqa 40 | from solvebio.cli.auth import whoami # noqa 41 | from solvebio.cli.auth import get_credentials # noqa 42 | 43 | # Show deprecation notice when starting IPython shell 44 | print_deprecation_notice() 45 | whoami() 46 | -------------------------------------------------------------------------------- /solvebio/resource/dataset_restore_task.py: -------------------------------------------------------------------------------- 1 | from .apiresource import ListableAPIResource 2 | from .apiresource import CreateableAPIResource 3 | from .solveobject import convert_to_solve_object 4 | from .task import Task 5 | 6 | import time 7 | 8 | 9 | class DatasetRestoreTask(CreateableAPIResource, ListableAPIResource): 10 | """ 11 | DatasetRestoreTask represents the task to restore an archived dataset 12 | """ 13 | RESOURCE_VERSION = 2 14 | 15 | LIST_FIELDS = ( 16 | ('id', 'ID'), 17 | ('dataset_id', 'Dataset'), 18 | ('vault_id', 'Vault'), 19 | ('status', 'Status'), 20 | ('created_at', 'Created'), 21 | ) 22 | 23 | @property 24 | def dataset(self): 25 | response = self._client.get(self['dataset']['url'], {}) 26 | return convert_to_solve_object(response, client=self._client) 27 | 28 | def follow(self, loop=True, sleep_seconds=Task.SLEEP_WAIT_DEFAULT): 29 | if self.status == 'queued': 30 | print("Waiting for Dataset Restore task (id = {0}) to start..." 31 | .format(self.id)) 32 | 33 | status = self.status 34 | while self.status in ['queued', 'running']: 35 | if self.status != status: 36 | print("Restore is now {0} (was {1})" 37 | .format(self.status, status)) 38 | status = self.status 39 | 40 | if self.status == 'running': 41 | progress = self.metadata.get('progress', {}).get('restore') 42 | if progress: 43 | print("Restore '{0}' is {1} (Progress: {2}%% completed)".format(self.id, self.status, progress)) 44 | else: 45 | print("Restore '{0}' is {1}".format(self.id, self.status)) 46 | 47 | if not loop: 48 | return 49 | 50 | time.sleep(sleep_seconds) 51 | self.refresh() 52 | 53 | if self.status == 'completed': 54 | print('Restore complete! Dataset is now available') 55 | -------------------------------------------------------------------------------- /solvebio/resource/dataset_snapshot_task.py: -------------------------------------------------------------------------------- 1 | from .apiresource import ListableAPIResource 2 | from .apiresource import CreateableAPIResource 3 | from .solveobject import convert_to_solve_object 4 | from .task import Task 5 | 6 | import time 7 | 8 | 9 | class DatasetSnapshotTask(CreateableAPIResource, ListableAPIResource): 10 | """ 11 | DatasetSnapshotTask represents the task to snapshot and archive a dataset 12 | """ 13 | RESOURCE_VERSION = 2 14 | 15 | LIST_FIELDS = ( 16 | ('id', 'ID'), 17 | ('dataset_id', 'Dataset'), 18 | ('vault_id', 'Vault'), 19 | ('status', 'Status'), 20 | ('created_at', 'Created'), 21 | ) 22 | 23 | @property 24 | def dataset(self): 25 | response = self._client.get(self['dataset']['url'], {}) 26 | return convert_to_solve_object(response, client=self._client) 27 | 28 | def follow(self, loop=True, sleep_seconds=Task.SLEEP_WAIT_DEFAULT): 29 | if self.status == 'queued': 30 | print("Waiting for Dataset Snapshot task (id = {0}) to start..." 31 | .format(self.id)) 32 | 33 | status = self.status 34 | while self.status in ['queued', 'running']: 35 | if self.status != status: 36 | print("Snapshot is now {0} (was {1})" 37 | .format(self.status, status)) 38 | status = self.status 39 | 40 | if self.status == 'running': 41 | progress = self.metadata.get('progress', {}).get('snapshot') 42 | if progress: 43 | print("Snapshot '{0}' is {1} (Progress: {2}%% completed)".format(self.id, self.status, progress)) 44 | else: 45 | print("Snapshot '{0}' is {1}".format(self.id, self.status)) 46 | 47 | if not loop: 48 | return 49 | 50 | time.sleep(sleep_seconds) 51 | self.refresh() 52 | 53 | if self.status == 'completed': 54 | print('Snapshot complete! Dataset is now archived') 55 | -------------------------------------------------------------------------------- /solvebio/errors.py: -------------------------------------------------------------------------------- 1 | import logging 2 | logger = logging.getLogger('solvebio') 3 | 4 | 5 | class NotFoundError(Exception): 6 | pass 7 | 8 | 9 | class FileUploadError(Exception): 10 | pass 11 | 12 | 13 | class SolveError(Exception): 14 | """Exceptions tailored to the kinds of errors from a SolveBio API 15 | request""" 16 | default_message = ('Unexpected error communicating with SolveBio. ' 17 | 'If this problem persists, let us know at ' 18 | 'support@solvebio.com.') 19 | 20 | def __init__(self, message=None, response=None): 21 | self.json_body = {} 22 | self.status_code = None 23 | self.message = message or self.default_message 24 | self.field_errors = [] 25 | self.response = response 26 | 27 | if response is not None: 28 | self.status_code = response.status_code 29 | try: 30 | self.json_body = response.json() 31 | logger.debug( 32 | 'API Response (%d): %s' 33 | % (self.status_code, self.json_body)) 34 | except: 35 | logger.debug( 36 | 'API Response (%d): Response does not contain JSON.' 37 | % self.status_code) 38 | 39 | if self.status_code == 400: 40 | self.message = 'Bad Request ({})'.format(response.url) 41 | elif response.status_code == 401: 42 | self.message = '401 Unauthorized ({})'.format(response.url) 43 | elif response.status_code == 403: 44 | self.message = '403 Forbidden ({})'.format(response.url) 45 | elif response.status_code == 404: 46 | self.message = '404 Not Found ({})'.format(response.url) 47 | 48 | # Handle other keys 49 | for k, v in list(self.json_body.items()): 50 | if k in ["non_field_errors", "detail"]: 51 | self.message += '\nError: ' 52 | else: 53 | self.message += '\nError (%s): ' % k 54 | 55 | # can be a list, dict, string 56 | self.message += str(v) 57 | 58 | def __str__(self): 59 | return self.message 60 | -------------------------------------------------------------------------------- /solvebio/test/test_apiresource.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from .helper import SolveBioTestCase 4 | 5 | 6 | class APIResourceTests(SolveBioTestCase): 7 | 8 | def test_apiresource_iteration(self): 9 | public_vault = self.client.Vault.get_by_full_path('quartzbio:public') 10 | n_folders = len(list(public_vault.folders(depth=0))) 11 | 12 | folder_iter = public_vault.folders(depth=0) 13 | for i, j in enumerate(folder_iter): 14 | pass 15 | self.assertTrue(i == n_folders - 1) 16 | 17 | # Iterating again should be the same 18 | for i, j in enumerate(folder_iter): 19 | pass 20 | self.assertTrue(i == n_folders - 1) 21 | 22 | def test_apiresource_serialize_metadata(self): 23 | folder_no_metadata = self.client.Object.\ 24 | get_or_create_by_full_path('~/{}'.format(uuid.uuid4()), object_type='folder') 25 | 26 | metadata = folder_no_metadata.metadata 27 | foo_tuple = ('foo', 'bar') 28 | 29 | # foo key-value pair is not in 'metadata' object 30 | self.assertFalse(foo_tuple in metadata.items()) 31 | 32 | metadata['foo'] = 'bar' 33 | params = folder_no_metadata.serialize(folder_no_metadata) 34 | 35 | # Test that foo key-value pair is set as value to 'metadata' 36 | # key in 'params' dict that will be serialized 37 | self.assertTrue(foo_tuple in params['metadata'].items()) 38 | 39 | folder_with_metadata = self.client.Object.\ 40 | get_or_create_by_full_path('~/{}'.format(uuid.uuid4()), 41 | object_type='folder', 42 | metadata=dict([foo_tuple])) 43 | 44 | metadata = folder_with_metadata.metadata 45 | foo_1_tuple = ('foo1', 'bar1') 46 | 47 | # foo_1 key-value pair is not in 'metadata' object 48 | self.assertFalse(foo_1_tuple in metadata.items()) 49 | 50 | metadata['foo1'] = 'bar1' 51 | 52 | params = folder_with_metadata.serialize(folder_with_metadata) 53 | params_items = params['metadata'].items() 54 | 55 | # Test that both foo and foo_1 key-value pairs are set as value to 'metadata' 56 | # key in 'params' dict that will be serialized 57 | self.assertTrue(foo_tuple in params_items) 58 | self.assertTrue(foo_1_tuple in params_items) 59 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/usage.py: -------------------------------------------------------------------------------- 1 | import dash 2 | import dash_html_components as html 3 | import dash_core_components as dcc 4 | import flask 5 | 6 | from solvebio.contrib.dash import SolveBioDash 7 | 8 | 9 | # Initialize the Dash app with SolveBio auth. 10 | app = SolveBioDash( 11 | name=__name__, 12 | title='Example Dash App', 13 | app_url='http://localhost:8050', 14 | client_id='zxpjuufs7k65f26zyq3vq4hqqw2dmu13x77p7qg2', 15 | solvebio_url='http://my.solvebio.com') 16 | 17 | 18 | def current_user(): 19 | if app.auth: 20 | user = flask.g.client.User.retrieve() 21 | return [ 22 | html.Div(children='Logged-in as: {}'.format(user.full_name)), 23 | html.A('Log out', href='/_dash-logout') 24 | ] 25 | else: 26 | return [ 27 | html.P('(SolveBio Auth not configured)') 28 | ] 29 | 30 | 31 | def layout(): 32 | return html.Div([ 33 | html.H1('Welcome to your Dash+SolveBio app'), 34 | html.P(current_user()), 35 | dcc.Dropdown( 36 | id='dropdown', 37 | options=[{'label': i, 'value': i} for i in ['A', 'B']], 38 | value='A' 39 | ), 40 | dcc.Graph(id='graph') 41 | ], className="container") 42 | 43 | 44 | app.layout = html.Div([ 45 | dcc.Location(id='url', refresh=False), 46 | html.Div(id='page-content') 47 | ]) 48 | 49 | 50 | # 51 | # Your Dash app callback functions 52 | # 53 | 54 | @app.callback( 55 | dash.dependencies.Output('page-content', 'children'), 56 | [dash.dependencies.Input('url', 'pathname')]) 57 | def display_page(pathname): 58 | """Render the layout depending on the pathname.""" 59 | return layout() 60 | 61 | 62 | @app.callback( 63 | dash.dependencies.Output('graph', 'figure'), 64 | [dash.dependencies.Input('dropdown', 'value')]) 65 | def update_graph(dropdown_value): 66 | return { 67 | 'layout': { 68 | 'title': 'Graph of {}'.format(dropdown_value), 69 | 'margin': { 70 | 'l': 20, 71 | 'b': 20, 72 | 'r': 10, 73 | 't': 60 74 | } 75 | }, 76 | 'data': [{'x': [1, 2, 3], 'y': [4, 1, 2]}] 77 | } 78 | 79 | 80 | app.css.append_css({ 81 | "external_url": "https://codepen.io/chriddyp/pen/bWLwgP.css" 82 | }) 83 | 84 | if __name__ == '__main__': 85 | app.run_server(debug=True) 86 | -------------------------------------------------------------------------------- /solvebio/resource/datasetexport.py: -------------------------------------------------------------------------------- 1 | from .apiresource import ListableAPIResource 2 | from .apiresource import DeletableAPIResource 3 | from .apiresource import CreateableAPIResource 4 | from .apiresource import DownloadableAPIResource 5 | from .task import Task 6 | from .solveobject import convert_to_solve_object 7 | 8 | import time 9 | 10 | 11 | class DatasetExport(CreateableAPIResource, ListableAPIResource, 12 | DownloadableAPIResource, DeletableAPIResource): 13 | """ 14 | DatasetExport represent an export task that takes 15 | a Dataset or filtered Dataset (Query) and exports 16 | the contents to a flat file (CSV, JSON, or XLSX). 17 | 18 | For interactive use, DatasetExport can be "followed" to watch 19 | the progression of the task. 20 | """ 21 | RESOURCE_VERSION = 2 22 | 23 | LIST_FIELDS = ( 24 | ('id', 'ID'), 25 | ('documents_count', 'Records'), 26 | ('format', 'Format'), 27 | ('status', 'Status'), 28 | ('created_at', 'Created'), 29 | ) 30 | 31 | @property 32 | def dataset(self): 33 | response = self._client.get(self['dataset']['url'], {}) 34 | return convert_to_solve_object(response, client=self._client) 35 | 36 | def follow(self, loop=True, sleep_seconds=Task.SLEEP_WAIT_DEFAULT): 37 | if self.status == 'queued': 38 | print("Waiting for export (id = {0}) to start..." 39 | .format(self.id)) 40 | 41 | export_status = self.status 42 | while self.status in ['queued', 'running']: 43 | if self.status != export_status: 44 | print("Export is now {0} (was {1})" 45 | .format(self.status, export_status)) 46 | export_status = self.status 47 | 48 | if self.status == 'running': 49 | print("Export '{0}' is {1}: {2}/{3} records exported" 50 | .format(self.id, 51 | self.status, 52 | self.metadata.get('progress', {}).get('processed_records', 0), # noqa 53 | self.documents_count)) 54 | 55 | if not loop: 56 | return 57 | 58 | time.sleep(sleep_seconds) 59 | self.refresh() 60 | 61 | if self.status == 'completed': 62 | print('Export complete! Run ' 63 | '.download(path=) to download.') 64 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/solvebio_dash.py: -------------------------------------------------------------------------------- 1 | import dash 2 | import flask 3 | 4 | import os 5 | from random import randint 6 | 7 | import solvebio 8 | 9 | from .solvebio_auth import SolveBioAuth 10 | 11 | 12 | class SolveBioDash(dash.Dash): 13 | APP_URL = os.environ.get('APP_URL', 'http://127.0.0.1:8050') 14 | SOLVEBIO_URL = os.environ.get('SOLVEBIO_URL', 'http://my.solvebio.com') 15 | SECRET_KEY = os.environ.get('SECRET_KEY', str(randint(0, 1000000))) 16 | 17 | def __init__(self, name, *args, **kwargs): 18 | # Default HTML page title 19 | self.title = kwargs.pop('title', name) 20 | 21 | app_url = kwargs.pop('app_url', self.APP_URL) 22 | solvebio_url = kwargs.pop('solvebio_url', self.SOLVEBIO_URL) 23 | api_host = kwargs.pop('api_host', None) # Extract api_host before passing to Dash 24 | 25 | # OAuth2 credentials 26 | client_id = kwargs.pop('client_id', 27 | os.environ.get('CLIENT_ID')) 28 | client_secret = kwargs.pop('client_secret', 29 | os.environ.get('CLIENT_SECRET')) 30 | grant_type = kwargs.pop('grant_type', None) 31 | salt = kwargs.pop('salt', None) or name 32 | 33 | server = flask.Flask(name) 34 | server.secret_key = kwargs.pop('secret_key', self.SECRET_KEY) 35 | kwargs['server'] = server 36 | 37 | super(SolveBioDash, self).__init__(name, *args, **kwargs) 38 | 39 | if client_id: 40 | self.auth = SolveBioAuth( 41 | self, 42 | app_url, 43 | client_id, 44 | salt=salt, 45 | client_secret=client_secret, 46 | grant_type=grant_type, 47 | solvebio_url=solvebio_url, 48 | api_host=api_host) 49 | else: 50 | self.auth = None 51 | print("WARNING: No SolveBio client ID found. " 52 | "Your app (but not your data) will be publicly accessible.") 53 | 54 | @server.before_request 55 | def set_solve_client(): 56 | oauth_token = flask.request.cookies.get( 57 | SolveBioAuth.TOKEN_COOKIE_NAME) 58 | if oauth_token: 59 | flask.g.client = solvebio.SolveClient( 60 | token=oauth_token, token_type='Bearer') 61 | else: 62 | # Use global credentials (if any) 63 | flask.g.client = solvebio.SolveClient() 64 | -------------------------------------------------------------------------------- /solvebio/contrib/streamlit/app_example.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import pandas as pd 3 | 4 | from dotenv import load_dotenv 5 | 6 | # Load environment variables from .env file 7 | load_dotenv() 8 | 9 | from solvebio_streamlit import SolveBioStreamlit 10 | # from solvebio.contrib.streamlit.solvebio_streamlit import SolveBioStreamlit 11 | 12 | 13 | @st.cache 14 | def get_personal_vault_items(vault): 15 | """Getting items from personal vaulet (caching using Streamlit cache mechanism)""" 16 | 17 | files = vault.files() 18 | folders = vault.folders() 19 | datasets = vault.datasets() 20 | 21 | return files, folders, datasets 22 | 23 | 24 | def streamlit_demo_app(): 25 | """Streamlit demo app 26 | 27 | It fetches the SolveClient (initialised with OAuth2 token) from Streamlit session state 28 | and makes API call through that client. 29 | """ 30 | 31 | # Getting the sovle client from the Streamlit session state 32 | solvebio_client = st.session_state.solvebio_client 33 | 34 | st.title("Solvebio app") 35 | 36 | # SolveBio user 37 | user = solvebio_client.User.retrieve() 38 | st.header("{}'s personal vault overview:".format(user["first_name"])) 39 | 40 | # Personal vault 41 | vault = solvebio_client.Vault.get_personal_vault() 42 | st.write(vault["description"]) 43 | 44 | files, folders, datasets = get_personal_vault_items(vault) 45 | 46 | # Visualising the stats from personal vault 47 | data = { 48 | "Number of files": [files["total"]], 49 | "Number of datasets": [datasets["total"]], 50 | "Number of folders": [folders["total"]], 51 | } 52 | chart_data = pd.DataFrame.from_dict( 53 | data, columns=["Number of items"], orient="index" 54 | ) 55 | st.write(chart_data) 56 | st.bar_chart(chart_data) 57 | 58 | # Listing items from personal vault 59 | st.subheader("List selected items") 60 | option = st.radio("Select SolveBio platform:", ("Files", "Folders", "Datasets")) 61 | if option == "Files": 62 | for item in files: 63 | st.markdown("- {}".format(item["filename"])) 64 | elif option == "Folders": 65 | for item in folders: 66 | st.markdown("- {}".format(item["filename"])) 67 | elif option == "Datasets": 68 | for item in datasets: 69 | st.markdown("- {}".format(item["filename"])) 70 | 71 | 72 | # Wrapping Streamlit app with SolveBio OAuth2 73 | secure_app = SolveBioStreamlit() 74 | secure_app.wrap(streamlit_app=streamlit_demo_app) 75 | -------------------------------------------------------------------------------- /solvebio/resource/group.py: -------------------------------------------------------------------------------- 1 | from .apiresource import ListableAPIResource 2 | from .apiresource import DeletableAPIResource 3 | from .apiresource import UpdateableAPIResource 4 | from .apiresource import CreateableAPIResource 5 | from .solveobject import convert_to_solve_object 6 | 7 | 8 | class Group(CreateableAPIResource, ListableAPIResource, 9 | UpdateableAPIResource, DeletableAPIResource): 10 | """ 11 | A Group represents a group of users with shared permissions for a vault. 12 | """ 13 | RESOURCE_VERSION = 1 14 | 15 | LIST_FIELDS = ( 16 | ('id', 'ID'), 17 | ('name', 'Name'), 18 | ('memberships_count', 'Members'), 19 | ('vaults_count', 'Vaults'), 20 | ('role', 'Role'), 21 | ('description', 'Description'), 22 | ) 23 | 24 | def members(self, **params): 25 | response = self._client.get(self.memberships_url, params) 26 | results = convert_to_solve_object(response, client=self._client) 27 | results.set_tabulate( 28 | ['id', 'full_name', 'username', 'email', 'role'], 29 | headers=['ID', 'Full Name', 'Username', 'Email', 'Role'], 30 | aligns=['left', 'left', 'left', 'left', 'left'], sort=False) 31 | 32 | return results 33 | 34 | def _get_vaults(self, **params): 35 | response = self._client.get(self.vaults_url, params) 36 | return convert_to_solve_object(response, client=self._client) 37 | 38 | def vaults(self, **params): 39 | results = self._get_vaults(**params) 40 | results.set_tabulate( 41 | ['id', 'name', 'permission', 'vault_permissions'], 42 | headers=['ID', 'Vault', 'Permission', 'Permissions Detail'], 43 | aligns=['left', 'left', 'left', 'left'], sort=False) 44 | 45 | return results 46 | 47 | def datasets(self, **params): 48 | from . import Object 49 | vaults = self._get_vaults(**params) 50 | objects_url = Object.class_url() + '?object_type=dataset&' + \ 51 | '&'.join(['vault_id={0}'.format(v.id) for v in vaults]) 52 | response = self._client.get(objects_url, params) 53 | datasets = convert_to_solve_object(response, client=self._client) 54 | datasets.set_tabulate( 55 | ['id', 'vault_name', 'path', 56 | 'dataset_documents_count', 'dataset_description'], 57 | headers=['ID', 'Vault', 'Path', 'Documents', 'Description'], 58 | aligns=['left', 'left', 'left', 'left', 'left'], sort=False) 59 | 60 | return datasets 61 | -------------------------------------------------------------------------------- /examples/download_vault_folder.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import solvebio 4 | 5 | 6 | def download_vault_folder(remote_path, local_path, dry_run=False, force=False): 7 | """Recursively downloads a folder in a vault to a local directory. 8 | Only downloads files, not datasets.""" 9 | 10 | local_path = os.path.normpath(os.path.expanduser(local_path)) 11 | if not os.access(local_path, os.W_OK): 12 | raise Exception( 13 | 'Write access to local path ({}) is required' 14 | .format(local_path)) 15 | 16 | full_path, path_dict = solvebio.Object.validate_full_path(remote_path) 17 | vault = solvebio.Vault.get_by_full_path(path_dict['vault']) 18 | print('Downloading all files from {} to {}'.format(full_path, local_path)) 19 | 20 | if path_dict['path'] == '/': 21 | parent_object_id = None 22 | else: 23 | parent_object = solvebio.Object.get_by_full_path( 24 | remote_path, assert_type='folder') 25 | parent_object_id = parent_object.id 26 | 27 | # Scan the folder for all sub-folders and create them locally 28 | print('Creating local directory structure at: {}'.format(local_path)) 29 | if not os.path.exists(local_path): 30 | if not dry_run: 31 | os.makedirs(local_path) 32 | 33 | folders = vault.folders(parent_object_id=parent_object_id) 34 | for f in folders: 35 | path = os.path.normpath(local_path + f.path) 36 | if not os.path.exists(path): 37 | print('Creating folder: {}'.format(path)) 38 | if not dry_run: 39 | os.makedirs(path) 40 | 41 | files = vault.files(parent_object_id=parent_object_id) 42 | for f in files: 43 | path = os.path.normpath(local_path + f.path) 44 | if os.path.exists(path): 45 | if force: 46 | # Delete the local copy 47 | print('Deleting local file (force download): {}'.format(path)) 48 | if not dry_run: 49 | os.remove(path) 50 | else: 51 | print('Skipping file (already exists): {}'.format(path)) 52 | continue 53 | 54 | print('Downloading file: {}'.format(path)) 55 | if not dry_run: 56 | f.download(path) 57 | 58 | 59 | def main(): 60 | if len(sys.argv) < 3: 61 | print('Usage: {} '.format(sys.argv[0])) 62 | sys.exit(1) 63 | 64 | solvebio.login() 65 | download_vault_folder(sys.argv[1], sys.argv[2], dry_run=True) 66 | 67 | 68 | if __name__ == '__main__': 69 | main() 70 | -------------------------------------------------------------------------------- /solvebio/cli/auth.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import solvebio 4 | from ..client import client 5 | from .credentials import get_credentials 6 | from .credentials import save_credentials 7 | from .credentials import delete_credentials 8 | 9 | 10 | def login_and_save_credentials(*args): 11 | """ 12 | CLI command to login and persist credentials to a file 13 | """ 14 | args = args[0] 15 | 16 | solvebio.login( 17 | api_host=args.api_host, 18 | api_key=args.api_key, 19 | access_token=args.access_token, 20 | # name=args.name, 21 | # version=args.version, 22 | debug=args.debug, 23 | ) 24 | 25 | # Print information about the current user 26 | if not client.is_logged_in(): 27 | print("login: client is not logged in!") 28 | solvebio.print_deprecation_notice() 29 | 30 | # Verify if user has provided the wrong credentials file 31 | if client._host: 32 | if suggested_host := client.validate_host_is_www_url(client._host): 33 | print( 34 | f"Provided API host is: `{client._host}`. " 35 | f"Did you perhaps mean `{suggested_host}`?" 36 | ) 37 | return 38 | 39 | user = client.whoami() 40 | print_user(user) 41 | 42 | # fixme: how to detect if login was successful 43 | save_credentials( 44 | user["email"].lower(), 45 | client._auth.token, 46 | client._auth.token_type, 47 | solvebio.get_api_host(), 48 | ) 49 | print("Updated local credentials file.") 50 | 51 | 52 | def logout(*args): 53 | """ 54 | Delete's the user's locally-stored credentials. 55 | """ 56 | if get_credentials(): 57 | delete_credentials() 58 | print("You have been logged out.") 59 | else: 60 | print("You are not logged-in.") 61 | solvebio.print_deprecation_notice() 62 | 63 | 64 | def whoami(*args, **kwargs): 65 | """ 66 | Prints information about the current user. 67 | Assumes the user is already logged-in. 68 | """ 69 | try: 70 | user = client.whoami() 71 | except Exception as e: 72 | print("{} (code: {})".format(e.message, e.status_code)) 73 | else: 74 | print_user(user) 75 | 76 | 77 | def print_user(user): 78 | """ 79 | Prints information about the current user. 80 | """ 81 | email = user["email"] 82 | domain = user["account"]["domain"] 83 | print( 84 | f'You are logged-in to the "{domain}" domain as {email}' 85 | f" (server: {solvebio.get_api_host()})." 86 | ) 87 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/tests/utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | TIMEOUT = 20 # Seconds 5 | 6 | 7 | def clean_history(driver, domains): 8 | temp = driver.get_location() 9 | for domain in domains: 10 | driver.open(domain) 11 | driver.delete_all_visible_cookies() 12 | driver.open(temp) 13 | 14 | 15 | def invincible(func): 16 | def wrap(): 17 | try: 18 | return func() 19 | except: 20 | pass 21 | return wrap 22 | 23 | 24 | def switch_windows(driver): 25 | new_window_handle = None 26 | while not new_window_handle: 27 | for handle in driver.window_handles: 28 | if handle != driver.current_window_handle: 29 | new_window_handle = handle 30 | break 31 | driver.switch_to.window(new_window_handle) 32 | return new_window_handle 33 | 34 | 35 | class WaitForTimeout(Exception): 36 | """This should only be raised inside the `wait_for` function.""" 37 | pass 38 | 39 | 40 | def wait_for(condition_function, get_message=lambda: '', *args, **kwargs): 41 | """ 42 | Waits for condition_function to return True or raises WaitForTimeout. 43 | :param (function) condition_function: Should return True on success. 44 | :param args: Optional args to pass to condition_function. 45 | :param kwargs: Optional kwargs to pass to condition_function. 46 | if `timeout` is in kwargs, it will be used to override TIMEOUT 47 | :raises: WaitForTimeout If condition_function doesn't return True in time. 48 | Usage: 49 | def get_element(selector): 50 | # some code to get some element or return a `False`-y value. 51 | selector = '.js-plotly-plot' 52 | try: 53 | wait_for(get_element, selector) 54 | except WaitForTimeout: 55 | self.fail('element never appeared...') 56 | plot = get_element(selector) # we know it exists. 57 | """ 58 | def wrapped_condition_function(): 59 | """We wrap this to alter the call base on the closure.""" 60 | if args and kwargs: 61 | return condition_function(*args, **kwargs) 62 | if args: 63 | return condition_function(*args) 64 | if kwargs: 65 | return condition_function(**kwargs) 66 | return condition_function() 67 | 68 | if 'timeout' in kwargs: 69 | timeout = kwargs['timeout'] 70 | del kwargs['timeout'] 71 | else: 72 | timeout = TIMEOUT 73 | 74 | start_time = time.time() 75 | while time.time() < start_time + timeout: 76 | if wrapped_condition_function(): 77 | return True 78 | time.sleep(0.5) 79 | 80 | raise WaitForTimeout(get_message()) 81 | -------------------------------------------------------------------------------- /solvebio/resource/manifest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import glob 5 | 6 | from urllib.parse import urlparse 7 | 8 | import solvebio 9 | 10 | 11 | class Manifest(object): 12 | """ 13 | Manifests aren't strictly resources, they represent a list 14 | of remote files (URLs) with additional information that 15 | can be used for validation (size and MD5). 16 | """ 17 | manifest = None 18 | 19 | def __init__(self): 20 | self.manifest = {'files': []} 21 | 22 | def add_file(self, path, **kwargs): 23 | default_upload_path = solvebio.Vault.get_or_create_uploads_path() 24 | vault = solvebio.Vault.get_personal_vault() 25 | print("Uploading file: {0} to {1}".format(path, default_upload_path)) 26 | file_ = solvebio.Object.upload_file(path, default_upload_path, 27 | vault.full_path) 28 | print("Successfuly uploaded file {0} (id:{1} size:{2} md5:{3})" 29 | .format(path, file_.id, file_.size, file_.md5)) 30 | 31 | self.manifest['files'].append({ 32 | 'object_id': file_.id, 33 | 'name': file_.filename, 34 | 'md5': file_.md5, 35 | 'size': file_.size, 36 | 'reader_params': kwargs.get('reader_params'), 37 | 'entity_params': kwargs.get('entity_params'), 38 | 'validation_params': kwargs.get('validation_params') 39 | }) 40 | 41 | def add_url(self, url, **kwargs): 42 | manifest_item = dict(url=url, **kwargs) 43 | self.manifest['files'].append(manifest_item) 44 | 45 | def add(self, *args): 46 | """ 47 | Add one or more files or URLs to the manifest. 48 | If files contains a glob, it is expanded. 49 | 50 | All files are uploaded to SolveBio. The Upload 51 | object is used to fill the manifest. 52 | """ 53 | def _is_supported_url(path): 54 | p = urlparse(path) 55 | return bool(p.scheme) and p.scheme in ['https', 'http'] 56 | 57 | for path in args: 58 | path = os.path.expanduser(path) 59 | if _is_supported_url(path): 60 | self.add_url(path) 61 | elif os.path.isfile(path): 62 | self.add_file(path) 63 | elif os.path.isdir(path): 64 | for f in os.listdir(path): 65 | self.add_file(f) 66 | elif glob.glob(path): 67 | for f in glob.glob(path): 68 | self.add_file(f) 69 | else: 70 | raise ValueError( 71 | 'Path: "{0}" is not a valid format or does not exist. ' 72 | 'Manifest paths must be local files, directories, blobs ' 73 | 'or URLs with http:// or https://.' 74 | .format(path) 75 | ) 76 | -------------------------------------------------------------------------------- /solvebio/test/test_dataset.py: -------------------------------------------------------------------------------- 1 | from .helper import SolveBioTestCase 2 | 3 | 4 | class DatasetTests(SolveBioTestCase): 5 | """ 6 | Test Dataset, DatasetField, and Facets 7 | """ 8 | 9 | def test_dataset_retrieval(self): 10 | dataset = self.client.Dataset.get_by_full_path( 11 | self.TEST_DATASET_FULL_PATH) 12 | self.assertTrue('id' in dataset, 13 | 'Should be able to get id in dataset') 14 | 15 | check_fields = ['class_name', 16 | 'created_at', 17 | 'data_url', 18 | 'vault_id', 19 | 'vault_object_id', 20 | 'description', 21 | 'fields_url', 22 | 'id', 23 | 'updated_at', 24 | 'url', 25 | 'documents_count'] 26 | 27 | for f in check_fields: 28 | self.assertTrue(f in dataset, '{0} field is present'.format(f)) 29 | 30 | def test_dataset_tree_traversal_shortcuts(self): 31 | dataset = self.client.Dataset.get_by_full_path( 32 | self.TEST_DATASET_FULL_PATH) 33 | 34 | # get vault object 35 | self.assertEqual(dataset.vault_object.full_path, 36 | self.TEST_DATASET_FULL_PATH) 37 | 38 | # get vault object parent 39 | self.assertEqual( 40 | dataset.vault_object.parent.full_path, 41 | '/'.join(self.TEST_DATASET_FULL_PATH.split('/')[:-1]) 42 | ) 43 | 44 | # get vault 45 | self.assertEqual( 46 | dataset.vault_object.vault.full_path, 47 | ':'.join(self.TEST_DATASET_FULL_PATH.split(':')[:-1]) 48 | ) 49 | 50 | def test_dataset_fields(self): 51 | dataset = self.client.Object.get_by_full_path( 52 | self.TEST_DATASET_FULL_PATH) 53 | fields = dataset.fields() 54 | dataset_field = fields.data[0] 55 | self.assertTrue('id' in dataset_field, 56 | 'Should be able to get id in list of dataset fields') 57 | 58 | check_fields = set(['class_name', 'created_at', 59 | 'data_type', 'dataset_id', 'title', 60 | 'description', 'facets_url', 61 | 'ordering', 'is_hidden', 'is_valid', 62 | 'is_list', 'entity_type', 'expression', 63 | 'name', 'updated_at', 'is_read_only', 64 | 'depends_on', 65 | 'id', 'url', 'vault_id', 'url_template']) 66 | self.assertSetEqual(set(dataset_field.keys()), check_fields) 67 | 68 | def test_dataset_facets(self): 69 | dataset = self.client.Object.get_by_full_path( 70 | self.TEST_DATASET_FULL_PATH) 71 | field = dataset.fields('status') 72 | facets = field.facets() 73 | self.assertTrue(len(facets['facets']) >= 0) 74 | -------------------------------------------------------------------------------- /solvebio/utils/humanize.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # From Humanize (jmoiron/humanize) 4 | # 5 | # Copyright (c) 2010 Jason Moiron and Contributors 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining 8 | # a copy of this software and associated documentation files (the 9 | # "Software"), to deal in the Software without restriction, including 10 | # without limitation the rights to use, copy, modify, merge, publish, 11 | # distribute, sublicense, and/or sell copies of the Software, and to 12 | # permit persons to whom the Software is furnished to do so, subject to 13 | # the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be 16 | # included in all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 21 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 22 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 24 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 | 26 | """Bits & Bytes related humanization.""" 27 | 28 | suffixes = { 29 | 'decimal': ('kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'), 30 | 'binary': ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'), 31 | 'gnu': "KMGTPEZY", 32 | } 33 | 34 | 35 | def naturalsize(value, binary=False, gnu=False, format='%.1f'): 36 | """Format a number of byteslike a human readable filesize (eg. 10 kB). By 37 | default, decimal suffixes (kB, MB) are used. Passing binary=true will use 38 | binary suffixes (KiB, MiB) are used and the base will be 2**10 instead of 39 | 10**3. If ``gnu`` is True, the binary argument is ignored and GNU-style 40 | (ls -sh style) prefixes are used (K, M) with the 2**10 definition. 41 | Non-gnu modes are compatible with jinja2's ``filesizeformat`` filter.""" 42 | if gnu: 43 | suffix = suffixes['gnu'] 44 | elif binary: 45 | suffix = suffixes['binary'] 46 | else: 47 | suffix = suffixes['decimal'] 48 | 49 | base = 1024 if (gnu or binary) else 1000 50 | bytes = float(value) 51 | 52 | if bytes == 1 and not gnu: 53 | return '1 Byte' 54 | elif bytes < base and not gnu: 55 | return '%d Bytes' % bytes 56 | elif bytes < base and gnu: 57 | return '%dB' % bytes 58 | 59 | for i, s in enumerate(suffix): 60 | unit = base ** (i + 2) 61 | if bytes < unit and not gnu: 62 | return (format + ' %s') % ((base * bytes / unit), s) 63 | elif bytes < unit and gnu: 64 | return (format + '%s') % ((base * bytes / unit), s) 65 | 66 | if gnu: 67 | return (format + '%s') % ((base * bytes / unit), s) 68 | 69 | return (format + ' %s') % ((base * bytes / unit), s) 70 | -------------------------------------------------------------------------------- /solvebio/test/test_credentials.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import shutil 5 | import sys 6 | import unittest 7 | import solvebio 8 | import solvebio.cli.credentials as creds 9 | import contextlib 10 | 11 | 12 | @contextlib.contextmanager 13 | def nostdout(): 14 | savestderr = sys.stdout 15 | 16 | class Devnull(object): 17 | def write(self, _): 18 | pass 19 | sys.stdout = Devnull() 20 | try: 21 | yield 22 | finally: 23 | sys.stdout = savestderr 24 | 25 | 26 | class TestCredentials(unittest.TestCase): 27 | def setUp(self): 28 | self.solvebiodir = os.path.join(os.path.dirname(__file__), 29 | 'data', '.solvebio') 30 | self.api_host = solvebio.get_api_host() 31 | solvebio.client._host = 'https://api.solvebio.com' 32 | 33 | def tearDown(self): 34 | solvebio.client._host = self.api_host 35 | if os.path.isdir(self.solvebiodir): 36 | shutil.rmtree(self.solvebiodir) 37 | 38 | def test_credentials(self): 39 | 40 | datadir = os.path.join(os.path.dirname(__file__), 'data') 41 | os.environ['HOME'] = datadir 42 | 43 | # Make sure we don't have have the test solvebio directory 44 | if os.path.isdir(self.solvebiodir): 45 | shutil.rmtree(self.solvebiodir) 46 | 47 | cred_file = creds.netrc.path() 48 | self.assertTrue(os.path.exists(cred_file), 49 | "cred file created when it doesn't exist first") 50 | 51 | self.assertEqual(creds.get_credentials(), None, 52 | 'Should not find credentials') 53 | 54 | test_credentials_file = os.path.join(datadir, 'test_creds') 55 | shutil.copy(test_credentials_file, cred_file) 56 | 57 | auths = creds.get_credentials() 58 | self.assertTrue(auths is not None, 'Should find credentials') 59 | 60 | solvebio.client._host = 'https://example.com' 61 | 62 | auths = creds.get_credentials() 63 | self.assertEqual(auths, None, 64 | 'Should not find credentials for host {0}' 65 | .format(solvebio.api_host)) 66 | 67 | solvebio.client._host = 'https://api.solvebio.com' 68 | creds.delete_credentials() 69 | auths = creds.get_credentials() 70 | self.assertEqual(auths, None, 71 | 'Should not find removed credentials for ' 72 | 'host {0}'.format(solvebio.get_api_host())) 73 | 74 | pair = ('testagain@solvebio.com', 'b00b00',) 75 | creds.save_credentials(*pair) 76 | auths = creds.get_credentials() 77 | self.assertTrue(auths is not None, 78 | 'Should get newly set credentials for ' 79 | 'host {0}'.format(solvebio.get_api_host())) 80 | 81 | expected = (solvebio.get_api_host(), pair[0], 'Token', pair[1]) 82 | self.assertEqual(auths, expected, 'Should get back creds we saved') 83 | -------------------------------------------------------------------------------- /solvebio/resource/datasetmigration.py: -------------------------------------------------------------------------------- 1 | from .apiresource import ListableAPIResource 2 | from .apiresource import DeletableAPIResource 3 | from .apiresource import CreateableAPIResource 4 | from .solveobject import convert_to_solve_object 5 | from .task import Task 6 | from .datasetcommit import follow_commits 7 | 8 | import time 9 | 10 | 11 | class DatasetMigration(CreateableAPIResource, ListableAPIResource, 12 | DeletableAPIResource): 13 | """ 14 | DatasetMigration represent an task that copies data between 15 | two Datasets or modifies data within a single Dataset. 16 | 17 | For interactive use, DatasetMigration can be "followed" to watch 18 | the progression of the task. 19 | """ 20 | RESOURCE_VERSION = 2 21 | 22 | LIST_FIELDS = ( 23 | ('id', 'ID'), 24 | ('status', 'Status'), 25 | ('source', 'Source'), 26 | ('target', 'Target'), 27 | ('documents_count', 'Records'), 28 | ('created_at', 'Created'), 29 | ('updated_at', 'Updated'), 30 | ) 31 | 32 | @property 33 | def source(self): 34 | response = self._client.get(self['source']['url'], {}) 35 | return convert_to_solve_object(response, client=self._client) 36 | 37 | @property 38 | def target(self): 39 | response = self._client.get(self['target']['url'], {}) 40 | return convert_to_solve_object(response, client=self._client) 41 | 42 | def follow(self, loop=True, sleep_seconds=Task.SLEEP_WAIT_DEFAULT): 43 | _status = self.status 44 | if self.status == 'queued': 45 | print("Waiting for migration (id = {0}) to start..." 46 | .format(self.id)) 47 | 48 | while self.status in ['queued', 'running']: 49 | if self.status != _status: 50 | print("Migration is now {0} (was {1})" 51 | .format(self.status, _status)) 52 | _status = self.status 53 | 54 | if self.status == 'running': 55 | processed_records = self.metadata\ 56 | .get('progress', {})\ 57 | .get('processed_records', 0) 58 | print("Migration '{0}' is {1}: {2}/{3} records migrated" 59 | .format(self.id, 60 | self.status, 61 | processed_records, 62 | self.documents_count)) 63 | 64 | if not loop: 65 | return 66 | 67 | time.sleep(sleep_seconds) 68 | self.refresh() 69 | 70 | if self.status == 'failed': 71 | print("Migration failed.") 72 | print("Reason: {}".format(self.error_message)) 73 | return 74 | 75 | if self.status == 'canceled': 76 | print("Migration was canceled") 77 | print("Reason: {}".format(self.error_message)) 78 | return 79 | 80 | # Follow commits until complete 81 | follow_commits(self, sleep_seconds) 82 | 83 | print("View your migrated data: " 84 | "https://my.solvebio.com/data/{0}" 85 | .format(self['target']['id'])) 86 | -------------------------------------------------------------------------------- /solvebio/cli/ipython.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | 5 | def _print(msg): 6 | """ 7 | Handle Python 2 interactive shells without requiring 8 | new print() format. 9 | """ 10 | sys.stdout.write(msg + '\n') 11 | 12 | 13 | def launch_ipython_shell(args): # pylint: disable=unused-argument 14 | """Open the SolveBio shell (IPython wrapper)""" 15 | try: 16 | import IPython # noqa 17 | except ImportError: 18 | _print("The SolveBio Python shell requires IPython.\n" 19 | "To install, type: 'pip install ipython'") 20 | return False 21 | 22 | if hasattr(IPython, "version_info"): 23 | if IPython.version_info > (5, 0, 0, ''): 24 | return launch_ipython_5_shell(args) 25 | 26 | _print("WARNING: Please upgrade IPython (you are running version: {})" 27 | .format(IPython.__version__)) 28 | return launch_ipython_legacy_shell(args) 29 | 30 | 31 | def launch_ipython_5_shell(args): 32 | """Open the SolveBio shell (IPython wrapper) with IPython 5+""" 33 | import IPython # noqa 34 | from traitlets.config import Config 35 | 36 | c = Config() 37 | path = os.path.dirname(os.path.abspath(__file__)) 38 | 39 | try: 40 | # see if we're already inside IPython 41 | get_ipython # pylint: disable=undefined-variable 42 | _print("WARNING: Running IPython within IPython.") 43 | except NameError: 44 | c.InteractiveShell.banner1 = 'SolveBio Python shell started.\n' 45 | 46 | c.InteractiveShellApp.exec_files = ['{}/ipython_init.py'.format(path)] 47 | IPython.start_ipython(argv=[], config=c) 48 | 49 | 50 | def launch_ipython_legacy_shell(args): # pylint: disable=unused-argument 51 | """Open the SolveBio shell (IPython wrapper) for older IPython versions""" 52 | try: 53 | from IPython.config.loader import Config 54 | except ImportError: 55 | _print("The SolveBio Python shell requires IPython.\n" 56 | "To install, type: 'pip install ipython'") 57 | return False 58 | 59 | try: 60 | # see if we're already inside IPython 61 | get_ipython # pylint: disable=undefined-variable 62 | except NameError: 63 | cfg = Config() 64 | prompt_config = cfg.PromptManager 65 | prompt_config.in_template = '[SolveBio] In <\\#>: ' 66 | prompt_config.in2_template = ' .\\D.: ' 67 | prompt_config.out_template = 'Out<\\#>: ' 68 | banner1 = '\nSolveBio Python shell started.' 69 | 70 | exit_msg = 'Quitting SolveBio shell.' 71 | else: 72 | _print("Running nested copies of IPython.") 73 | cfg = Config() 74 | banner1 = exit_msg = '' 75 | 76 | # First import the embeddable shell class 77 | try: 78 | from IPython.terminal.embed import InteractiveShellEmbed 79 | except ImportError: 80 | # pylint: disable=import-error,no-name-in-module 81 | from IPython.frontend.terminal.embed import InteractiveShellEmbed 82 | 83 | path = os.path.dirname(os.path.abspath(__file__)) 84 | init_file = '{}/ipython_init.py'.format(path) 85 | exec(compile(open(init_file).read(), init_file, 'exec'), 86 | globals(), locals()) 87 | 88 | InteractiveShellEmbed(config=cfg, banner1=banner1, exit_msg=exit_msg)() 89 | -------------------------------------------------------------------------------- /solvebio/resource/datasetimport.py: -------------------------------------------------------------------------------- 1 | from .apiresource import ListableAPIResource 2 | from .apiresource import DeletableAPIResource 3 | from .apiresource import CreateableAPIResource 4 | from .apiresource import UpdateableAPIResource 5 | from .solveobject import convert_to_solve_object 6 | from .task import Task 7 | from .datasetcommit import follow_commits 8 | 9 | import time 10 | 11 | 12 | class DatasetImport(CreateableAPIResource, ListableAPIResource, 13 | UpdateableAPIResource, DeletableAPIResource): 14 | """ 15 | DatasetImports represent an import task that takes 16 | either an object_id or a file manifest (list of file URLs) 17 | and converts them to a SolveBio-compatible format which can 18 | then be indexed by a dataset. 19 | 20 | For interactive use, DatasetImport can be "followed" to watch 21 | the progression of an import job. 22 | """ 23 | RESOURCE_VERSION = 2 24 | 25 | LIST_FIELDS = ( 26 | ('id', 'ID'), 27 | ('title', 'Title'), 28 | ('description', 'Description'), 29 | ('status', 'Status'), 30 | ('created_at', 'Created'), 31 | ) 32 | 33 | @property 34 | def dataset(self): 35 | return convert_to_solve_object(self['dataset'], client=self._client) 36 | 37 | def follow(self, loop=True, sleep_seconds=Task.SLEEP_WAIT_DEFAULT): 38 | 39 | if self.status == 'queued': 40 | print("Waiting for import (id = {0}) to start..." 41 | .format(self.id)) 42 | 43 | import_status = self.status 44 | while self.status in ['queued', 'running']: 45 | 46 | if self.status != import_status: 47 | print("Import is now {0} (was {1})" 48 | .format(self.status, import_status)) 49 | import_status = self.status 50 | # When status changes to running, indicate 51 | # pre-processing step. 52 | if self.status == 'running': 53 | print("Processing and validating file(s), " 54 | "this may take a few minutes...") 55 | elif self.status == 'running': 56 | records_count = self.metadata.get("progress", {}) \ 57 | .get("processed_records", 0) 58 | print("Import {0} is {1}: {2} records processed".format( 59 | self.id, self.status, records_count)) 60 | else: 61 | print("Import {0} is {1}".format(self.id, self.status)) 62 | 63 | if not loop: 64 | return 65 | 66 | time.sleep(sleep_seconds) 67 | self.refresh() 68 | 69 | if self.status == 'failed': 70 | print("Import processing and validation failed.") 71 | print("Reason: {}".format(self.error_message)) 72 | return 73 | 74 | if self.status == 'canceled': 75 | print("Import processing and validation was canceled") 76 | print("Reason: {}".format(self.error_message)) 77 | return 78 | 79 | # Follow commits until complete 80 | follow_commits(self, sleep_seconds) 81 | 82 | print("View your imported data: " 83 | "https://my.solvebio.com/data/{0}" 84 | .format(self['dataset']['id'])) 85 | -------------------------------------------------------------------------------- /solvebio/annotate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .client import client 4 | 5 | import logging 6 | logger = logging.getLogger('solvebio') 7 | 8 | 9 | class Annotator(object): 10 | """ 11 | Runs the synchronous annotate endpoint against 12 | batches of results from a query. 13 | """ 14 | CHUNK_SIZE = 100 15 | 16 | # Allows pre-setting a SolveClient 17 | _client = None 18 | 19 | def __init__(self, fields, **kwargs): 20 | self._client = kwargs.pop('client', None) or self._client or client 21 | 22 | self.buffer = [] 23 | self.fields = fields 24 | 25 | # Pop annotator_params from kwargs 26 | annotator_param_keys = [ 27 | 'annotator', 28 | 'debug', 29 | 'include_errors', 30 | 'post_annotation_expression', 31 | 'pre_annotation_expression' 32 | ] 33 | self.annotator_params = {} 34 | for key in annotator_param_keys: 35 | if key in kwargs: 36 | self.annotator_params[key] = kwargs.pop(key) 37 | 38 | self.data = kwargs.get('data') 39 | 40 | def annotate(self, records, **kwargs): 41 | """Annotate a set of records with stored fields. 42 | 43 | Args: 44 | records: A list or iterator (can be a Query object) 45 | chunk_size: The number of records to annotate at once (max 500). 46 | 47 | Returns: 48 | A generator that yields one annotated record at a time. 49 | """ 50 | # Update annotator_params with any kwargs 51 | self.annotator_params.update(**kwargs) 52 | chunk_size = self.annotator_params.get('chunk_size', self.CHUNK_SIZE) 53 | 54 | chunk = [] 55 | for i, record in enumerate(records): 56 | chunk.append(record) 57 | if (i + 1) % chunk_size == 0: 58 | for r in self._execute(chunk): 59 | yield r 60 | chunk = [] 61 | 62 | if chunk: 63 | for r in self._execute(chunk): 64 | yield r 65 | chunk = [] 66 | 67 | def _execute(self, chunk): 68 | data = { 69 | 'records': chunk, 70 | 'fields': self.fields, 71 | 'annotator_params': self.annotator_params, 72 | 'data': self.data 73 | } 74 | 75 | for r in self._client.post('/v1/annotate', data)['results']: 76 | yield r 77 | 78 | 79 | class Expression(object): 80 | """Runs a single SolveBio expression.""" 81 | 82 | # Allows pre-setting a SolveClient 83 | _client = None 84 | 85 | def __init__(self, expr, **kwargs): 86 | self.expr = expr 87 | self._client = kwargs.get('client') or self._client or client 88 | 89 | def evaluate(self, data=None, data_type='string', is_list=False): 90 | """Evaluates the expression with the provided context and format.""" 91 | payload = { 92 | 'data': data, 93 | 'expression': self.expr, 94 | 'data_type': data_type, 95 | 'is_list': is_list 96 | } 97 | res = self._client.post('/v1/evaluate', payload) 98 | return res['result'] 99 | 100 | def __repr__(self): 101 | return ''.format(self.expr) 102 | -------------------------------------------------------------------------------- /solvebio/resource/datasetcommit.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from .apiresource import ListableAPIResource 4 | from .apiresource import CreateableAPIResource 5 | from .apiresource import UpdateableAPIResource 6 | from .apiresource import DownloadableAPIResource 7 | from .solveobject import convert_to_solve_object 8 | from .task import Task 9 | 10 | 11 | def follow_commits(task, sleep_seconds): 12 | """Utility used to wait for commits""" 13 | while True: 14 | unfinished_commits = [ 15 | c for c in task.dataset_commits 16 | if c.status in ['queued', 'running'] 17 | ] 18 | 19 | if not unfinished_commits: 20 | print("All commits have finished processing") 21 | break 22 | 23 | print("{0}/{1} commits have finished processing" 24 | .format(len(unfinished_commits), 25 | len(task.dataset_commits))) 26 | 27 | # Prints a status for each one 28 | for commit in unfinished_commits: 29 | commit.follow(loop=False, sleep_seconds=sleep_seconds) 30 | 31 | time.sleep(sleep_seconds) 32 | 33 | # refresh Task to get fresh dataset commits 34 | task.refresh() 35 | 36 | 37 | class DatasetCommit(CreateableAPIResource, ListableAPIResource, 38 | UpdateableAPIResource, DownloadableAPIResource): 39 | """ 40 | DatasetCommits represent a change made to a Dataset. 41 | """ 42 | RESOURCE_VERSION = 2 43 | 44 | LIST_FIELDS = ( 45 | ('id', 'ID'), 46 | ('title', 'Title'), 47 | ('description', 'Description'), 48 | ('status', 'Status'), 49 | ('created_at', 'Created'), 50 | ) 51 | 52 | @property 53 | def dataset(self): 54 | return convert_to_solve_object(self['dataset'], client=self._client) 55 | 56 | @property 57 | def parent_object(self): 58 | """ Get the commit objects parent Import or Migration """ 59 | from . import types 60 | parent_klass = types.get(self.parent_job_model.split('.')[1]) 61 | return parent_klass.retrieve(self.parent_job_id, client=self._client) 62 | 63 | def follow(self, loop=True, sleep_seconds=Task.SLEEP_WAIT_DEFAULT): 64 | # Follow unfinished commits 65 | while self.status in ['queued', 'running']: 66 | if self.status == 'running': 67 | print("Commit {3} is {0}: {1}/{2} records indexed" 68 | .format(self.status, self.records_modified, 69 | self.records_total, self.id)) 70 | else: 71 | print("Commit {0} is {1}".format(self.id, self.status)) 72 | 73 | # When following a parent DatasetImport we do not want to 74 | # loop for status updates. It will handle its own looping 75 | # so break out of loop and return here. 76 | if not loop: 77 | break 78 | 79 | # sleep 80 | time.sleep(sleep_seconds) 81 | 82 | # refresh status 83 | self.refresh() 84 | 85 | if loop: 86 | print("Commit '{0}' ({1}) is {2}".format(self.title, 87 | self.id, 88 | self.status)) 89 | print("View your imported data: " 90 | "https://my.solvebio.com/data/{0}" 91 | .format(self['dataset']['id'])) 92 | -------------------------------------------------------------------------------- /solvebio/resource/solveobject.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys 4 | 5 | from ..client import client 6 | from .util import json 7 | 8 | 9 | def convert_to_solve_object(resp, **kwargs): 10 | from . import types 11 | 12 | _client = kwargs.pop('client', None) 13 | 14 | if isinstance(resp, list): 15 | return [convert_to_solve_object(i, client=_client) for i in resp] 16 | elif isinstance(resp, dict) and not isinstance(resp, SolveObject): 17 | resp = resp.copy() 18 | klass_name = resp.get('class_name') 19 | if isinstance(klass_name, str): 20 | klass = types.get(klass_name, SolveObject) 21 | else: 22 | klass = SolveObject 23 | return klass.construct_from(resp, client=_client) 24 | else: 25 | return resp 26 | 27 | 28 | class SolveObject(dict): 29 | """Base class for all SolveBio API resource objects""" 30 | ID_ATTR = 'id' 31 | 32 | # Allows pre-setting a SolveClient 33 | _client = None 34 | 35 | def __init__(self, id=None, **params): 36 | super(SolveObject, self).__init__() 37 | 38 | self._client = params.pop('client', self._client or client) 39 | 40 | # store manually updated values for partial updates 41 | self._unsaved_values = set() 42 | 43 | if id: 44 | self[self.ID_ATTR] = id 45 | 46 | def __setattr__(self, k, v): 47 | if k[0] == '_' or k in self.__dict__: 48 | return super(SolveObject, self).__setattr__(k, v) 49 | else: 50 | self[k] = v 51 | 52 | def __getattr__(self, k): 53 | if k[0] == '_': 54 | raise AttributeError(k) 55 | 56 | try: 57 | return self[k] 58 | except KeyError as err: 59 | raise AttributeError(*err.args) 60 | 61 | def __setitem__(self, k, v): 62 | super(SolveObject, self).__setitem__(k, v) 63 | self._unsaved_values.add(k) 64 | 65 | @classmethod 66 | def construct_from(cls, values, **kwargs): 67 | """Used to create a new object from an HTTP response""" 68 | instance = cls(values.get(cls.ID_ATTR), **kwargs) 69 | instance.refresh_from(values) 70 | return instance 71 | 72 | def refresh_from(self, values): 73 | self.clear() 74 | self._unsaved_values = set() 75 | 76 | for k, v in values.items(): 77 | super(SolveObject, self).__setitem__( 78 | k, convert_to_solve_object(v, client=self._client)) 79 | 80 | def request(self, method, url, **kwargs): 81 | response = self._client.request(method, url, **kwargs) 82 | return convert_to_solve_object(response, client=self._client) 83 | 84 | def __repr__(self): 85 | if isinstance(self.get('class_name'), str): 86 | ident_parts = [self.get('class_name')] 87 | else: 88 | ident_parts = [type(self).__name__] 89 | 90 | if isinstance(self.get(self.ID_ATTR), int): 91 | ident_parts.append( 92 | '%s=%d' % (self.ID_ATTR, self.get(self.ID_ATTR),)) 93 | 94 | _repr = '<%s at %s> JSON: %s' % ( 95 | ' '.join(ident_parts), hex(id(self)), str(self)) 96 | 97 | if sys.version_info[0] < 3: 98 | return _repr.encode('utf-8') 99 | 100 | return _repr 101 | 102 | def __str__(self): 103 | return json.dumps(self, sort_keys=True, indent=2) 104 | 105 | @property 106 | def solvebio_id(self): 107 | return self.id 108 | -------------------------------------------------------------------------------- /solvebio/utils/printing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import sys 5 | import subprocess 6 | import locale 7 | import logging 8 | 9 | logger = logging.getLogger('solvebio') 10 | 11 | try: 12 | # reload() for Python3 13 | from importlib import reload 14 | except ImportError: 15 | pass 16 | 17 | std_handles = [sys.stdin, sys.stdout, sys.stderr] 18 | try: 19 | # Switch from the default input ASCII encoding to the default locale. 20 | # The Python runtime will use this when it has to decode a 21 | # string buffer to unicode. This is not needed in Python3. 22 | 23 | # However reload(sys), used below resets stdin, stdout, and stderr 24 | # which is bad if they've already been reassigned. An ipython 25 | # notebook shell, for example, sets up its own stdout. 26 | # See GitHub issue #43 and #21. 27 | reload(sys).setdefaultencoding(locale.getdefaultlocale()[1]) 28 | locale.setlocale(locale.LC_ALL, '') 29 | except: 30 | pass 31 | finally: 32 | sys.stdin, sys.stdout, sys.stderr = std_handles 33 | 34 | 35 | # Set rows and columns and colors 36 | 37 | def set_from_env(name, default_value): 38 | try: 39 | return int(os.environ[name]) 40 | except: 41 | return default_value 42 | 43 | 44 | TTY_ROWS = set_from_env('LINES', 24) 45 | TTY_COLS = set_from_env('COLUMNS', 80) 46 | 47 | TTY_COLORS = True 48 | 49 | if sys.stdout.isatty(): 50 | try: 51 | with open(os.devnull, 'w') as fnull: 52 | rows, cols = subprocess.check_output( 53 | ['stty', 'size'], 54 | stderr=fnull).split() 55 | TTY_ROWS = int(rows) 56 | TTY_COLS = int(cols) 57 | except: 58 | logger.warn('Cannot detect terminal column width.\nUsing value ' 59 | 'from environment variables and/or internal defaults.') 60 | else: 61 | TTY_COLORS = False 62 | 63 | 64 | def pretty_int(num): 65 | return locale.format_string("%d", int(num), grouping=True) 66 | 67 | 68 | # Basic color support 69 | 70 | def green(text): 71 | if not TTY_COLORS: 72 | return text 73 | return '\033[32m' + text + '\033[39m' 74 | 75 | 76 | def red(text): 77 | if not TTY_COLORS: 78 | return text 79 | return '\033[31m' + text + '\033[39m' 80 | 81 | 82 | def yellow(text): 83 | if not TTY_COLORS: 84 | return text 85 | return '\033[33m' + text + '\033[39m' 86 | 87 | 88 | def blue(text): 89 | if not TTY_COLORS: 90 | return text 91 | return '\033[34m' + text + '\033[39m' 92 | 93 | 94 | def pager(fn, **kwargs): 95 | try: 96 | import tty 97 | fd = sys.stdin.fileno() 98 | old = tty.tcgetattr(fd) 99 | tty.setcbreak(fd) 100 | 101 | def getchar(): 102 | sys.stdin.read(1) 103 | except (ImportError, AttributeError): 104 | tty = None 105 | 106 | def getchar(): 107 | sys.stdin.readline()[:-1][:1] 108 | 109 | try: 110 | page = 1 111 | res = fn(page=page, **kwargs) 112 | has_next = res.links['next'] 113 | sys.stdout.write(str(res) + '\n') 114 | 115 | while has_next: 116 | sys.stdout.write('-- More --') 117 | sys.stdout.flush() 118 | c = getchar() 119 | page += 1 120 | res = fn(page=page, **kwargs) 121 | has_next = res.links['next'] 122 | 123 | if c in ('q', 'Q'): 124 | sys.stdout.write('\r \r') 125 | break 126 | 127 | sys.stdout.write('\n' + str(res) + '\n') 128 | finally: 129 | if tty: 130 | tty.tcsetattr(fd, tty.TCSAFLUSH, old) 131 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/tests/test_solvebio_auth_integration.py: -------------------------------------------------------------------------------- 1 | from dash.dependencies import Input, Output 2 | import dash 3 | import dash_html_components as html 4 | import dash_core_components as dcc 5 | import time 6 | 7 | from solvebio.contrib.dash import SolveBioAuth 8 | 9 | from .IntegrationTests import IntegrationTests 10 | from .utils import switch_windows 11 | 12 | from .credentials import OAUTH_CLIENT_ID 13 | from .credentials import OAUTH_USERNAME 14 | from .credentials import OAUTH_PASSWORD 15 | from .credentials import OAUTH_DOMAIN 16 | 17 | 18 | class Tests(IntegrationTests): 19 | test_client_id = OAUTH_CLIENT_ID 20 | test_domain = OAUTH_DOMAIN 21 | test_username = OAUTH_USERNAME 22 | test_password = OAUTH_PASSWORD 23 | 24 | def solvebio_auth_login_flow(self, username, pw, domain, 25 | url_base_pathname): 26 | app = dash.Dash(__name__, url_base_pathname=url_base_pathname) 27 | app.layout = html.Div([ 28 | dcc.Input( 29 | id='input', 30 | value='initial value' 31 | ), 32 | html.Div(id='output') 33 | ]) 34 | 35 | @app.callback(Output('output', 'children'), [Input('input', 'value')]) 36 | def update_output(new_value): 37 | return new_value 38 | 39 | SolveBioAuth( 40 | app, 41 | 'http://localhost:8050{}'.format(url_base_pathname), 42 | self.test_client_id 43 | ) 44 | 45 | self.startServer(app) 46 | 47 | time.sleep(10) 48 | self.percy_snapshot('login screen - {} {} {}'.format( 49 | username, pw, url_base_pathname)) 50 | try: 51 | self.wait_for_element_by_id('dash-auth--login__container') 52 | except Exception as e: 53 | print(self.wait_for_element_by_tag_name('body').html) 54 | raise e 55 | 56 | self.driver.find_element_by_id('dash-auth--login__button').click() 57 | time.sleep(5) 58 | switch_windows(self.driver) 59 | 60 | # Domain selection 61 | time.sleep(5) 62 | self.wait_for_element_by_css_selector( 63 | 'input[name="domain"]' 64 | ).send_keys(domain) 65 | self.driver.find_element_by_css_selector( 66 | 'button[type="submit"]' 67 | ).click() 68 | 69 | # Login page 70 | time.sleep(10) 71 | self.wait_for_element_by_css_selector( 72 | 'input[name="email"]' 73 | ).send_keys(username) 74 | self.wait_for_element_by_css_selector( 75 | 'input[name="password"]' 76 | ).send_keys(pw) 77 | self.driver.find_element_by_css_selector( 78 | 'button[type="submit"]' 79 | ).click() 80 | 81 | # OAuth authorize page 82 | time.sleep(10) 83 | self.percy_snapshot('oauth screen - {} {} {}'.format( 84 | username, pw, url_base_pathname)) 85 | self.wait_for_element_by_css_selector( 86 | 'button[type="submit"]' 87 | ).click() 88 | 89 | def private_app_authorized(self, url_base_pathname): 90 | self.solvebio_auth_login_flow( 91 | self.test_username, 92 | self.test_password, 93 | self.test_domain, 94 | url_base_pathname 95 | ) 96 | switch_windows(self.driver) 97 | time.sleep(5) 98 | self.percy_snapshot('private_app_authorized - {}'.format( 99 | url_base_pathname)) 100 | try: 101 | el = self.wait_for_element_by_id('output') 102 | except: 103 | print((self.driver.find_element_by_tag_name('body').html)) 104 | self.assertEqual(el.text, 'initial value') 105 | 106 | def test_private_app_authorized_index(self): 107 | self.private_app_authorized('/') 108 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![SolveBio Python Package](https://github.com/solvebio/solvebio-python/workflows/SolveBio%20Python%20Package/badge.svg) 2 | 3 | 4 | SolveBio Python Client 5 | ====================== 6 | 7 | # This package is deprecated. Please install and use quartzbio instead: 8 | 9 | ```bash 10 | pip install quartzbio 11 | ``` 12 | 13 | **This package will no longer be maintained after March 31, 2026.** 14 | 15 | **Migration guide: [https://github.com/quartzbio/quartzbio-python](https://github.com/quartzbio/quartzbio-python)** 16 | 17 | This is the SolveBio Python package and command-line interface (CLI). 18 | This module is tested against Python 2.7, 3.6, 3.7, 3.8, 3.10, PyPy and PyPy3. 19 | 20 | Developer documentation is available at [docs.solvebio.com](https://docs.solvebio.com). For more information about SolveBio visit [www.solvebio.com](https://www.solvebio.com). 21 | 22 | 23 | 24 | Installation & Setup 25 | -------------------- 26 | 27 | Install `solvebio` using `pip`: 28 | 29 | pip install solvebio 30 | 31 | 32 | For interactive use, we recommend installing `IPython` and `gnureadline`: 33 | 34 | pip install ipython 35 | pip install gnureadline 36 | 37 | 38 | To log in, type: 39 | 40 | solvebio login 41 | 42 | 43 | Enter your SolveBio credentials and you should be good to go! 44 | 45 | 46 | Install from Git 47 | ---------------- 48 | 49 | pip install -e git+https://github.com/solvebio/solvebio-python.git#egg=solve 50 | 51 | 52 | Development 53 | ----------- 54 | 55 | git clone https://github.com/solvebio/solvebio-python.git 56 | cd solve-python/ 57 | python setup.py develop 58 | 59 | 60 | Or install `tox` and run: 61 | 62 | pip install tox 63 | tox 64 | 65 | 66 | Releasing 67 | --------- 68 | 69 | You will need to [configure Twine](https://twine.readthedocs.io/en/latest/#installation) in order to push to PyPI. 70 | 71 | Maintainers can release solvebio-python to PyPI with the following steps: 72 | 73 | bumpversion 74 | git push --tags 75 | make changelog 76 | make release 77 | 78 | 79 | 80 | Support 81 | ------- 82 | 83 | Developer documentation is available at [docs.solvebio.com](https://docs.solvebio.com). 84 | 85 | If you experience problems with this package, please [create a GitHub Issue](https://github.com/solvebio/solvebio-python/issues). 86 | 87 | For all other requests, please [email SolveBio Support](mailto:support@solvebio.com). 88 | 89 | 90 | Configuring the Client 91 | ------- 92 | 93 | The SolveBio python client can be configured by setting system environment variables. 94 | Supported environment variables are: 95 | 96 | `SOLVEBIO_API_HOST` 97 | - The URL of the target API backend. 98 | If not specified the value from the local credentials file will be used. 99 | 100 | `SOLVEBIO_ACCESS_TOKEN` 101 | - The OAuth2 access token for authenticating with the API. 102 | 103 | `SOLVEBIO_API_KEY` 104 | - The API Key to use for authenticating with the API. 105 | 106 | The lookup order for credentials is: 107 | 1. Access Token 108 | 2. API Key 109 | 3. Local Credentials file 110 | 111 | `SOLVEBIO_LOGLEVEL` 112 | - The log level at which to log messages. 113 | If not specified the default log level will be WARN. 114 | 115 | `SOLVEBIO_LOGFILE` 116 | - The file in which to write log messages. 117 | If the file does not exist it will be created. 118 | If not specified '~/.solvebio/solvebio.log' will be used by default. 119 | 120 | `SOLVEBIO_RETRY_ALL` 121 | - Flag for enabling aggressive retries for failed requests to the API. 122 | When truthy, the client will attempt to retry a failed request regardless of the type of operation. 123 | This includes idempotent and nonidempotent operations: 124 | "HEAD", "GET", "PUT", "POST", "PATCH", "DELETE", "OPTIONS", "TRACE" 125 | If this value is not set it will default to false and retries will only be enabled for idempotent operations. 126 | -------------------------------------------------------------------------------- /recipes/tests/test_recipes_sync.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import pytest 4 | import solvebio as sb 5 | import recipes.sync_recipes as sync_recipes 6 | import mock 7 | from click.testing import CliRunner 8 | 9 | 10 | def get_input(name): 11 | return os.path.join(os.path.dirname(__file__), "test_inputs", name) 12 | 13 | 14 | @pytest.fixture 15 | def mock_dataset_template_retrieve(monkeypatch): 16 | def mock(*args, **kwargs): 17 | ds = sb.DatasetTemplate(id="fake") 18 | 19 | monkeypatch.setattr(sync_recipes.sb.DatasetTemplate, "retrieve", mock) 20 | monkeypatch.setattr(sync_recipes.sb.DatasetTemplate, "delete", mock) 21 | monkeypatch.setattr(sync_recipes.sb.DatasetTemplate, "create", mock) 22 | monkeypatch.setattr(sync_recipes.sb.DatasetTemplate, "all", mock) 23 | 24 | 25 | runner = CliRunner() 26 | 27 | 28 | @pytest.fixture 29 | def mock_user_retrieve(monkeypatch): 30 | def retrieve(*args, **kwargs): 31 | usr = sb.User(id="fake", full_name="Name", domain="Domain", account="Account") 32 | usr.full_name = "full name" 33 | usr.id = "3851" 34 | return usr 35 | monkeypatch.setattr(sync_recipes.sb.User, "retrieve", retrieve) 36 | 37 | def test_sync_recipe(mock_dataset_template_retrieve, 38 | mock_user_retrieve): 39 | with pytest.raises(SystemExit) as e: 40 | sync_recipes.sync_recipes(["--help"]) 41 | assert e.value.code == 0 42 | 43 | mock.patch('Solvebio.login', mock.MagicMock()) 44 | 45 | # Invalid file 46 | with pytest.raises(SystemExit) as e: 47 | config = get_input("non_existing_file.yml") 48 | sync_recipes.sync(["--name", "cDNA Change (v1.0.2)", config]) 49 | assert e.value.code == 2 50 | 51 | # Valid commands 52 | config = get_input("valid.yml") 53 | result = runner.invoke(sync_recipes.sync, 54 | args=["--name", "cDNA Change (v1.0.2)", config], 55 | input="y") 56 | assert result.exit_code == 0 57 | assert "create cDNA Change (v1.0.2)" in result.output 58 | 59 | result = runner.invoke(sync_recipes.delete, 60 | args=["--name", "cDNA Change (v1.0.2)", config], 61 | input="y") 62 | assert result.exit_code == 0 63 | assert "Requested recipe cDNA Change (v1.0.2) doesn't exist in SolveBio!" in result.output 64 | 65 | result = runner.invoke(sync_recipes.sync, 66 | args=["--all", config], 67 | input="N") 68 | assert result.exit_code == 0 69 | assert "create cDNA Change (v1.0.2)" in result.output 70 | assert "Aborted" in result.output 71 | 72 | result = runner.invoke(sync_recipes.delete, 73 | args=["--all", config], 74 | input="y") 75 | assert result.exit_code == 0 76 | assert "Requested recipe cDNA Change (v1.0.2) doesn't exist in SolveBio!" in result.output 77 | 78 | config2 = get_input("valid2.yml") 79 | result = runner.invoke(sync_recipes.sync, 80 | args=["--all", config2], 81 | input="y") 82 | 83 | assert result.exit_code == 0 84 | assert "create Gene (v1.0.3)" in result.output 85 | assert "create Protein Change (v1.0.4)" in result.output 86 | 87 | result = runner.invoke(sync_recipes.delete, 88 | args=["--all", config2], 89 | input="y") 90 | assert result.exit_code == 0 91 | assert "Requested recipe Gene (v1.0.3) doesn't exist in SolveBio!" in result.output 92 | assert "Requested recipe Protein Change (v1.0.4) doesn't exist in SolveBio!" in result.output 93 | 94 | yml_export_file = get_input("export.yml") 95 | # Test export mode with public/account recipes 96 | result = runner.invoke(sync_recipes.export, 97 | args=["--public-recipes", yml_export_file]) 98 | os.remove(yml_export_file) 99 | assert "Exporting all public recipes" in result.output 100 | -------------------------------------------------------------------------------- /solvebio/test/test_vault.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from .helper import SolveBioTestCase 4 | 5 | 6 | class VaultTests(SolveBioTestCase): 7 | 8 | def test_vaults(self): 9 | vaults = self.client.Vault.all() 10 | vault = vaults.data[0] 11 | self.assertTrue('id' in vault, 12 | 'Should be able to get id in vault') 13 | 14 | vault2 = self.client.Vault.retrieve(vault.id) 15 | self.assertEqual(vault, vault2, 16 | "Retrieving vault id {0} found by all()" 17 | .format(vault.id)) 18 | 19 | check_fields = [ 20 | 'account_id', 'created_at', 'description', 'has_children', 21 | 'has_folder_children', 'id', 'is_deleted', 'is_public', 22 | 'last_synced', 'name', 'permissions', 'provider', 23 | 'require_unique_paths', 'updated_at', 'url', 'user_id', 24 | 'vault_properties', 'vault_type' 25 | ] 26 | 27 | for f in check_fields: 28 | self.assertTrue(f in vault, '{0} field is present'.format(f)) 29 | 30 | def test_vault_paths(self): 31 | user = self.client.User.retrieve() 32 | domain = user.account.domain 33 | user_vault = 'user-{0}'.format(user.id) 34 | 35 | vaults = self.client.Vault.all() 36 | for vault in vaults: 37 | v, v_paths = self.client.Vault.validate_full_path(vault.full_path) 38 | self.assertEqual(v, vault.full_path) 39 | 40 | test_cases = [ 41 | ['myVault/', '{0}:myVault'.format(domain)], 42 | ['myVault', '{0}:myVault'.format(domain)], 43 | ['{0}:myVault'.format(domain), '{0}:myVault'.format(domain)], 44 | ['acme:myVault', 'acme:myVault'], 45 | ['myVault/folder1/folder2: xyz', '{0}:myVault'.format(domain)], 46 | ['acme:myVault/folder1/folder2: xyz', 'acme:myVault'], 47 | ['acme:myVault:/folder1/folder2: xyz', 'acme:myVault'], 48 | # The following are the "new" vault/path formats: 49 | ['~/', '{0}:{1}'.format(domain, user_vault)], 50 | ['acme:myVault/uploads_folder', 'acme:myVault'], 51 | ['myVault/uploads_folder', '{0}:myVault'.format(domain)], 52 | ] 53 | for case, expected in test_cases: 54 | v, v_paths = self.client.Vault.validate_full_path(case) 55 | self.assertEqual(v, expected) 56 | 57 | error_test_cases = [ 58 | '', 59 | '/', 60 | ':', 61 | ':/', 62 | '::/', 63 | 'x:', 64 | # Underscore in domain 65 | 'my_Domain:myVault:/the/heack', 66 | # Space in domain 67 | 'my Domain:my:Vault:/the/heack', 68 | # Too many colons 69 | 'myDomain:my:Vault:/the/heack', 70 | 'oops:myDomain:myVault', 71 | ] 72 | for case in error_test_cases: 73 | with self.assertRaises(Exception): 74 | v, v_paths = self.client.Vault.validate_full_path(case) 75 | 76 | @unittest.skip("Skip because API Host on GH pipelines doesn't support versioning.") 77 | def test_vault_versioning(self): 78 | vault = self.client.Vault.get_personal_vault() 79 | initial_versioning_status = vault["versioning"] 80 | assert initial_versioning_status in ["enabled", "disabled", "suspended"] 81 | 82 | self.addCleanup(VaultTests.clean_up_after_vault_versioning, vault, initial_versioning_status) 83 | 84 | vault.enable_versioning() 85 | assert "enabled" == vault["versioning"] 86 | vault.disable_versioning() 87 | assert "disabled" == vault["versioning"] 88 | vault.suspend_versioning() 89 | assert "suspended" == vault["versioning"] 90 | 91 | @staticmethod 92 | def clean_up_after_vault_versioning(vault, versioning_status): 93 | if versioning_status == "enabled": 94 | vault.enable_versioning() 95 | elif versioning_status == "disabled": 96 | vault.disable_versioning() 97 | elif versioning_status == "suspended": 98 | vault.suspend_versioning() 99 | assert versioning_status == vault["versioning"] 100 | -------------------------------------------------------------------------------- /solvebio/contrib/streamlit/solvebio_streamlit.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | 4 | import streamlit as st 5 | import solvebio 6 | 7 | from solvebio_auth import SolveBioOAuth2 8 | 9 | 10 | class SolveBioStreamlit: 11 | """SolveBio OAuth2 wrapper for restricting access to Streamlit apps""" 12 | 13 | # App settings loaded from environment variables or .env file 14 | CLIENT_ID = os.environ.get("CLIENT_ID", "Application (client) Id") 15 | CLIENT_SECRET = os.environ.get("CLIENT_SECRET", "Application (client) secret") 16 | APP_URL = os.environ.get("APP_URL", "http://localhost:5000") 17 | 18 | def solvebio_login_component(self, authorization_url): 19 | """Streamlit component for logging into SolveBio""" 20 | 21 | st.title("Secure Streamlit App") 22 | st.write( 23 | """ 24 |

25 | Log in to SolveBio to continue 26 |

27 | This app requires a SolveBio account.
28 | Contact Support 29 | """.format( 30 | authorization_url 31 | ), 32 | unsafe_allow_html=True, 33 | ) 34 | 35 | def get_token_from_session(self): 36 | """Reads token from streamlit session state. 37 | 38 | If token is in the session state then the user is authorized to use the app and vice versa. 39 | """ 40 | 41 | if "token" not in st.session_state: 42 | token = None 43 | else: 44 | token = st.session_state.token 45 | 46 | return token 47 | 48 | def wrap(self, streamlit_app): 49 | """SolveBio OAuth2 wrapper around streamlit app""" 50 | 51 | # SolveBio OAuth2 client 52 | oauth_client = SolveBioOAuth2(self.CLIENT_ID, self.CLIENT_SECRET) 53 | authorization_url = oauth_client.get_authorization_url( 54 | self.APP_URL, 55 | ) 56 | 57 | # Authorization token from Streamlit session state 58 | oauth_token = self.get_token_from_session() 59 | 60 | if oauth_token is None: 61 | # User is not authrized to use the app 62 | try: 63 | # Trying to get the authorization token from the url if successfully authorized 64 | code = st.experimental_get_query_params()["code"] 65 | 66 | # Remove authorization token from the url params 67 | params = {} 68 | st.experimental_set_query_params(**params) 69 | 70 | except: 71 | # Display SolveBio login until user is successfully authorized 72 | self.solvebio_login_component(authorization_url) 73 | else: 74 | try: 75 | # Getting the token from token API by sending the authorization code 76 | oauth_token = asyncio.run( 77 | oauth_client.get_access_token(code, self.APP_URL) 78 | ) 79 | except: 80 | st.error( 81 | "This account is not allowed or page was refreshed. Please login again." 82 | ) 83 | self.solvebio_login_component(authorization_url) 84 | else: 85 | # Check if token has expired: 86 | if oauth_token.is_expired(): 87 | st.error("Login session has ended. Please login again.") 88 | self.solvebio_login_component(authorization_url) 89 | else: 90 | # User is now authenticated and authorized to use the app 91 | 92 | # SolveClient to acces API 93 | solvebio_client = solvebio.SolveClient( 94 | token=oauth_token["access_token"], 95 | token_type=oauth_token["token_type"], 96 | ) 97 | 98 | # Saving token and solvebio client to the Streamlit session state 99 | st.session_state.solvebio_client = solvebio_client 100 | st.session_state.token = oauth_token 101 | 102 | streamlit_app() 103 | else: 104 | # User is authorized to the the app 105 | streamlit_app() 106 | -------------------------------------------------------------------------------- /recipes/sync_recipe_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | helper functions for managing recipes 3 | """ 4 | import sys 5 | from collections import OrderedDict 6 | 7 | import solvebio as sb 8 | import ruamel.yaml as yaml 9 | import click 10 | 11 | 12 | def create_recipe(description): 13 | fields = [description['fields']] 14 | recipe_version = description['version'] 15 | recipe_name = description['name'] 16 | recipe_description = description['description'] 17 | is_public = description['is_public'] 18 | sb.DatasetTemplate.create( 19 | name="{} (v{})".format(recipe_name, recipe_version), 20 | description="{}".format(recipe_description) if recipe_description else None, 21 | template_type="recipe", 22 | tags=["recipe"], 23 | is_public=is_public, 24 | version=recipe_version, 25 | annotator_params={ 26 | "annotator": "parallel" 27 | }, 28 | fields=fields 29 | ) 30 | 31 | 32 | def delete_recipe(recipe_name): 33 | existing_recipe = sb.DatasetTemplate.all(name=recipe_name) 34 | if not existing_recipe: 35 | click.echo("{} doesn't exist!".format(recipe_name)) 36 | return 37 | for recipe in existing_recipe: 38 | recipe.delete(force=True) 39 | 40 | 41 | def sync_recipe(recipe): 42 | delete_recipe("{} (v{})".format(recipe['name'], recipe['version'])) 43 | create_recipe(recipe) 44 | 45 | 46 | def get_recipe_by_name_from_yml(all_recipes, name): 47 | for recipe in all_recipes: 48 | if recipe["name"] in name and recipe["version"] in name: 49 | return recipe 50 | click.echo("{} doesn't exist in the provided YAML file!".format(name)) 51 | return None 52 | 53 | 54 | def load_recipes_from_yaml(yml_file): 55 | with open(yml_file, 'r') as yml: 56 | y = yaml.YAML() 57 | all_recipes = y.load(yml) 58 | return all_recipes['recipes'] 59 | 60 | 61 | def get_public_recipes(): 62 | public_recipes = [] 63 | all_templates = sb.DatasetTemplate.all() 64 | if all_templates: 65 | for template in all_templates: 66 | if template['template_type'] == "recipe" and template['is_public']: 67 | public_recipes.append(template) 68 | 69 | return public_recipes 70 | 71 | 72 | def get_account_recipes(user): 73 | account_recipes = [] 74 | all_templates = sb.DatasetTemplate.all() 75 | if all_templates: 76 | for template in all_templates: 77 | if template['template_type'] == "recipe" \ 78 | and template['account'] == user["account"]["id"]: 79 | account_recipes.append(template) 80 | 81 | return account_recipes 82 | 83 | 84 | def export_recipes_to_yaml(recipes, yml_file): 85 | with open(yml_file, 'w') as outfile: 86 | class RecipeDumper(yaml.Dumper): 87 | pass 88 | 89 | class literal(str): 90 | pass 91 | 92 | def _dict_representer(dumper, data): 93 | return dumper.represent_mapping( 94 | yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items()) 95 | 96 | def _literal_representer(dumper, data): 97 | return dumper.represent_scalar( 98 | u'tag:yaml.org,2002:str', data, style='|') 99 | 100 | RecipeDumper.add_representer(dict, _dict_representer) 101 | RecipeDumper.add_representer(literal, _literal_representer) 102 | 103 | yaml_recipes = [] 104 | for r in recipes: 105 | recipe_expression = literal(str(dict(r)['fields'][0]['expression'])) 106 | dict(r)['fields'][0]['expression'] = recipe_expression 107 | recipe_details = { 108 | "name": dict(r)['name'], 109 | "description": dict(r)['description'], 110 | "template_type": "recipe", 111 | "is_public": "True" if dict(r)['is_public'] else "False", 112 | "version": dict(r)['version'], 113 | "annotator_params": { 114 | "annotator": "parallel" 115 | }, 116 | "fields": {i: dict(r)['fields'][0][i] 117 | for i in dict(r)['fields'][0] 118 | if i not in ['state', 'dictitems'] and dict(r)['fields'][0][i]} 119 | } 120 | yaml_recipes.append(recipe_details) 121 | yaml.dump({"recipes": yaml_recipes}, outfile, RecipeDumper, encoding='utf-8', 122 | default_flow_style=False) 123 | 124 | print("Wrote recipe to file: {}".format(yml_file)) 125 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/tests/IntegrationTests.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | import multiprocessing 3 | import requests 4 | import time 5 | import unittest 6 | import percy 7 | import sys 8 | import os 9 | 10 | from .utils import invincible, wait_for 11 | 12 | 13 | class IntegrationTests(unittest.TestCase): 14 | def percy_snapshot(cls, name): 15 | if ('PERCY_PROJECT' in os.environ and 16 | os.environ['PERCY_PROJECT'] == 'solvebio/contrib/dash'): 17 | 18 | snapshot_name = '{} - Py{}'.format( 19 | name, sys.version_info.major 20 | ).replace('/', '-') 21 | try: 22 | cls.percy_runner.snapshot( 23 | name=snapshot_name 24 | ) 25 | except Exception: 26 | print('Saving "{}" failed'.format(snapshot_name)) 27 | 28 | @classmethod 29 | def setUpClass(cls): 30 | super(IntegrationTests, cls).setUpClass() 31 | cls.driver = webdriver.Chrome() 32 | 33 | if ('PERCY_PROJECT' in os.environ and 34 | os.environ['PERCY_PROJECT'] == 'solvebio/contrib/dash'): 35 | loader = percy.ResourceLoader( 36 | webdriver=cls.driver 37 | ) 38 | cls.percy_runner = percy.Runner(loader=loader) 39 | 40 | cls.percy_runner.initialize_build() 41 | 42 | @classmethod 43 | def tearDownClass(cls): 44 | super(IntegrationTests, cls).tearDownClass() 45 | cls.driver.quit() 46 | if ('PERCY_PROJECT' in os.environ and 47 | os.environ['PERCY_PROJECT'] == 'solvebio/contrib/dash'): 48 | cls.percy_runner.finalize_build() 49 | 50 | def setUp(self): 51 | super(IntegrationTests, self).setUp() 52 | self.driver = webdriver.Chrome() 53 | 54 | def wait_for_element_by_id(id): 55 | wait_for(lambda: None is not invincible( 56 | lambda: self.driver.find_element_by_id(id) 57 | )) 58 | return self.driver.find_element_by_id(id) 59 | self.wait_for_element_by_id = wait_for_element_by_id 60 | 61 | def wait_for_element_by_css_selector(css_selector): 62 | wait_for(lambda: None is not invincible( 63 | lambda: self.driver.find_element_by_css_selector(css_selector) 64 | )) 65 | return self.driver.find_element_by_css_selector(css_selector) 66 | self.wait_for_element_by_css_selector = \ 67 | wait_for_element_by_css_selector 68 | 69 | def tearDown(self): 70 | super(IntegrationTests, self).tearDown() 71 | time.sleep(5) 72 | self.server_process.terminate() 73 | time.sleep(5) 74 | self.driver.quit() 75 | 76 | def startServer(self, app): 77 | def run(): 78 | app.scripts.config.serve_locally = True 79 | app.run_server( 80 | port=8050, 81 | debug=False, 82 | processes=2 83 | ) 84 | 85 | # Run on a separate process so that it doesn't block 86 | self.server_process = multiprocessing.Process(target=run) 87 | self.server_process.start() 88 | time.sleep(15) 89 | 90 | # Visit the dash page 91 | try: 92 | self.driver.get('http://localhost:8050{}'.format( 93 | app.config['routes_pathname_prefix']) 94 | ) 95 | except: 96 | print('Failed attempt to load page, trying again') 97 | print(self.server_process) 98 | print(self.server_process.is_alive()) 99 | time.sleep(5) 100 | print(requests.get('http://localhost:8050')) 101 | self.driver.get('http://localhost:8050') 102 | 103 | time.sleep(0.5) 104 | 105 | # Inject an error and warning logger 106 | logger = ''' 107 | window.tests = {}; 108 | window.tests.console = {error: [], warn: [], log: []}; 109 | 110 | var _log = console.log; 111 | var _warn = console.warn; 112 | var _error = console.error; 113 | 114 | console.log = function() { 115 | window.tests.console.log.push({ 116 | method: 'log', 117 | arguments: arguments 118 | }); 119 | return _log.apply(console, arguments); 120 | }; 121 | 122 | console.warn = function() { 123 | window.tests.console.warn.push({ 124 | method: 'warn', 125 | arguments: arguments 126 | }); 127 | return _warn.apply(console, arguments); 128 | }; 129 | 130 | console.error = function() { 131 | window.tests.console.error.push({ 132 | method: 'error', 133 | arguments: arguments 134 | }); 135 | return _error.apply(console, arguments); 136 | }; 137 | ''' 138 | self.driver.execute_script(logger) 139 | -------------------------------------------------------------------------------- /solvebio/test/test_tabulate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import unittest 4 | from solvebio.utils import tabulate as t 5 | from solvebio.utils import printing as p 6 | 7 | 8 | class TestTabulate(unittest.TestCase): 9 | 10 | def test_classify(self): 11 | 12 | self.assertEqual(t._isnumber('123.45'), True) 13 | self.assertEqual(t._isnumber('123'), True) 14 | self.assertEqual(t._isnumber('spam'), False) 15 | self.assertEqual(t._isint('123'), True) 16 | self.assertEqual(t._isint('123.45'), False) 17 | 18 | self.assertEqual(t._type(None), t._none_type) 19 | self.assertEqual(t._type('foo'), t._text_type) 20 | self.assertEqual(t._type('1'), t._int_type) 21 | self.assertEqual(t._type(1), t._int_type) 22 | self.assertEqual(t._type('\x1b[31m42\x1b[0m'), t._int_type) 23 | 24 | def test_align(self): 25 | self.assertEqual(t._afterpoint('123.45'), 2) 26 | self.assertEqual(t._afterpoint('1001'), -1) 27 | 28 | self.assertEqual(t._afterpoint('eggs'), -1) 29 | self.assertEqual(t._afterpoint('123e45'), 2) 30 | 31 | self.assertEqual(t._padleft(6, 'abcd'), ' abcd') 32 | self.assertEqual(t._padright(6, "abcd"), "abcd ") 33 | 34 | self.assertEqual(t._padboth(6, "abcd"), " abcd ") 35 | 36 | self.assertEqual(t._padboth(7, "abcd"), " abcd ") 37 | 38 | self.assertEqual(t._padright(2, 'abc'), 'abc') 39 | self.assertEqual(t._padleft(2, 'abc'), 'abc') 40 | self.assertEqual(t._padboth(2, 'abc'), 'abc') 41 | 42 | self.assertEqual( 43 | t._align_column( 44 | ["12.345", "-1234.5", "1.23", "1234.5", 45 | "1e+234", "1.0e234"], "decimal"), 46 | [' 12.345 ', '-1234.5 ', ' 1.23 ', 47 | ' 1234.5 ', ' 1e+234 ', ' 1.0e234']) 48 | 49 | def test_column_type(self): 50 | self.assertEqual(t._column_type(["1", "2"]), t._int_type) 51 | self.assertEqual(t._column_type(["1", "2.3"]), t._float_type) 52 | self.assertEqual(t._column_type(["1", "2.3", "four"]), t._text_type) 53 | self.assertEqual(t._column_type(["four", '\u043f\u044f\u0442\u044c']), 54 | t._text_type) 55 | 56 | self.assertEqual(t._column_type([None, "brux"]), t._text_type) 57 | self.assertEqual(t._column_type([1, 2, None]), t._int_type) 58 | 59 | def test_tabulate(self): 60 | p.TTY_COLS = 80 61 | tsv = t.simple_separated_format("\t") 62 | expected = """ 63 | foo 1 64 | spam\t23 65 | """ 66 | # [-1:1] below to remove leading and trailing "\n"s above 67 | got = t.tabulate([["foo", 1], ["spam", 23]], [], tsv) 68 | self.assertEqual(got, expected[1:-1], 69 | 'simple separated format table') 70 | #################################################################### 71 | 72 | expected = """ 73 | | abcd | 12345 | 74 | |--------+---------| 75 | | XY | 2 | 76 | | lmno | 4 | 77 | """ 78 | hrow = ['abcd', '12345'] 79 | tbl = [["XY", 2], ["lmno", 4]] 80 | 81 | # [-1:1] below to remove leading and trailing "\n"s above 82 | self.assertEqual(t.tabulate(tbl, hrow), expected[1:-1], 83 | 'org mode with header and unicode') 84 | 85 | ################################################################### 86 | 87 | expected = """ 88 | | Fields | Data | 89 | |-----------------------+-------------| 90 | | alternate_alleles | ['T'] | 91 | | clinical_origin | ['somatic'] | 92 | | clinical_significance | other | 93 | | gene_symbols | ['CPB1'] | 94 | """ 95 | data = [ 96 | ("gene_symbols", ["CPB1"]), 97 | ("clinical_significance", "other"), 98 | ("clinical_origin", ["somatic"]), 99 | ("alternate_alleles", ["T"]), ] 100 | got = t.tabulate(data, 101 | headers=('Fields', 'Data'), 102 | aligns=('right', 'left'), sort=True) 103 | 104 | # [-1:1] below to remove leading and trailing "\n"s above 105 | self.assertEqual(got, expected[1:-1], 106 | 'mixed data with arrays; close to actual ' + 107 | 'query output') 108 | 109 | expected = """ 110 | | Fields | Data | 111 | |-----------------------+-------------| 112 | | gene_symbols | ['CPB1'] | 113 | | clinical_significance | other | 114 | | clinical_origin | ['somatic'] | 115 | | alternate_alleles | ['T'] | 116 | """ 117 | got = t.tabulate(data, 118 | headers=('Fields', 'Data'), 119 | aligns=('right', 'left'), sort=False) 120 | self.assertEqual(got, expected[1:-1], 121 | 'mixed data with arrays; unsorted') 122 | 123 | 124 | if __name__ == "__main__": 125 | unittest.main() 126 | -------------------------------------------------------------------------------- /solvebio/cli/credentials.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from collections import namedtuple 3 | 4 | import solvebio 5 | 6 | from netrc import netrc as _netrc, NetrcParseError 7 | import os 8 | 9 | 10 | def as_netrc_machine(api_host): 11 | return api_host.replace("https://", "").replace("http://", "") 12 | 13 | 14 | class netrc(_netrc): 15 | """ 16 | Adds a save() method to netrc 17 | """ 18 | 19 | @staticmethod 20 | def path(): 21 | if os.name == "nt": 22 | # Windows 23 | path = "~\\_solvebio\\credentials" 24 | else: 25 | # *nix 26 | path = "~/.solvebio/credentials" 27 | 28 | try: 29 | path = os.path.expanduser(path) 30 | except KeyError: 31 | # os.path.expanduser can fail when $HOME is undefined and 32 | # getpwuid fails. See http://bugs.python.org/issue20164 33 | raise IOError("Could not find any home directory for '{0}'".format(path)) 34 | 35 | if not os.path.isdir(os.path.dirname(path)): 36 | os.makedirs(os.path.dirname(path)) 37 | 38 | # create an empty credentials file if it doesn't exist 39 | if not os.path.exists(path): 40 | try: 41 | open(path, "a").close() 42 | except IOError: 43 | raise Exception( 44 | "Could not create a SolveBio credentials file" 45 | " at '%s', permission denied." % path 46 | ) 47 | return path 48 | 49 | def save(self, path): 50 | """Dump the class data in the format of a .netrc file.""" 51 | rep = "" 52 | for host in self.hosts.keys(): 53 | attrs = self.hosts[host] 54 | rep = ( 55 | rep + "machine " + host + "\n\tlogin " + str(attrs[0]) + "\n" 56 | ) 57 | if attrs[1]: 58 | rep = rep + "\taccount " + str(attrs[1]) + "\n" 59 | rep = rep + "\tpassword " + str(attrs[2]) + "\n" 60 | 61 | f = open(path, "w") 62 | f.write(rep) 63 | f.close() 64 | 65 | 66 | class CredentialsError(BaseException): 67 | """ 68 | Raised if the credentials are not found. 69 | """ 70 | 71 | pass 72 | 73 | 74 | ApiCredentials = namedtuple( 75 | "ApiCredentials", ["api_host", "email", "token_type", "token"] 76 | ) 77 | 78 | 79 | def get_credentials(api_host: str = None) -> ApiCredentials: 80 | """ 81 | Returns the user's stored API key if a valid credentials file is found. 82 | Raises CredentialsError if no valid credentials file is found. 83 | """ 84 | 85 | try: 86 | netrc_obj = netrc(netrc.path()) 87 | if not netrc_obj.hosts: 88 | return None 89 | except (IOError, TypeError, NetrcParseError) as e: 90 | raise CredentialsError("Could not open credentials file: " + str(e)) 91 | 92 | netrc_host: str = None 93 | 94 | # if user provides a host, then find its token in the credentials file 95 | if api_host is not None: 96 | api_host = api_host.removeprefix("https://") 97 | if api_host in netrc_obj.hosts: 98 | netrc_host = api_host 99 | else: 100 | # login has failed for the requested host, 101 | # the rest of the credentials file is ignored 102 | return None 103 | else: 104 | # If there are no stored credentials for the default host, 105 | # but there are other stored credentials, use the first 106 | # available option that ends with '.api.quartzbio.com', 107 | netrc_host = next( 108 | filter(lambda h: h.endswith(".api.quartzbio.com"), netrc_obj.hosts), None 109 | ) 110 | 111 | # Otherwise use the first available. 112 | if netrc_host is None: 113 | netrc_host = next(iter(netrc_obj.hosts)) 114 | 115 | if netrc_host is not None: 116 | return ApiCredentials( 117 | "https://" + netrc_host, *netrc_obj.authenticators(netrc_host) 118 | ) 119 | return None 120 | 121 | 122 | def delete_credentials(): 123 | try: 124 | netrc_path = netrc.path() 125 | rc = netrc(netrc_path) 126 | except (IOError, TypeError, NetrcParseError) as e: 127 | raise CredentialsError("Could not open netrc file: " + str(e)) 128 | 129 | try: 130 | del rc.hosts[as_netrc_machine(solvebio.get_api_host())] 131 | except KeyError: 132 | pass 133 | else: 134 | rc.save(netrc_path) 135 | 136 | 137 | def save_credentials(email, token, token_type="Token", api_host=None): 138 | api_host = api_host or solvebio.get_api_host() 139 | 140 | try: 141 | netrc_path = netrc.path() 142 | rc = netrc(netrc_path) 143 | except (IOError, TypeError, NetrcParseError) as e: 144 | raise CredentialsError("Could not open netrc file: " + str(e)) 145 | 146 | # Overwrites any existing credentials 147 | rc.hosts[as_netrc_machine(api_host)] = (email, token_type, token) 148 | rc.save(netrc_path) 149 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/js/src/oauth-redirect-index.react.js: -------------------------------------------------------------------------------- 1 | /* global window:true, document:true */ 2 | 3 | import React, {Component} from 'react'; 4 | import ReactDOM from 'react-dom'; 5 | import cookie from 'cookie'; 6 | import queryString from 'query-string'; 7 | import {isEmpty} from 'ramda' 8 | 9 | require('../styles/application.scss'); 10 | 11 | const CONFIG = JSON.parse(document.getElementById('_auth-config').textContent); 12 | window.CONFIG = CONFIG; 13 | // const OAUTH_COOKIE_NAME = 'solvebio_oauth_token'; 14 | const LOGIN_PATHNAME = '_dash-login'; 15 | const IS_AUTHORIZED_PATHNAME = '_is-authorized'; 16 | 17 | /** 18 | * OAuth redirect component 19 | * - Looks for an oauth token in the URL as provided by the SolveBio redirect 20 | * - Make an API call to dash with that oauth token 21 | * - In response, Dash will set the oauth token as a cookie 22 | * if it is valid 23 | * Parent is component is responsible for rendering 24 | * this component in the appropriate context 25 | * (the URL redirect) 26 | */ 27 | class OauthRedirect extends Component { 28 | constructor(props) { 29 | super(props); 30 | this.state = { 31 | loginRequest: {}, 32 | authorizationRequest: {} 33 | } 34 | } 35 | 36 | componentDidMount() { 37 | // Support implicit and authorization-code flows. 38 | const {access_token, code, state} = queryString.parse(window.location.hash || window.location.search); 39 | this.setState({loginRequest: {status: 'loading'}}); 40 | const {requests_pathname_prefix} = CONFIG; 41 | 42 | window.console.warn(window.location.hash || window.location.search); 43 | window.console.warn({access_token, code}); 44 | 45 | // TODO - Polyfill 46 | fetch(`${requests_pathname_prefix}${LOGIN_PATHNAME}`, { 47 | method: 'POST', 48 | headers: { 49 | 'Accept': 'application/json', 50 | 'Content-Type': 'application/json', 51 | 'X-CSRFToken': cookie.parse(document.cookie)._csrf_token 52 | }, 53 | credentials: 'same-origin', 54 | body: JSON.stringify({access_token, code, state}) 55 | }).then(res => { 56 | this.setState({loginRequest: {status: res.status}}); 57 | return res.json().then(json => { 58 | this.setState({loginRequest: { 59 | status: res.status, 60 | content: json 61 | }}); 62 | }).then(() => { 63 | return fetch(`${requests_pathname_prefix}${IS_AUTHORIZED_PATHNAME}`, { 64 | method: 'GET', 65 | credentials: 'same-origin', 66 | headers: { 67 | 'Accept': 'application/json', 68 | 'Content-Type': 'application/json', 69 | 'X-CSRFToken': cookie.parse(document.cookie)._csrf_token 70 | } 71 | }).then(res => { 72 | this.setState({ 73 | authorizationRequest: { 74 | status: res.status 75 | } 76 | }); 77 | }) 78 | }); 79 | }).catch(err => { 80 | this.setState({loginRequest: {status: 500, content: err}}); 81 | }) 82 | } 83 | 84 | render() { 85 | const {authorizationRequest, loginRequest} = this.state; 86 | console.warn(authorizationRequest, loginRequest); 87 | let content; 88 | const loading =
{'Loading...'}
; 89 | if (isEmpty(loginRequest) || loginRequest.status === 'loading') { 90 | 91 | content = loading; 92 | 93 | } else if (loginRequest.status === 200) { 94 | 95 | if (authorizationRequest.status === 403) { 96 | content = ( 97 |
98 | {'You are not authorized to view this app'} 99 |
100 | ); 101 | } else if (authorizationRequest.status === 200) { 102 | window.close(); 103 | } else { 104 | content = loading; 105 | } 106 | 107 | } else { 108 | 109 | content = ( 110 |
111 |

{'Yikes! An error occurred trying to log in.'}

112 |

Please contact SolveBio Support or try again.

113 |

Additional information:

114 | { 115 | loginRequest.content ? 116 | 
{JSON.stringify(loginRequest.content)}
: 117 | null 118 | } 119 |
120 | ); 121 | 122 | } 123 | return ( 124 |
125 | {content} 126 |
127 | ); 128 | } 129 | } 130 | 131 | ReactDOM.render(, document.getElementById('react-root')); 132 | -------------------------------------------------------------------------------- /solvebio/contrib/dash/js/src/login-index.react.js: -------------------------------------------------------------------------------- 1 | /* global window:true, document:true */ 2 | 3 | import React, {Component} from 'react'; 4 | import ReactDOM from 'react-dom'; 5 | 6 | require('../styles/application.scss'); 7 | 8 | const CONFIG = JSON.parse(document.getElementById('_auth-config').textContent); 9 | window.CONFIG = CONFIG; 10 | const REDIRECT_URI_PATHNAME = '_oauth-redirect'; 11 | 12 | // http://stackoverflow.com/questions/4068373/center-a-popup-window-on-screen 13 | const PopupCenter = (url, title, w, h) => { 14 | // Fixes dual-screen position 15 | const screenLeft = window.screenLeft; 16 | const screenTop = window.screenTop; 17 | 18 | const width = window.innerWidth; 19 | const height = window.innerHeight; 20 | 21 | const left = ((width / 2) - (w / 2)) + screenLeft; 22 | const top = ((height / 2) - (h / 2)) + screenTop; 23 | const popupWindow = window.open( 24 | url, title, 25 | ('scrollbars=yes,width=' + w + 26 | ', height=' + h + ', top=' + top + 27 | ', left=' + left) 28 | ); 29 | return popupWindow; 30 | }; 31 | 32 | /** 33 | * Login displays an interface that guides the user through an oauth flow. 34 | * - Clicking on a login button will launch a new window with the SolveBio 35 | * - SolveBio will redirect that window to defined redirect URL when complete 36 | * - The component will render the oauth redirect page 37 | * - When the window is closed, will call its 38 | * `onClosed` prop 39 | */ 40 | class Login extends Component { 41 | constructor(props) { 42 | super(props); 43 | this.buildOauthUrl = this.buildOauthUrl.bind(this); 44 | this.oauthPopUp = this.oauthPopUp.bind(this); 45 | } 46 | 47 | buildOauthUrl() { 48 | const { 49 | oauth_client_id, 50 | oauth_response_type, 51 | oauth_state, 52 | solvebio_url, 53 | requests_pathname_prefix 54 | } = CONFIG; 55 | /* 56 | * There are a few things to consider when constructing the redirect_uri: 57 | * - Since Dash apps can have URLs (https://plot.ly/dash/urls), e.g. 58 | * `/page-1/another-page`, and so just appending the /_oauth-redirect 59 | * API path to the end of the current URL (window.location.href) isn't 60 | * safe because the API endpoint is `/_oauth-redirect` not e.g. 61 | * `/page-1/another-page/_oauth-redirect` 62 | * - Dash apps may be served by a proxy which prefixes a path to them. 63 | * This is what happens in Plotly On-Premise's Path-Based-Routing. 64 | * For example, the dash app may be rendered under `/my-dash-app/` 65 | * In this case, we can't just use window.location.origin because there 66 | * that would skip this pathname prefix. The config variable 67 | * `requests_pathname_prefix` contains this prefix (`/my-dash-app/`) 68 | * and is used to prefix all of the front-end API endpoints. 69 | * - Dash apps may be served on a subdomain. window.location.origin picks 70 | * up the subdomain. 71 | */ 72 | return ( 73 | `${solvebio_url}/authorize/?` + 74 | `response_type=${oauth_response_type}&` + 75 | `state=${oauth_state}&` + 76 | `client_id=${oauth_client_id}&` + 77 | `redirect_uri=${window.location.origin}${requests_pathname_prefix}${REDIRECT_URI_PATHNAME}` 78 | ); 79 | } 80 | 81 | oauthPopUp() { 82 | const popupWindow = PopupCenter( 83 | // TODO: Fix https://github.com/solvebio/www.solvebio.com/issues/1179 84 | // this.buildOauthUrl(), 'Authorization', '500', '500' 85 | this.buildOauthUrl(), 'Authorization', '1000', '1000' 86 | ); 87 | if (window.focus) { 88 | popupWindow.focus(); 89 | } 90 | window.popupWindow = popupWindow; 91 | const interval = setInterval(() => { 92 | if(popupWindow.closed) { 93 | clearInterval(interval); 94 | // Check if successful? 95 | window.location.reload(); 96 | } 97 | }, 100); 98 | } 99 | 100 | render() { 101 | return ( 102 |
103 |

{'Secure Dash App'}

104 | 105 |

106 | {'Log in to SolveBio to continue'} 107 |

108 | 109 | 112 | 113 |
114 | 115 | {'This app requires a SolveBio account.'} 116 | 117 |
118 | 119 | {'Contact Support'} 120 | 121 |
122 |
123 | ); 124 | } 125 | } 126 | 127 | ReactDOM.render(, document.getElementById('react-root')); 128 | -------------------------------------------------------------------------------- /recipes/sync_recipes.py: -------------------------------------------------------------------------------- 1 | import click 2 | import recipes.sync_recipe_utils as sr 3 | import solvebio as sb 4 | 5 | __version__ = '1.0.0' 6 | 7 | 8 | @click.group() 9 | @click.option('--access-token', help='Manually provide a SolveBio Access Token') 10 | @click.option('--api-host', help='Override the default SolveBio API host') 11 | @click.option('--api-key', help='Manually provide a SolveBio API key') 12 | @click.pass_context 13 | def sync_recipes(ctx, api_key=None, access_token=None, api_host=None): 14 | 15 | sb.login(api_key=api_key, access_token=access_token, api_host=api_host, 16 | version=__version__, name="SolveBio Recipes") 17 | user = sb.User.retrieve() 18 | 19 | click.echo('Logged-in as: {} ({})'.format( 20 | user.full_name, user.id)) 21 | 22 | 23 | @sync_recipes.command() 24 | @click.argument('recipes_file', nargs=1) 25 | @click.option('--name', help='Name of the recipe') 26 | @click.option('--all', is_flag=True, 27 | help='Apply the selected mode to all recipes in YAML file') 28 | @click.pass_context 29 | def sync(ctx, recipes_file=None, all=False, name=None): 30 | if (all and name): 31 | ctx.fail("Only one of the options --name or --all should be present!") 32 | elif not any([all, name]): 33 | ctx.fail("One of the options --name or --all should be present!") 34 | 35 | try: 36 | yml_recipes = sr.load_recipes_from_yaml(recipes_file) 37 | if all: 38 | for yml_recipe in yml_recipes: 39 | prompt_sync(yml_recipe) 40 | else: 41 | yml_recipe = sr.get_recipe_by_name_from_yml(yml_recipes, name) 42 | if yml_recipe: 43 | prompt_sync(yml_recipe) 44 | 45 | except Exception as e: 46 | ctx.fail(e) 47 | ctx.fail('Invalid path to YAML file with the recipes. Provide the full path to a ' 48 | 'yaml_with_recipes.yml file containing a description of recipes.') 49 | 50 | 51 | @sync_recipes.command() 52 | @click.argument('recipes_file', nargs=1, required=False) 53 | @click.option('--name', help='Name of the recipe') 54 | @click.option('--all', is_flag=True, 55 | help='Apply the selected mode to all recipes in YAML file') 56 | @click.pass_context 57 | def delete(ctx, recipes_file, all=False, name=None): 58 | if (all and name): 59 | ctx.fail("Only one of the options --name or --all should be present!") 60 | elif not any([all, name]): 61 | ctx.fail("One of the options --name or --all should be present!") 62 | 63 | try: 64 | yml_recipes = sr.load_recipes_from_yaml(recipes_file) 65 | if all: 66 | for yml_recipe in yml_recipes: 67 | prompt_delete("{} (v{})".format(yml_recipe['name'], yml_recipe['version'])) 68 | else: 69 | prompt_delete(name) 70 | 71 | except Exception as e: 72 | ctx.fail(e) 73 | ctx.fail('Invalid path to YAML file with the recipes. Provide the full path to a ' 74 | 'yaml_with_recipes.yml file containing a description of recipes.') 75 | 76 | 77 | @sync_recipes.command() 78 | @click.argument('recipes_file', nargs=1) 79 | @click.option('--account-recipes', is_flag=True, help='Export recipes for logged in account') 80 | @click.option('--public-recipes', is_flag=True, help='Export public recipes') 81 | @click.pass_context 82 | def export(ctx, recipes_file, account_recipes, public_recipes): 83 | if account_recipes: 84 | user = sb.User.retrieve() 85 | click.echo("Exporting recipes for account {} to {}." 86 | .format(user['account']['id'], recipes_file)) 87 | sr.export_recipes_to_yaml(sr.get_account_recipes(user), recipes_file) 88 | click.echo("Recipes successfully exported!") 89 | elif public_recipes: 90 | click.echo("Exporting all public recipes to {}.".format(recipes_file)) 91 | sr.export_recipes_to_yaml(sr.get_public_recipes(), recipes_file) 92 | click.echo("Recipes successfully exported!") 93 | else: 94 | ctx.fail("Export mode should be used with --account-recipes or --public-recipes!") 95 | return 96 | 97 | 98 | def prompt_sync(yml_recipe): 99 | if sb.DatasetTemplate.all(name="{} (v{})".format(yml_recipe['name'], yml_recipe['version'])): 100 | if click.confirm("Are you sure you want to sync {} (v{}) recipe?" 101 | .format(yml_recipe['name'], yml_recipe['version'])): 102 | sr.sync_recipe(yml_recipe) 103 | else: 104 | click.echo("Aborted.") 105 | elif click.confirm("Are you sure you want to create {} (v{}) recipe?" 106 | .format(yml_recipe['name'], yml_recipe['version'])): 107 | sr.create_recipe(yml_recipe) 108 | else: 109 | click.echo("Aborted.") 110 | 111 | 112 | def prompt_delete(name): 113 | if sb.DatasetTemplate.all(name=name): 114 | if click.confirm("Are you sure you want to delete {} recipe?".format(name)): 115 | sr.delete_recipe(name) 116 | else: 117 | click.echo("Aborted.") 118 | else: 119 | click.echo("Requested recipe {} doesn't exist in SolveBio!".format(name)) 120 | 121 | 122 | if __name__ == '__main__': 123 | sync_recipes() 124 | -------------------------------------------------------------------------------- /solvebio/cli/tutorial.md: -------------------------------------------------------------------------------- 1 | 2 | ____ _ ____ _ 3 | / ___| ___ | |_ _____| __ )(_) ___ 4 | \___ \ / _ \| \ \ / / _ \ _ \| |/ _ \ 5 | ___) | (_) | |\ V / __/ |_) | | (_) | 6 | |____/ \___/|_| \_/ \___|____/|_|\___/ 7 | For Python 8 | 9 | 10 | # Welcome to the SolveBio Python Tutorial! 11 | 12 | First, open the SolveBio Python shell by typing "solvebio". 13 | 14 | The SolveBio Shell is based on IPython. When you log in, it will automatically pick up your API key. 15 | You can always type "solvebio.help()" within the shell open up the online documentation. 16 | 17 | View this tutorial online: https://www.solvebio.com/docs/python-tutorial 18 | 19 | 20 | ## Navigate the Library 21 | 22 | 23 | To list all available vaults, run: 24 | 25 | Vault.all() 26 | 27 | 28 | Vaults are like filesystems on a computer. They can contain files, 29 | folders, and a special SolveBio-specific object called a Dataset. 30 | 31 | To list all datasets from all vaults, run: 32 | 33 | Dataset.all() 34 | 35 | To retrieve a dataset by its full path, send your account domain, the vault 36 | name, and the dataset path, all separated by a colon: 37 | 38 | Dataset.get_by_full_path('acme:test-vault:/path/to/my/dataset') 39 | 40 | Similarly, to retrieve a publicly available dataset, use the `solvebio` 41 | account domain, the `public` vault, and the appropriate dataset path: 42 | 43 | Dataset.get_by_full_path('solvebio:public:/ICGC/3.0.0-23/Donor') 44 | 45 | SolveBio maintains a list of publicly available datasets. To list them, 46 | run: 47 | 48 | vault = Vault.get_by_full_path('solvebio:public') 49 | vault.datasets() 50 | 51 | You can browse any Vault from the client simply by calling the appropriate 52 | methods on a Vault object: 53 | 54 | vault = Vault.get_by_full_path('solvebio:public') 55 | vault.files() 56 | vault.folders() 57 | vault.datasets() 58 | vault.ls() # Includes files, folders, and datatsets 59 | 60 | 61 | Every user has a dedicated, non-shareable, personal vault which is private 62 | to them. This vault can be obtained by running: 63 | 64 | vault = Vault.get_personal_vault() 65 | vault.name 66 | > 'user-3000' # This is automatically generated based on your User ID 67 | # and cannot be changed. 68 | 69 | 70 | The objects contained in a vault (files, folders, and datasets) can be 71 | retrieved directly using the Object class, or via the `objects` property of a 72 | Vault instance: 73 | 74 | Object.retrieve(488353213969592764) 75 | Object.all(path='/1000G') 76 | Object.all(path='/1000G', vault_id=7205) 77 | Object.all(path='/1000G', vault_name='public') 78 | 79 | vault = Vault.get_personal_vault() 80 | vault.objects.all(path='/1000G') 81 | vault.objects.all(query='GRC') 82 | 83 | 84 | ## Query a Dataset 85 | 86 | Every dataset in SolveBio can be queried the same way. You can build queries manually in the Python shell, or use our visual Workbench (https://www.solvebio.com/workbench). 87 | 88 | In this example, we will query the latest Variants dataset from ClinVar. 89 | 90 | dataset = Dataset.get_by_full_path('solvebio:public:/ClinVar/3.7.4-2017-01-04/Variants-GRCh38') 91 | dataset.query() 92 | 93 | 94 | The "query()" function returns a Python iterator so you can loop through all the results easily. 95 | 96 | To examine a single result more closely, you may treat the query response as a list of dictionaries: 97 | 98 | dataset.query()[0] 99 | 100 | 101 | You can also slice the result set like any other Python list: 102 | 103 | dataset.query()[0:100] 104 | 105 | 106 | ## Filter a Dataset 107 | 108 | To narrow down your query, you can filter on any field. For example, to get all variants in ClinVar that are Pathogenic, you would filter on the `clinical_significance` field for "Pathogenic": 109 | 110 | dataset.query().filter(clinical_significance='Pathogenic') 111 | 112 | 113 | By default, adding more filters will result in a boolean AND query (all filters must match): 114 | 115 | dataset.query().filter(clinical_significance='Pathogenic', review_status='single') 116 | 117 | 118 | Use the "Filter" class to do more advanced filtering. For example, combine a few filters using boolean OR: 119 | 120 | filters = Filter(clinical_significance='Pathogenic') | Filter(clinical_significance='Benign') 121 | dataset.query().filter(filters) 122 | 123 | 124 | ## Genomic Datasets 125 | 126 | Some SolveBio datasets are suffixed with a genome build (GRCh37, GRCh38, 127 | NCBI36) to indicate they are genomic datasets. Please ensure that you 128 | are using the dataset whose genomic build is compatible with your other 129 | tools and procedures. 130 | 131 | On genomic datasets, you may query by position (single nucleotide) or by range: 132 | 133 | dataset.query().position('chr1', 976629) 134 | > ... 135 | dataset.query().range('chr1', 976629, 1000000) 136 | > ... 137 | 138 | 139 | Position and range queries return all results that overlap with the specified coordinate(s). 140 | Add the parameter `exact=True` to request exact matches. 141 | 142 | 143 | dataset.query().position('chr1', 883516, exact=True) 144 | > ... 145 | dataset.query().range('chr9', 136289550, 136289579, exact=True) 146 | > ... 147 | 148 | 149 | ## Next Steps 150 | 151 | For more information on queries and filters, see the API reference: https://docs.solvebio.com/guides/quickstarts/python/ 152 | -------------------------------------------------------------------------------- /examples/template_2to3.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convert a template's expressions to Python 3. 3 | 4 | This script will convert all expressions in a template to Python 3 using the 2to3 tool. 5 | It will also show a diff of the changes detected and optionally save the changes to a new template. 6 | 7 | Please install the following dependencies for best results: 8 | 9 | 10 | pip install solvebio click black 11 | 12 | 13 | ## Usage 14 | 15 | Perform a diff of the changes detected in a template: 16 | 17 | python template_2to3.py YOUR_TEMPLATE_ID --api-host https://API_HOST.aws.quartz.bio --diff 18 | 19 | 20 | Format the diff using Black: 21 | 22 | python template_2to3.py YOUR_TEMPLATE_ID --api-host https://API_HOST.aws.quartz.bio --diff --format-diff 23 | 24 | 25 | Save a new copy of the template with the upgraded Python 3 expression: 26 | 27 | python template_2to3.py YOUR_TEMPLATE_ID --api-host https://API_HOST.aws.quartz.bio --save 28 | 29 | 30 | Overwrite the existing template with upgraded Python 3 expression: 31 | 32 | python template_2to3.py YOUR_TEMPLATE_ID --api-host https://API_HOST.aws.quartz.bio --save --overwrite 33 | 34 | 35 | """ 36 | 37 | import solvebio 38 | import click 39 | import subprocess 40 | import os 41 | import re 42 | import difflib 43 | import tempfile 44 | 45 | 46 | def convert_to_python3(code): 47 | # Run 2to3 on the given code 48 | with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: 49 | f.write(code) 50 | f.flush() 51 | process = subprocess.Popen( 52 | ["2to3", "-w", f.name], 53 | stdout=subprocess.PIPE, 54 | stderr=subprocess.PIPE, 55 | text=True, 56 | ) 57 | stdout, stderr = process.communicate() 58 | 59 | if process.returncode == 0: 60 | # Successfully converted 61 | with open(f.name, "r") as read_f: 62 | new_code = read_f.read() 63 | else: 64 | # Conversion failed 65 | print(f"Error: {stderr}") 66 | new_code = "FAILED TO CONVERT" 67 | 68 | f.close() 69 | os.remove(f.name) 70 | 71 | return new_code 72 | 73 | 74 | def diff_expressions(old, new): 75 | diff = list(difflib.ndiff(old.splitlines(), new.splitlines())) 76 | return "\n".join(diff) 77 | 78 | 79 | def format_expression(code): 80 | try: 81 | import black 82 | except ImportError: 83 | print("Black is not installed. Please install it using 'pip install black'.") 84 | return code 85 | 86 | try: 87 | mode = black.Mode(line_length=120, target_versions={black.TargetVersion.PY38}) 88 | formatted_code = black.format_str(code, mode=mode) 89 | return formatted_code 90 | except black.NothingChanged: 91 | return code 92 | 93 | 94 | @click.command() 95 | @click.argument("template_id") 96 | @click.option("--api-host", default="https://api.solvebio.com") 97 | @click.option("--diff", is_flag=True) 98 | @click.option("--format-diff", is_flag=True) 99 | @click.option("--save", is_flag=True) 100 | @click.option("--overwrite", is_flag=True) 101 | def convert_template( 102 | template_id, api_host, diff=False, format_diff=False, save=False, overwrite=False 103 | ): 104 | """ 105 | Convert a template's expressions to Python 3. 106 | 107 | Arguments: 108 | 109 | - TEMPLATE_ID: The ID of the template to convert. 110 | 111 | Options: 112 | 113 | --diff: Show a diff of the changes detected. 114 | --format-diff: Format the diff output using Black. 115 | --save: Save the changes to a new template. 116 | --overwrite: Overwrite the existing template with the new expressions. 117 | 118 | """ 119 | 120 | solvebio.login(api_host=api_host) 121 | 122 | if not save: 123 | print( 124 | "Running in dry-run mode. Use --save to save changes to a new template and " 125 | "--overwrite to modify the template in-place." 126 | ) 127 | 128 | # Retrieve a template from EDP 129 | template = solvebio.DatasetTemplate.retrieve(template_id) 130 | # Convert each field to Python 3 131 | for field in template.fields: 132 | if not field["expression"]: 133 | continue 134 | 135 | # Remove linebreaks and consecutive spaces from code snippet 136 | old_expression = re.sub( 137 | r"\s+", " ", field["expression"].replace("\n", " ") 138 | ).strip() 139 | new_expression = convert_to_python3(old_expression) 140 | if new_expression != old_expression: 141 | # Save the new expression 142 | field["expression"] = new_expression 143 | 144 | # Show a "diff" format for the difference detected 145 | print(f"Field {field['name']} changes detected") 146 | 147 | if diff: 148 | if format_diff: 149 | old_expression = format_expression(old_expression) 150 | new_expression = format_expression(new_expression) 151 | 152 | print(diff_expressions(old_expression, new_expression)) 153 | print() 154 | 155 | if save: 156 | if overwrite: 157 | print( 158 | f"Ovewriting existing template {template.id} with Python 3 expressions." 159 | ) 160 | template.save() 161 | else: 162 | print("Creating a new template with Python 3 expressions.") 163 | template.name = f"{template.name} py3" 164 | solvebio.DatasetTemplate.create(**template) 165 | 166 | print(f"Template {template.id} saved with Python 3 expressions.") 167 | 168 | 169 | if __name__ == "__main__": 170 | convert_template() 171 | -------------------------------------------------------------------------------- /solvebio/test/test_dataset_migrations.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import mock 4 | 5 | from .helper import SolveBioTestCase 6 | 7 | from solvebio.test.client_mocks import fake_migration_create 8 | 9 | 10 | class TestDatasetMigrations(SolveBioTestCase): 11 | 12 | @mock.patch('solvebio.resource.DatasetMigration.create') 13 | def test_migration_from_query(self, Create): 14 | Create.side_effect = fake_migration_create 15 | 16 | source = self.client.Dataset(1) 17 | target = self.client.Dataset(2) 18 | 19 | query = source\ 20 | .query(limit=10, fields=['my_field'])\ 21 | .filter(my_field=999) 22 | migration = query.migrate(target=target, 23 | commit_mode='overwrite', 24 | follow=False) 25 | self.assertEqual(migration.source_id, source.id) 26 | self.assertEqual(migration.target_id, target.id) 27 | self.assertEqual(migration.commit_mode, 'overwrite') 28 | self.assertEqual(migration.source_params['filters'], 29 | [('my_field', 999)]) 30 | 31 | @mock.patch('solvebio.resource.DatasetMigration.create') 32 | def test_migration_from_query_target_id(self, Create): 33 | Create.side_effect = fake_migration_create 34 | 35 | source = self.client.Dataset(1) 36 | target = self.client.Dataset(2) 37 | 38 | query = source\ 39 | .query(limit=10, fields=['my_field'])\ 40 | .filter(my_field=999) 41 | migration = query.migrate(target=target.id, 42 | commit_mode='overwrite', 43 | follow=False) 44 | self.assertEqual(migration.source_id, source.id) 45 | self.assertEqual(migration.target_id, target.id) 46 | self.assertEqual(migration.commit_mode, 'overwrite') 47 | self.assertEqual(migration.source_params['filters'], 48 | [('my_field', 999)]) 49 | 50 | @mock.patch('solvebio.resource.DatasetMigration.create') 51 | def test_migration_from_query_target_object(self, Create): 52 | Create.side_effect = fake_migration_create 53 | 54 | source = self.client.Object(1) 55 | source.dataset_id = source.id 56 | source.object_type = 'dataset' 57 | target = self.client.Object(2) 58 | target.object_type = 'dataset' 59 | 60 | query = source\ 61 | .query(limit=10, fields=['my_field'])\ 62 | .filter(my_field=999) 63 | migration = query.migrate(target=target, 64 | commit_mode='overwrite', 65 | follow=False) 66 | self.assertEqual(migration.source_id, source.id) 67 | self.assertEqual(migration.target_id, target.id) 68 | self.assertEqual(migration.commit_mode, 'overwrite') 69 | self.assertEqual(migration.source_params['filters'], 70 | [('my_field', 999)]) 71 | 72 | @mock.patch('solvebio.resource.DatasetMigration.create') 73 | def test_migration_from_query_target_object_id(self, Create): 74 | Create.side_effect = fake_migration_create 75 | 76 | source = self.client.Dataset(1) 77 | target = self.client.Object(2, object_type='dataset') 78 | 79 | query = source\ 80 | .query(limit=10, fields=['my_field'])\ 81 | .filter(my_field=999) 82 | migration = query.migrate(target=target.id, 83 | commit_mode='overwrite', 84 | follow=False) 85 | self.assertEqual(migration.source_id, source.id) 86 | self.assertEqual(migration.target_id, target.id) 87 | self.assertEqual(migration.commit_mode, 'overwrite') 88 | self.assertEqual(migration.source_params['filters'], 89 | [('my_field', 999)]) 90 | 91 | @mock.patch('solvebio.resource.DatasetMigration.create') 92 | def test_migration_target_dataset_id(self, Create): 93 | Create.side_effect = fake_migration_create 94 | 95 | source = self.client.Dataset(1) 96 | target = self.client.Dataset(2) 97 | migration = source.migrate(target=target.id, follow=False) 98 | self.assertEqual(migration.source_id, source.id) 99 | self.assertEqual(migration.target_id, target.id) 100 | self.assertEqual(migration.commit_mode, 'append') 101 | 102 | @mock.patch('solvebio.resource.DatasetMigration.create') 103 | def test_migration_target_dataset_dataset(self, Create): 104 | """Target is a Dataset object""" 105 | Create.side_effect = fake_migration_create 106 | 107 | source = self.client.Dataset(1) 108 | target = self.client.Dataset(2) 109 | migration = source.migrate(target=target, follow=False) 110 | self.assertEqual(migration.source_id, source.id) 111 | self.assertEqual(migration.target_id, target.id) 112 | self.assertEqual(migration.commit_mode, 'append') 113 | 114 | @mock.patch('solvebio.resource.DatasetMigration.create') 115 | def test_migration_target_dataset_object(self, Create): 116 | """Target is an Object of object_type=dataset""" 117 | Create.side_effect = fake_migration_create 118 | 119 | source = self.client.Object(1) 120 | source['object_type'] = 'dataset' 121 | target = self.client.Object(2) 122 | target['object_type'] = 'dataset' 123 | migration = source.migrate(target=target, follow=False) 124 | self.assertEqual(migration.source_id, source.id) 125 | self.assertEqual(migration.target_id, target.id) 126 | self.assertEqual(migration.commit_mode, 'append') 127 | 128 | # Test with source_params 129 | source_params = { 130 | 'fields': ['my_field'] 131 | } 132 | migration = source.migrate( 133 | target=target, source_params=source_params, 134 | commit_mode='overwrite', follow=False) 135 | self.assertEqual(migration.commit_mode, 'overwrite') 136 | self.assertEqual(migration.source_params['fields'], 137 | source_params['fields']) 138 | self.assertEqual(migration.source_id, source.id) 139 | self.assertEqual(migration.target_id, target.id) 140 | -------------------------------------------------------------------------------- /solvebio/auth.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Literal, Tuple 3 | 4 | from urllib.parse import urlparse 5 | 6 | import logging 7 | 8 | from requests.auth import AuthBase 9 | import requests 10 | 11 | from solvebio import SolveError 12 | from solvebio.cli.credentials import get_credentials, netrc 13 | 14 | logger = logging.getLogger("solvebio") 15 | 16 | 17 | class SolveBioTokenAuth(AuthBase): 18 | """Custom auth handler for SolveBio API token authentication""" 19 | 20 | def __init__(self, token=None, token_type="Token"): 21 | self.token = token 22 | self.token_type = token_type 23 | 24 | def __call__(self, r): 25 | if self.token: 26 | r.headers["Authorization"] = "{0} {1}".format(self.token_type, self.token) 27 | return r 28 | 29 | def __repr__(self): 30 | if self.token: 31 | return self.token_type 32 | else: 33 | return "Anonymous" 34 | 35 | 36 | def authenticate( 37 | host: str, 38 | token: str, 39 | token_type: Literal["Bearer", "Token"], 40 | *, 41 | raise_on_missing: bool = True, 42 | debug: bool = False 43 | ) -> Tuple[str, SolveBioTokenAuth]: 44 | """ 45 | Sets login credentials for SolveBio API authentication. 46 | 47 | :param str host: API host url 48 | :param str token: API access token or API key 49 | :param str token_type: API token type. `Bearer` is used for access tokens, while `Token` is used for API Keys. 50 | :param bool raise_on_missing: Raise an exception if no credentials are available. 51 | """ 52 | # used for debugging 53 | source_host = None 54 | source_token = None 55 | 56 | # Find credentials from environment variables 57 | if not host: 58 | host = ( 59 | os.environ.get("QUARTZBIO_API_HOST", None) or 60 | os.environ.get("EDP_API_HOST", None) or 61 | os.environ.get("SOLVEBIO_API_HOST", None) 62 | ) 63 | 64 | if not token: 65 | api_key = ( 66 | os.environ.get("QUARTZBIO_API_KEY", None) or 67 | os.environ.get("EDP_API_KEY", None) or 68 | os.environ.get("SOLVEBIO_API_KEY", None) 69 | ) 70 | 71 | access_token = ( 72 | os.environ.get("QUARTZ_ACCESS_TOKEN", None) or 73 | os.environ.get("EDP_ACCESS_TOKEN", None) or 74 | os.environ.get("SOLVEBIO_ACCESS_TOKEN", None) 75 | ) 76 | 77 | if access_token: 78 | token = access_token 79 | token_type = "Bearer" 80 | elif api_key: 81 | token = api_key 82 | token_type = "Token" 83 | 84 | if token: 85 | source_token = 'envvars' 86 | else: 87 | source_token = 'params' 88 | 89 | # Find credentials from local credentials file 90 | if not token: 91 | if creds := get_credentials(host): 92 | token_type = creds.token_type 93 | token = creds.token 94 | 95 | if host is None: 96 | # this happens when user/ennvars provided no API host for the login command 97 | # but the credentials file still contains login credentials 98 | host = creds.api_host 99 | 100 | if host: 101 | source_host = 'creds' 102 | if token: 103 | source_token = 'creds' 104 | 105 | if debug: 106 | # this will tell the user where QB Client found the credentials from 107 | creds_path = netrc.path() 108 | print('\n'.join([ 109 | "Login Debug:", 110 | f"--> Host: {host}\n (source: {source_host})", 111 | f"--> Token Type: {token_type}\n (source: {source_token})", 112 | "\n1) source: params", 113 | " Means that you've passed this through the login CLI command:", 114 | " quartzbio login --host --access_token ", 115 | "\n or the quartzbio.login function:", 116 | " import quartzbio", 117 | " quartzbio.login(debug=True)", 118 | "\n2) source: creds", 119 | " Means that the QB client has saved your credentials in:", 120 | f" {creds_path}", 121 | "\n3) source: envvars", 122 | " Means that you've set your credentials through environment variables:", 123 | " QUARTZBIO_API_HOST", 124 | " QUARTZBIO_ACCESS_TOKEN", 125 | " QUARTZBIO_API_KEY", 126 | ])) 127 | 128 | if not host: 129 | if raise_on_missing and not debug: 130 | raise SolveError("No QuartzBio API host is set") 131 | else: 132 | return host, None 133 | 134 | host = validate_api_host_url(host) 135 | 136 | # If the domain ends with .solvebio.com, determine if 137 | # we are being redirected. If so, update the url with the new host 138 | # and log a warning. 139 | if host and host.rstrip("/").endswith(".api.solvebio.com"): 140 | old_host = host.rstrip("/") 141 | response = requests.head(old_host, allow_redirects=True) 142 | # Strip the port number from the host for comparison 143 | new_host = validate_api_host_url(response.url).rstrip("/").replace(":443", "") 144 | 145 | if old_host != new_host: 146 | logger.warning( 147 | 'API host redirected from "{}" to "{}", ' 148 | "please update your local credentials file".format(old_host, new_host) 149 | ) 150 | host = new_host 151 | 152 | if token is not None: 153 | auth = SolveBioTokenAuth(token, token_type) 154 | else: 155 | auth = None 156 | 157 | # TODO: warn user if WWW url is provided in edp_login! 158 | 159 | from solvebio import _set_cached_api_host 160 | _set_cached_api_host(host) 161 | 162 | return host, auth 163 | 164 | 165 | def validate_api_host_url(url): 166 | """ 167 | Validate SolveBio API host url. 168 | 169 | Valid urls must not be empty and 170 | must contain either HTTP or HTTPS scheme. 171 | """ 172 | 173 | # Default to https if no scheme is set 174 | if "://" not in url: 175 | url = "https://" + url 176 | 177 | parsed = urlparse(url) 178 | if parsed.scheme not in ["http", "https"]: 179 | raise SolveError( 180 | "Invalid API host: %s. " "Missing url scheme (HTTP or HTTPS)." % url 181 | ) 182 | elif not parsed.netloc: 183 | raise SolveError("Invalid API host: %s." % url) 184 | 185 | return parsed.geturl() 186 | -------------------------------------------------------------------------------- /installer: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # This is the guided installer for the SolveBio Python Client: 4 | # 5 | # curl -skL install.solvebio.com/python | bash 6 | # 7 | 8 | echo 9 | echo " ____ _ ____ _" 10 | echo " / ___| ___ | |_ _____| __ )(_) ___" 11 | echo " \___ \ / _ \| \ \ / / _ \ _ \| |/ _ \\" 12 | echo " ___) | (_) | |\ V / __/ |_) | | (_) |" 13 | echo " |____/ \___/|_| \_/ \___|____/|_|\___/" 14 | echo 15 | echo " Copyright © Solve, Inc. . All rights reserved." 16 | echo 17 | 18 | shopt -s extglob 19 | 20 | function fail_exit() { 21 | echo 22 | echo " ##################################################" 23 | echo 24 | echo " Sorry, SolveBio for Python could not be installed." 25 | echo 26 | echo " You can try manually installing the module with:" 27 | echo " pip install solvebio" 28 | echo 29 | echo " Contact us at support@solvebio.com for help." 30 | echo " In your email, please copy/paste the output of:" 31 | echo " cat ${LOG}" 32 | echo 33 | echo " ##################################################" 34 | echo 35 | exit 36 | } 37 | 38 | trap ctrl_c INT 39 | 40 | function ctrl_c() { 41 | echo 42 | echo 43 | echo " Installation was aborted..." 44 | fail_exit 45 | } 46 | 47 | echo " Installing the SolveBio Python Client..." 48 | 49 | # Setup the log 50 | LOG=/tmp/solvebio-python.log 51 | echo "SolveBio Python Guided Installer log" > $LOG 52 | echo `date` >> $LOG 53 | 54 | # Check Python versions 55 | echo " Looking for a supported Python installation..." 56 | PATHS=`echo $PATH | sed -e 's/:/ /g'` 57 | 58 | # For each path in $PATH, look for python 59 | for path in $PATHS; do 60 | PYTHONS=`find $path -regex ".*/python*" -o -regex ".*/python2.[567]" 2>> $LOG | sort` 61 | 62 | for PYTHON in $PYTHONS; do 63 | if [[ ! -x $PYTHON ]] ; then 64 | continue 65 | fi 66 | 67 | PYTHON_VERSION=`$PYTHON -V 2>&1 | cut -d " " -f 2` 68 | if [[ $PYTHON_VERSION =~ 2.[67].[0-9] ]]; then 69 | PYTHON_FOUND='1' 70 | break 71 | fi 72 | done 73 | 74 | if [ "$PYTHON_FOUND" == '1' ]; then 75 | break 76 | fi 77 | done 78 | 79 | if [ "$PYTHON_FOUND" == '1' ]; then 80 | echo " Found Python ${PYTHON_VERSION} at ${PYTHON}!" 81 | # TODO: allow optional Python path 82 | # echo -n "Which Python do you want to use (press enter for: ${PYTHON}) ? " 83 | # read PYTHON_ALT 84 | # if [[ -x $PYTHON_ALT ]]; then 85 | # PYTHON=$PYTHON_ALT 86 | # fi 87 | echo " Using ${PYTHON}..." 88 | else 89 | echo "Error: SolveBio for Python requires Python >= 2.6.5 and < 3.0" 90 | fail_exit 91 | fi 92 | 93 | # Check OpenSSL version 94 | OPENSSL_VERSION=`$PYTHON -c "import ssl; print ssl.OPENSSL_VERSION" 2>> $LOG` 95 | if [ "$OPENSSL_VERSION" == "" ]; then 96 | echo 97 | echo 98 | echo "Error: SolveBio for Python requires a more recent version of Python-OpenSSL." 99 | fail_exit 100 | fi 101 | 102 | OPENSSL_VERSION_MAJOR=`$PYTHON -c "import ssl; print ssl.OPENSSL_VERSION_INFO[0]"` 103 | OPENSSL_VERSION_MINOR=`$PYTHON -c "import ssl; print ssl.OPENSSL_VERSION_INFO[1]"` 104 | OPENSSL_VERSION_PATCH=`$PYTHON -c "import ssl; print ssl.OPENSSL_VERSION_INFO[2]"` 105 | 106 | if [ "$OPENSSL_VERSION_MAJOR" -lt "1" ]; then 107 | if [ "$OPENSSL_VERSION_MINOR" -lt "9" ] || [ "$OPENSSL_VERSION_PATCH" -lt "8" ]; then 108 | echo 109 | echo "Error: SolveBio for Python requires OpenSSL >= 0.9.8" 110 | echo " Your Python SSL module is linked to $OPENSSL_VERSION" 111 | fail_exit 112 | fi 113 | fi 114 | 115 | # Detect if in Virtualenv (use sudo if not) 116 | VIRTUALENV=`$PYTHON -c "import sys; print sys.real_prefix" 2>> $LOG` 117 | if [ "$VIRTUALENV" == "" ]; then 118 | PYTHON_SUDO="sudo ${PYTHON}" 119 | echo " IMPORTANT: Your computer's password may be required. It will NOT be sent to SolveBio." 120 | else 121 | PYTHON_SUDO=$PYTHON 122 | fi 123 | 124 | # Check for working pip 125 | PIP_VERSION=`$PYTHON -m pip --version 2>&1` 126 | if [[ ! "$PIP_VERSION" =~ (^pip .*) ]]; then 127 | echo " It looks like pip is not installed. We will now attempt to install it." 128 | $PYTHON_SUDO -m easy_install -q pip 2>&1 >> $LOG 129 | 130 | PIP_VERSION=`$PYTHON -m pip --version 2>&1` 131 | if [[ ! "$PIP_VERSION" =~ (^pip .*) ]]; then 132 | echo 133 | echo "Error: There seems to be a problem installing pip..." 134 | echo " To install pip manually, run the following command:" 135 | echo " curl https://raw.github.com/pypa/pip/master/contrib/get-pip.py | ${PYTHON}" 136 | echo 137 | fail_exit 138 | else 139 | echo " pip was successfully installed!" 140 | fi 141 | fi 142 | 143 | PIP_FLAGS="" 144 | # disable the use of wheels if pip supports them 145 | if [ $($PYTHON -m pip install --help | grep "\-\-no\-use\-wheel" -c) -ne 0 ]; then 146 | # force-reinstall in case the user has the .whl installed already 147 | PIP_FLAGS="--force-reinstall --no-use-wheel" 148 | fi 149 | 150 | # Detect/install gnureadline only on Mac OS X 151 | UNAME=`uname` 152 | if [[ "$UNAME" == 'Darwin' ]]; then 153 | READLINE=`$PYTHON -c "import gnureadline; print gnureadline.__name__" 2>> $LOG` 154 | if [ "$READLINE" != "gnureadline" ]; then 155 | echo " Installing gnureadline..." 156 | $PYTHON_SUDO -m pip install gnureadline $PIP_FLAGS 2>&1 >> $LOG 157 | if [ $? -ne 0 ]; then 158 | echo 159 | echo "Warning: gnureadline could not be installed." 160 | echo 161 | fi 162 | fi 163 | fi 164 | 165 | IPYTHON=`$PYTHON -c "import IPython; print IPython.__name__" 2>> $LOG` 166 | if [ "$IPYTHON" != "IPython" ]; then 167 | echo " Installing IPython..." 168 | $PYTHON_SUDO -m pip install ipython $PIP_FLAGS 2>&1 >> $LOG 169 | 170 | if [ $? -ne 0 ]; then 171 | fail_exit 172 | fi 173 | fi 174 | 175 | echo " Installing SolveBio for Python..." 176 | $PYTHON_SUDO -m pip install solvebio $PIP_FLAGS --upgrade 2>&1 >> $LOG 177 | 178 | INSTALL_COUNT=`$PYTHON -m pip freeze | grep "^solvebio=" -c` 179 | if [ "$INSTALL_COUNT" -ge "0" ]; then 180 | VERSION=`$PYTHON -c "import solvebio; print solvebio.version.VERSION"` 181 | echo 182 | echo " ##############################################" 183 | echo 184 | echo " Success! SolveBio for Python v${VERSION} is now installed." 185 | echo 186 | echo " Please run 'solvebio login' to finish the setup." 187 | echo 188 | echo " ##############################################" 189 | echo 190 | else 191 | fail_exit 192 | fi 193 | --------------------------------------------------------------------------------