├── docs ├── _static │ └── .gitkeep ├── _templates │ └── .gitkeep ├── Makefile ├── make.bat ├── ohapi.rst ├── index.rst ├── tests.rst ├── conf.py └── cli.rst ├── testing_extras ├── empty_file.txt ├── lorem_ipsum_partial.txt └── lorem_ipsum.txt ├── ohapi ├── tests │ ├── data │ │ ├── test_directory │ │ │ ├── file_1.json │ │ │ └── file_2.json │ │ ├── test_id_dir │ │ │ └── sample.txt │ │ ├── metadata_proj_file_key_works.csv │ │ └── metadata_proj_member_key.csv │ ├── _config_params_api.py.example │ ├── test_cassettes.py │ ├── test_projects.py │ ├── test_test.py │ └── test_api.py ├── __init__.py ├── cassettes │ ├── test_oauth2_token_exchange__invalid_refresh.yaml │ ├── test_oauth2_token_exchange__invalid_code.yaml │ ├── test_oauth2_token_exchange__invalid_client.yaml │ ├── test_oauth2_token_exchange__invalid_secret.yaml │ ├── test_update_data_expired_master_access_token.yaml │ ├── test_update_data_invalid_master_access_token.yaml │ ├── __init__.py │ ├── test_get_page_invalid_access_token.yaml │ ├── test_get_page_with_results.yaml │ ├── test_message_valid_access_token.yaml │ ├── test_delete_file__valid_access_token.yaml │ ├── test_delete_file_project_member_id_given.yaml │ ├── test_message_all_members_true_project_member_id_none.yaml │ ├── test_oauth2_token_exchange__valid_refresh.yaml │ ├── test_delete_file_project_member_id_invalid.yaml │ ├── test_message_expired_access_token.yaml │ ├── test_message_invalid_access_token.yaml │ ├── test_delete_file__expired_access_token.yaml │ ├── test_delete_file__invalid_access_token.yaml │ ├── test_message_all_members_false_project_member_id_not_none_valid.yaml │ ├── test_oauth2_token_exchange__valid_code.yaml │ ├── test_upload_file_invalid_metadata_without_description.yaml │ ├── test_upload_aws_invalid_metadata_with_description.yaml │ ├── test_upload_aws_invalid_metadata_without_description.yaml │ ├── test_upload_file_invalid_metadata_with_description.yaml │ ├── test_upload_aws_expired_access_token.yaml │ ├── test_upload_aws_invalid_access_token.yaml │ ├── test_message_all_members_false_projectmemberid_has_invalid_char.yaml │ ├── test_message_all_members_false_projectmemberid_has_invalid_digit.yaml │ ├── test_upload_file_expired_access_token.yaml │ ├── test_upload_file_remote_info_not_none_valid.yaml │ ├── test_upload_file_remote_info_not_none_matching_file_size.yaml │ ├── test_upload_file_remote_info_not_none_invalid_metadata.yaml │ ├── test_upload_file_remote_info_not_none_invalid_metadata_with_desc.yaml │ ├── test_upload_file_remote_info_not_none_expired_access_token.yaml │ ├── test_upload_file_remote_info_not_none_invalid_access_token.yaml │ ├── test_update_data_valid_master_access_token.yaml │ ├── test_upload_aws_valid_access_token.yaml │ ├── test_upload_stream_valid.yaml │ ├── test_upload_valid_file_valid_access_token.yaml │ └── test_download_file_valid_url.yaml ├── public.py ├── projects.py ├── api.py └── utils_fs.py ├── setup.cfg ├── .coveragerc ├── .hound.yml ├── .pep257 ├── .flake8 ├── dev-requirements.txt ├── .travis.yml ├── .github ├── pull_request_template.md └── issue_template.md ├── LICENSE ├── .gitignore ├── setup.py ├── CONTRIBUTING.md └── README.md /docs/_static/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/_templates/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testing_extras/empty_file.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ohapi/tests/data/test_directory/file_1.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ohapi/tests/data/test_directory/file_2.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ohapi/tests/data/test_id_dir/sample.txt: -------------------------------------------------------------------------------- 1 | 12345678 -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | */__init__.py 4 | */tests/* 5 | -------------------------------------------------------------------------------- /.hound.yml: -------------------------------------------------------------------------------- 1 | python: 2 | enabled: true 3 | config_file: .flake8 4 | -------------------------------------------------------------------------------- /.pep257: -------------------------------------------------------------------------------- 1 | [pep257] 2 | ignore = D100,D102,D105,D200,D203,D204,D205,D400 3 | -------------------------------------------------------------------------------- /ohapi/__init__.py: -------------------------------------------------------------------------------- 1 | from .projects import OHProject # noqa 2 | from . import api, command_line, public # noqa 3 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = 3 | .git, 4 | __pycache__, 5 | docs, 6 | ignore = D100,D102,D105,D200,D203,D204,D205,D400 7 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | pytest-cov 2 | vcrpy 3 | flake8 4 | arrow 5 | humanfriendly 6 | requests 7 | click 8 | mock 9 | sphinx 10 | sphinx-click 11 | -------------------------------------------------------------------------------- /ohapi/tests/data/metadata_proj_file_key_works.csv: -------------------------------------------------------------------------------- 1 | filename,tags,description,md5,creation_date 2 | testdata.txt,,,fa61a92e21a2597900cbde09d8ddbc1a,2016-08-23T15:23:22.277060+00:00 3 | testdata.json,json,,577da9879649acaf17226a6461bd19c8,2016-08-23T16:06:16.415039+00:00 4 | -------------------------------------------------------------------------------- /testing_extras/lorem_ipsum_partial.txt: -------------------------------------------------------------------------------- 1 | Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor 2 | incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis 3 | nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. 4 | -------------------------------------------------------------------------------- /ohapi/tests/_config_params_api.py.example: -------------------------------------------------------------------------------- 1 | params = { 2 | 'ACCESS_TOKEN': 'ACCESS-TOKEN-HERE', 3 | 'CLIENT_ID_VALID': 'CLIENT-ID-HERE', 4 | 'CLIENT_SECRET_VALID': 'CLIENT-SECRET-HERE', 5 | 'CODE_VALID': 'CODE-HERE', 6 | 'REFRESH_TOKEN_VALID': 'REFRESH-TOKEN-HERE', 7 | } 8 | -------------------------------------------------------------------------------- /ohapi/tests/test_cassettes.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from ohapi.cassettes import get_vcr, valid_cassettes 4 | 5 | 6 | class CasettesTest(TestCase): 7 | def setUp(self): 8 | pass 9 | 10 | def test_get_vcr(self): 11 | get_vcr() 12 | 13 | def test_valid_cassettes(self): 14 | valid_cassettes() 15 | -------------------------------------------------------------------------------- /ohapi/tests/data/metadata_proj_member_key.csv: -------------------------------------------------------------------------------- 1 | project_member_id,filename,tags,description,md5,creation_date 2 | 01234567,testdata.txt,,,fa61a92e21a2597900cbde09d8ddbc1a,2016-08-23T15:23:22.277060+00:00 3 | 01234567,testdata.json,json,,577da9879649acaf17226a6461bd19c8,2016-08-23T16:06:16.415039+00:00 4 | 12345678,testdata.txt,,,fa61a92e21a2597900cbde09d8ddbc1a,2016-09-20T10:10:59.863201+00:00 5 | 12345678,testdata.json,json,,577da9879649acaf17226a6461bd19c8,2016-09-20T10:10:59.859201+00:00 6 | -------------------------------------------------------------------------------- /testing_extras/lorem_ipsum.txt: -------------------------------------------------------------------------------- 1 | Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor 2 | incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis 3 | nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. 4 | Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore 5 | eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt 6 | in culpa qui officia deserunt mollit anim id est laborum. 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.5" 4 | - "3.6" 5 | 6 | install: 7 | - pip install -e . 8 | - pip install -r dev-requirements.txt 9 | before_script: 10 | - mkdir bin 11 | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > bin/cc-test-reporter 12 | - chmod +x bin/cc-test-reporter 13 | - bin/cc-test-reporter before-build 14 | 15 | script: 16 | - py.test 17 | - py.test --cov=. --cov-report xml:coverage.xml 18 | 19 | after_success: 20 | - bin/cc-test-reporter after-build -t coverage.py 21 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = open-humans-api 8 | SOURCEDIR = . 9 | BUILDDIR = docs_html#_build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=open-humans-api 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/ohapi.rst: -------------------------------------------------------------------------------- 1 | ``ohapi`` and submodules 2 | ======================== 3 | 4 | The ``open-humans-api`` package will be available in Python as ``ohapi``. 5 | 6 | In turn ``ohapi`` contains multiple sub modules to interact with different things 7 | like the project & member-based APIs, the public data API and methods for 8 | working with files on the local filesystem. 9 | 10 | ohapi.api module 11 | ---------------- 12 | 13 | .. automodule:: ohapi.api 14 | :members: 15 | :undoc-members: 16 | :show-inheritance: 17 | 18 | 19 | ohapi.public module 20 | ------------------- 21 | 22 | .. automodule:: ohapi.public 23 | :members: 24 | :undoc-members: 25 | :show-inheritance: 26 | 27 | ohapi.projects module 28 | --------------------- 29 | 30 | .. automodule:: ohapi.projects 31 | :members: 32 | :undoc-members: 33 | :show-inheritance: 34 | 35 | ohapi.utils\_fs module 36 | ---------------------- 37 | 38 | .. automodule:: ohapi.utils_fs 39 | :members: 40 | :undoc-members: 41 | :show-inheritance: 42 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_oauth2_token_exchange__invalid_refresh.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: refresh_token=REFRESHTOKEN&grant_type=refresh_token 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Authorization: [XXXXXXXX] 8 | Connection: [keep-alive] 9 | Content-Length: ['58'] 10 | Content-Type: [application/x-www-form-urlencoded] 11 | User-Agent: [python-requests/2.18.4] 12 | method: POST 13 | uri: https://www.openhumans.org/oauth2/token/ 14 | response: 15 | body: {string: '{"error": "invalid_grant"}'} 16 | headers: 17 | Cache-Control: [no-store] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sat, 10 Mar 2018 04:03:27 GMT'] 22 | Pragma: [no-cache] 23 | Server: [Cowboy] 24 | Vary: ['Accept-Language, Cookie'] 25 | Via: [1.1 vegur] 26 | X-Frame-Options: [SAMEORIGIN] 27 | status: {code: 401, message: Unauthorized} 28 | version: 1 29 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_oauth2_token_exchange__invalid_code.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: redirect_uri=http%3A%2F%2F127.0.0.1%3A5000%2Fauthorize_openhumans%2F&grant_type=authorization_code&code=CODE 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Authorization: [XXXXXXXX] 8 | Connection: [keep-alive] 9 | Content-Length: ['134'] 10 | Content-Type: [application/x-www-form-urlencoded] 11 | User-Agent: [python-requests/2.18.4] 12 | method: POST 13 | uri: https://www.openhumans.org/oauth2/token/ 14 | response: 15 | body: {string: '{"error": "invalid_grant"}'} 16 | headers: 17 | Cache-Control: [no-store] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sat, 10 Mar 2018 04:00:12 GMT'] 22 | Pragma: [no-cache] 23 | Server: [Cowboy] 24 | Vary: ['Accept-Language, Cookie'] 25 | Via: [1.1 vegur] 26 | X-Frame-Options: [SAMEORIGIN] 27 | status: {code: 401, message: Unauthorized} 28 | version: 1 29 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## General Checkups 4 | - [ ] Have you checked that there aren't other open pull requests for the same issue/update/change? 5 | - [ ] If your code includes new features and not just bug fixes/copy editing: Did you include new tests? 6 | 7 | 10 | 11 | ## Description 12 | 13 | 14 | ## Related Issue 15 | 18 | 19 | ## Example 20 | 22 | 23 | 25 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_oauth2_token_exchange__invalid_client.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: grant_type=authorization_code&code=CODE&redirect_uri=http%3A%2F%2F127.0.0.1%3A5000%2Fauthorize_openhumans%2F 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Authorization: [XXXXXXXX] 8 | Connection: [keep-alive] 9 | Content-Length: ['115'] 10 | Content-Type: [application/x-www-form-urlencoded] 11 | User-Agent: [python-requests/2.18.4] 12 | method: POST 13 | uri: https://www.openhumans.org/oauth2/token/ 14 | response: 15 | body: {string: '{"error": "invalid_client"}'} 16 | headers: 17 | Cache-Control: [no-store] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sat, 10 Mar 2018 04:01:07 GMT'] 22 | Pragma: [no-cache] 23 | Server: [Cowboy] 24 | Vary: ['Accept-Language, Cookie'] 25 | Via: [1.1 vegur] 26 | X-Frame-Options: [SAMEORIGIN] 27 | status: {code: 401, message: Unauthorized} 28 | version: 1 29 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_oauth2_token_exchange__invalid_secret.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: grant_type=authorization_code&redirect_uri=http%3A%2F%2F127.0.0.1%3A5000%2Fauthorize_openhumans%2F&code=CODE 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Authorization: [XXXXXXXX] 8 | Connection: [keep-alive] 9 | Content-Length: ['134'] 10 | Content-Type: [application/x-www-form-urlencoded] 11 | User-Agent: [python-requests/2.18.4] 12 | method: POST 13 | uri: https://www.openhumans.org/oauth2/token/ 14 | response: 15 | body: {string: '{"error": "invalid_client"}'} 16 | headers: 17 | Cache-Control: [no-store] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sat, 10 Mar 2018 04:01:41 GMT'] 22 | Pragma: [no-cache] 23 | Server: [Cowboy] 24 | Vary: ['Accept-Language, Cookie'] 25 | Via: [1.1 vegur] 26 | X-Frame-Options: [SAMEORIGIN] 27 | status: {code: 401, message: Unauthorized} 28 | version: 1 29 | -------------------------------------------------------------------------------- /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## General Checkups 4 | - [ ] Have you checked that there isn't already an existing issue that describes what you report below? 5 | - [ ] Have you checked that there isn't already an open pull requests for this issue/update/change? 6 | 7 | 10 | 11 | 12 | ## Description 13 | 14 | 15 | ## Related Issue(s) 16 | 19 | 20 | ## Example 21 | 22 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_update_data_expired_master_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.9.1] 9 | method: GET 10 | uri: https://www.openhumans.org/api/direct-sharing/project/members/?access_token=ACCESSTOKEN 11 | response: 12 | body: {string: '{"detail":"Expired token."}'} 13 | headers: 14 | Allow: ['GET, HEAD, OPTIONS'] 15 | Cache-Control: ['must-revalidate, no-store, no-cache, max-age=0'] 16 | Connection: [close] 17 | Content-Language: [en] 18 | Content-Type: [application/json] 19 | Date: ['Mon, 19 Mar 2018 05:38:59 GMT'] 20 | Expires: ['Mon, 19 Mar 2018 05:38:59 GMT'] 21 | Last-Modified: ['Mon, 19 Mar 2018 05:38:59 GMT'] 22 | Server: [Cowboy] 23 | Vary: ['Accept, Accept-Language, Cookie'] 24 | Via: [1.1 vegur] 25 | Www-Authenticate: [Bearer realm="api"] 26 | X-Frame-Options: [SAMEORIGIN] 27 | status: {code: 401, message: Unauthorized} 28 | version: 1 29 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_update_data_invalid_master_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.9.1] 9 | method: GET 10 | uri: https://www.openhumans.org/api/direct-sharing/project/members/?access_token=ACCESSTOKEN 11 | response: 12 | body: {string: '{"detail":"Invalid token."}'} 13 | headers: 14 | Allow: ['GET, HEAD, OPTIONS'] 15 | Cache-Control: ['must-revalidate, no-store, no-cache, max-age=0'] 16 | Connection: [close] 17 | Content-Language: [en] 18 | Content-Type: [application/json] 19 | Date: ['Mon, 19 Mar 2018 05:38:59 GMT'] 20 | Expires: ['Mon, 19 Mar 2018 05:39:00 GMT'] 21 | Last-Modified: ['Mon, 19 Mar 2018 05:39:00 GMT'] 22 | Server: [Cowboy] 23 | Vary: ['Accept, Accept-Language, Cookie'] 24 | Via: [1.1 vegur] 25 | Www-Authenticate: [Bearer realm="api"] 26 | X-Frame-Options: [SAMEORIGIN] 27 | status: {code: 401, message: Unauthorized} 28 | version: 1 29 | -------------------------------------------------------------------------------- /ohapi/cassettes/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | VCR records of Open Humans API calls. 3 | 4 | Example usage: 5 | 6 | >>> from ohapi.cassettes import get_vcr 7 | >>> from ohapi.api import oauth2_token_exchange 8 | >>> with ohapi_vcr.use_cassette('test_oauth2_token_exchange__valid_code'): 9 | oauth2_token_exchange( 10 | client_id='clientid', client_secret='clientsecret', 11 | redirect_uri='http://127.0.0.1:5000/authorize/', 12 | code='codegoeshere') 13 | 14 | {'token_type': 'Bearer', 'access_token': 'returnedaccesstoken', 15 | 'refresh_token': 'returnedrefreshtoken', 'expires_in': 36000, 16 | 'scope': 'american-gut read wildlife open-humans write pgp go-viral'} 17 | """ 18 | import os 19 | 20 | import vcr 21 | 22 | 23 | def valid_cassettes(): 24 | base_dir = os.path.dirname(__file__) 25 | return [x for x in os.listdir(base_dir) if x.endswith('.yaml')] 26 | 27 | 28 | def get_vcr(): 29 | base_dir = os.path.dirname(__file__) 30 | my_vcr = vcr.VCR(record_mode='none', 31 | path_transformer=vcr.VCR.ensure_suffix('.yaml'), 32 | cassette_library_dir=base_dir) 33 | return my_vcr 34 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_get_page_invalid_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.9.1] 9 | method: GET 10 | uri: https://www.openhumans.org/api/direct-sharing/project/exchange-member/?access_token=invalid_token 11 | response: 12 | body: {string: '{"detail":"Authentication credentials were not provided."}'} 13 | headers: 14 | Allow: ['GET, HEAD, OPTIONS'] 15 | Cache-Control: ['no-cache, no-store, must-revalidate, max-age=0'] 16 | Connection: [close] 17 | Content-Language: [en] 18 | Content-Type: [application/json] 19 | Date: ['Tue, 06 Mar 2018 09:27:43 GMT'] 20 | Expires: ['Tue, 06 Mar 2018 09:27:43 GMT'] 21 | Last-Modified: ['Tue, 06 Mar 2018 09:27:43 GMT'] 22 | Server: [Cowboy] 23 | Vary: ['Accept, Accept-Language, Cookie'] 24 | Via: [1.1 vegur] 25 | Www-Authenticate: [Bearer realm="api"] 26 | X-Frame-Options: [SAMEORIGIN] 27 | status: {code: 401, message: Unauthorized} 28 | version: 1 29 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_get_page_with_results.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.9.1] 9 | method: GET 10 | uri: https://www.openhumans.org/api/direct-sharing/project/exchange-member/?access_token=ACCESSTOKEN 11 | response: 12 | body: {string: '{"created":"created_date_time","project_member_id":"PMI","message_permission":true,"sources_shared":[],"username":"test_user","data":[]}'} 13 | headers: 14 | Allow: ['GET, HEAD, OPTIONS'] 15 | Cache-Control: ['max-age=0, no-cache, must-revalidate, no-store'] 16 | Connection: [close] 17 | Content-Language: [en] 18 | Content-Type: [application/json] 19 | Date: ['Tue, 06 Mar 2018 09:27:43 GMT'] 20 | Expires: ['Tue, 06 Mar 2018 09:27:44 GMT'] 21 | Last-Modified: ['Tue, 06 Mar 2018 09:27:44 GMT'] 22 | Server: [Cowboy] 23 | Vary: ['Accept, Accept-Language, Cookie'] 24 | Via: [1.1 vegur] 25 | X-Frame-Options: [SAMEORIGIN] 26 | status: {code: 200, message: OK} 27 | version: 1 28 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_message_valid_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: subject=test_subject&message=test_message 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['41'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/message/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '"success"'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-cache, max-age=0, no-store, must-revalidate'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sun, 11 Mar 2018 15:33:42 GMT'] 22 | Expires: ['Sun, 11 Mar 2018 15:33:42 GMT'] 23 | Last-Modified: ['Sun, 11 Mar 2018 15:33:42 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 200, message: OK} 29 | version: 1 30 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_delete_file__valid_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=59319749&all_files=True 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['41'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.18.4] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/delete/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"ids":[]}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['must-revalidate, max-age=0, no-cache, no-store'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Wed, 14 Mar 2018 15:36:19 GMT'] 22 | Expires: ['Wed, 14 Mar 2018 15:36:19 GMT'] 23 | Last-Modified: ['Wed, 14 Mar 2018 15:36:19 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 200, message: OK} 29 | version: 1 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Open Humans Foundation and contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_delete_file_project_member_id_given.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=59319749&all_files=True 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['41'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.18.4] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/delete/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"ids":[34188]}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-store, max-age=0, must-revalidate, no-cache'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Tue, 13 Mar 2018 12:32:21 GMT'] 22 | Expires: ['Tue, 13 Mar 2018 12:32:22 GMT'] 23 | Last-Modified: ['Tue, 13 Mar 2018 12:32:22 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 200, message: OK} 29 | version: 1 30 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_message_all_members_true_project_member_id_none.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: all_members=True&message=test_message&subject=test_subject 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['58'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/message/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '"success"'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['must-revalidate, max-age=0, no-cache, no-store'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sun, 11 Mar 2018 15:47:10 GMT'] 22 | Expires: ['Sun, 11 Mar 2018 15:47:11 GMT'] 23 | Last-Modified: ['Sun, 11 Mar 2018 15:47:11 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 200, message: OK} 29 | version: 1 30 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_oauth2_token_exchange__valid_refresh.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: refresh_token=REFRESHTOKEN&grant_type=refresh_token 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Authorization: [XXXXXXXX] 8 | Connection: [keep-alive] 9 | Content-Length: ['69'] 10 | Content-Type: [application/x-www-form-urlencoded] 11 | User-Agent: [python-requests/2.18.4] 12 | method: POST 13 | uri: https://www.openhumans.org/oauth2/token/ 14 | response: 15 | body: {string: '{"expires_in": 36000, "refresh_token": "newrefreshtoken", 16 | "scope": "american-gut read wildlife open-humans write pgp go-viral", "token_type": 17 | "Bearer", "access_token": "newaccesstoken"}'} 18 | headers: 19 | Cache-Control: [no-store] 20 | Connection: [close] 21 | Content-Language: [en] 22 | Content-Type: [application/json] 23 | Date: ['Sat, 10 Mar 2018 04:02:19 GMT'] 24 | Pragma: [no-cache] 25 | Server: [Cowboy] 26 | Vary: ['Accept-Language, Cookie'] 27 | Via: [1.1 vegur] 28 | X-Frame-Options: [SAMEORIGIN] 29 | status: {code: 200, message: OK} 30 | version: 1 31 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_delete_file_project_member_id_invalid.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=1234&all_files=True 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['37'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.18.4] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/delete/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"project_member_id":"project_member_id is invalid"}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-store, max-age=0, must-revalidate, no-cache'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Tue, 13 Mar 2018 17:18:00 GMT'] 22 | Expires: ['Tue, 13 Mar 2018 17:18:00 GMT'] 23 | Last-Modified: ['Tue, 13 Mar 2018 17:18:00 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 400, message: Bad Request} 29 | version: 1 30 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_message_expired_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: message=test_message&subject=test_subject 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['41'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/message/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"detail":"Expired token."}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['must-revalidate, max-age=0, no-cache, no-store'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sun, 11 Mar 2018 15:41:40 GMT'] 22 | Expires: ['Sun, 11 Mar 2018 15:41:41 GMT'] 23 | Last-Modified: ['Sun, 11 Mar 2018 15:41:41 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | Www-Authenticate: [Bearer realm="api"] 28 | X-Frame-Options: [SAMEORIGIN] 29 | status: {code: 401, message: Unauthorized} 30 | version: 1 31 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_message_invalid_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: message=test_message&subject=test_subject 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['41'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/message/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"detail": "Invalid token."}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-cache, max-age=0, no-store, must-revalidate'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sun, 11 Mar 2018 15:45:02 GMT'] 22 | Expires: ['Sun, 11 Mar 2018 15:45:02 GMT'] 23 | Last-Modified: ['Sun, 11 Mar 2018 15:45:02 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | Www-Authenticate: [Bearer realm="api"] 28 | X-Frame-Options: [SAMEORIGIN] 29 | status: {code: 401, message: Unauthorized} 30 | version: 1 31 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_delete_file__expired_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: all_files=True&project_member_id=59319749 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['41'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.18.4] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/delete/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"detail":"Expired token."}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-cache, max-age=0, must-revalidate, no-store'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Wed, 14 Mar 2018 13:10:03 GMT'] 22 | Expires: ['Wed, 14 Mar 2018 13:10:04 GMT'] 23 | Last-Modified: ['Wed, 14 Mar 2018 13:10:04 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | Www-Authenticate: [Bearer realm="api"] 28 | X-Frame-Options: [SAMEORIGIN] 29 | status: {code: 401, message: Unauthorized} 30 | version: 1 31 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_delete_file__invalid_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: all_files=True&project_member_id=59319749 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['41'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.18.4] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/delete/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"detail":"Invalid token."}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-store, max-age=0, must-revalidate, no-cache'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Tue, 13 Mar 2018 12:33:06 GMT'] 22 | Expires: ['Tue, 13 Mar 2018 12:33:06 GMT'] 23 | Last-Modified: ['Tue, 13 Mar 2018 12:33:06 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | Www-Authenticate: [Bearer realm="api"] 28 | X-Frame-Options: [SAMEORIGIN] 29 | status: {code: 401, message: Unauthorized} 30 | version: 1 31 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_message_all_members_false_project_member_id_not_none_valid.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_ids=VALIDPMI1&project_member_ids=VALIDPMI2&subject=testsubject&all_members=False&message=testmessage 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['115'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/message/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '"success"'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['must-revalidate, max-age=0, no-cache, no-store'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sun, 11 Mar 2018 16:30:13 GMT'] 22 | Expires: ['Sun, 11 Mar 2018 16:30:13 GMT'] 23 | Last-Modified: ['Sun, 11 Mar 2018 16:30:13 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 200, message: OK} 29 | version: 1 30 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_oauth2_token_exchange__valid_code.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: redirect_uri=http%3A%2F%2F127.0.0.1%3A5000%2Fauthorize_openhumans%2F&code=CODE&grant_type=authorization_code 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['134'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.18.4] 11 | Authorization: [XXXXXXXX] 12 | method: POST 13 | uri: https://www.openhumans.org/oauth2/token/ 14 | response: 15 | body: {string: '{"expires_in": 36000, "refresh_token": "returnedrefreshtoken", 16 | "scope": "american-gut read wildlife open-humans write pgp go-viral", 17 | "token_type": "Bearer", "access_token": "returnedaccesstoken"}'} 18 | headers: 19 | Cache-Control: [no-store] 20 | Connection: [close] 21 | Content-Language: [en] 22 | Content-Type: [application/json] 23 | Date: ['Sat, 10 Mar 2018 03:57:17 GMT'] 24 | Pragma: [no-cache] 25 | Server: [Cowboy] 26 | Vary: ['Accept-Language, Cookie'] 27 | Via: [1.1 vegur] 28 | X-Frame-Options: [SAMEORIGIN] 29 | status: {code: 200, message: OK} 30 | version: 1 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # Project-specific config for different PyPI username for twine. 28 | # e.g. `twine upload --config-file ./.pypirc dist/*` 29 | .pypirc 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *,cover 50 | .hypothesis/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | 59 | # Sphinx documentation 60 | docs/docs_html/* 61 | 62 | # PyBuilder 63 | target/ 64 | 65 | #Ipython Notebook 66 | .ipynb_checkpoints 67 | 68 | #Secret data used in development 69 | ohapi/tests/_config_params_api.py 70 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_file_invalid_metadata_without_description.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=PROJECTMEMBERID&metadata=%7B%7D&filename=lorem_ipsum.txt 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['67'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.18.4] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"metadata":["\"description\" is a required field of the metadata"]}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['max-age=0, must-revalidate, no-cache, no-store'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Mon, 02 Jul 2018 23:02:15 GMT'] 22 | Expires: ['Mon, 02 Jul 2018 23:02:15 GMT'] 23 | Last-Modified: ['Mon, 02 Jul 2018 23:02:15 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 400, message: Bad Request} 29 | version: 1 30 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_aws_invalid_metadata_with_description.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=VALID_PMI1&metadata=%7B%22description%22%3A+%22Test+data%22%7D&filename=foo 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['97'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"metadata":["\"tags\" is a required field of the metadata"]}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-store, max-age=0, must-revalidate, no-cache'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sun, 27 May 2018 12:37:27 GMT'] 22 | Expires: ['Sun, 27 May 2018 12:37:27 GMT'] 23 | Last-Modified: ['Sun, 27 May 2018 12:37:27 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 400, message: Bad Request} 29 | version: 1 30 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_aws_invalid_metadata_without_description.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=VALID_PMI1&filename=foo&metadata=%7B%22tags%22%3A+%5B%22text%22%5D%7D 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['91'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"metadata":["\"description\" is a required field of the metadata"]}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-store, max-age=0, must-revalidate, no-cache'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sun, 27 May 2018 12:44:59 GMT'] 22 | Expires: ['Sun, 27 May 2018 12:45:00 GMT'] 23 | Last-Modified: ['Sun, 27 May 2018 12:45:00 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 400, message: Bad Request} 29 | version: 1 30 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. open-humans-api documentation master file, created by 2 | sphinx-quickstart on Tue Mar 20 13:49:57 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to open-humans-api's documentation! 7 | ******************************************* 8 | 9 | ``open-humans-api`` is a Python package that wraps the API methods of 10 | `Open Humans `_ for easier use in your own 11 | Python applications and websites. 12 | 13 | It also installs a 14 | set of command line utilities that can be used to 15 | 16 | * download public files 17 | 18 | * for a given Project 19 | * for a given Member 20 | 21 | * Upload files for a project through a ``master_access_token`` or OAuth2 ``access_token`` 22 | * Download files for a project through a ``master_access_token`` or OAuth2 ``access_token`` 23 | 24 | Installation 25 | ============ 26 | 27 | You can install ``open-humans-api`` through ``pip``: 28 | 29 | .. code-block:: Shell 30 | 31 | pip install open-humans-api 32 | 33 | .. toctree:: 34 | :maxdepth: 2 35 | :caption: Contents: 36 | 37 | cli 38 | ohapi 39 | tests 40 | 41 | 42 | Indices and tables 43 | ================== 44 | 45 | * :ref:`genindex` 46 | * :ref:`modindex` 47 | * :ref:`search` 48 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_file_invalid_metadata_with_description.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=PROJECTMEMBERID&metadata=%7B%22description%22%3A+%22Lorem+ipsum+text%22%7D&filename=lorem_ipsum.txt 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['110'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.18.4] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"metadata":["\"tags\" is a required field of the metadata"]}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['max-age=0, must-revalidate, no-cache, no-store'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Mon, 02 Jul 2018 23:01:47 GMT'] 22 | Expires: ['Mon, 02 Jul 2018 23:01:47 GMT'] 23 | Last-Modified: ['Mon, 02 Jul 2018 23:01:47 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 400, message: Bad Request} 29 | version: 1 30 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_aws_expired_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=validPMI1&metadata=%7B%22tags%22%3A+%5B%22text%22%5D%2C+%22description%22%3A+%22Test+data%22%7D&filename=foo 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['131'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"detail": "Expired token."}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['must-revalidate, max-age=0, no-cache, no-store'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sun, 27 May 2018 12:34:27 GMT'] 22 | Expires: ['Sun, 27 May 2018 12:34:27 GMT'] 23 | Last-Modified: ['Sun, 27 May 2018 12:34:27 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | Www-Authenticate: [Bearer realm="api"] 28 | X-Frame-Options: [SAMEORIGIN] 29 | status: {code: 401, message: Unauthorized} 30 | version: 1 31 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_aws_invalid_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: metadata=%7B%22description%22%3A+%22Test+data%22%2C+%22tags%22%3A+%5B%22text%22%5D%7D&project_member_id=validPMI1&filename=foo 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['131'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"detail": "Invalid token."}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-store, max-age=0, must-revalidate, no-cache'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sun, 27 May 2018 12:31:10 GMT'] 22 | Expires: ['Sun, 27 May 2018 12:31:11 GMT'] 23 | Last-Modified: ['Sun, 27 May 2018 12:31:11 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | Www-Authenticate: [Bearer realm="api"] 28 | X-Frame-Options: [SAMEORIGIN] 29 | status: {code: 401, message: Unauthorized} 30 | version: 1 31 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_message_all_members_false_projectmemberid_has_invalid_char.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_ids=abcdef1&project_member_ids=test&message=test_message&subject=test_subject&all_members=False 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['110'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/message/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"errors": {"project_member_ids":["Project member IDs are always 15 | 8 digits long."]}}'} 16 | headers: 17 | Allow: ['POST, OPTIONS'] 18 | Cache-Control: ['must-revalidate, max-age=0, no-cache, no-store'] 19 | Connection: [close] 20 | Content-Language: [en] 21 | Content-Type: [application/json] 22 | Date: ['Sun, 11 Mar 2018 15:55:45 GMT'] 23 | Expires: ['Sun, 11 Mar 2018 15:55:46 GMT'] 24 | Last-Modified: ['Sun, 11 Mar 2018 15:55:46 GMT'] 25 | Server: [Cowboy] 26 | Vary: ['Accept, Accept-Language, Cookie'] 27 | Via: [1.1 vegur] 28 | X-Frame-Options: [SAMEORIGIN] 29 | status: {code: 400, message: Bad Request} 30 | version: 1 31 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_message_all_members_false_projectmemberid_has_invalid_digit.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_ids=invalidPMI1&project_member_ids=invalidPMI2&subject=test_subject&all_members=False&message=test_message 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['115'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/message/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"errors": {"project_member_ids":["Invalid project member ID(s): 15 | invalidPMI2"]}}'} 16 | headers: 17 | Allow: ['POST, OPTIONS'] 18 | Cache-Control: ['must-revalidate, max-age=0, no-cache, no-store'] 19 | Connection: [close] 20 | Content-Language: [en] 21 | Content-Type: [application/json] 22 | Date: ['Sun, 11 Mar 2018 16:17:55 GMT'] 23 | Expires: ['Sun, 11 Mar 2018 16:17:56 GMT'] 24 | Last-Modified: ['Sun, 11 Mar 2018 16:17:56 GMT'] 25 | Server: [Cowboy] 26 | Vary: ['Accept, Accept-Language, Cookie'] 27 | Via: [1.1 vegur] 28 | X-Frame-Options: [SAMEORIGIN] 29 | status: {code: 400, message: Bad Request} 30 | version: 1 31 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_file_expired_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=PROJECTMEMBERID&metadata=%7B%22tags%22%3A+%5B%22text%22%5D%2C+%22description%22%3A+%22Lorem+ipsum+text%22%7D&filename=lorem_ipsum.txt 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['144'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.18.4] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"detail":"Expired token."}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-store, no-cache, max-age=0, must-revalidate'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Mon, 02 Jul 2018 23:01:34 GMT'] 22 | Expires: ['Mon, 02 Jul 2018 23:01:34 GMT'] 23 | Last-Modified: ['Mon, 02 Jul 2018 23:01:34 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | Www-Authenticate: [Bearer realm="api"] 28 | X-Frame-Options: [SAMEORIGIN] 29 | status: {code: 401, message: Unauthorized} 30 | version: 1 31 | -------------------------------------------------------------------------------- /docs/tests.rst: -------------------------------------------------------------------------------- 1 | Development & Testing 2 | ********************* 3 | 4 | ``open-humans-api`` is under active development and we have implemented a couple 5 | of tests to check that changes don't break anything. 6 | 7 | If you want to contribute 8 | `you can do so on GitHub `_ and 9 | `find us on Slack `_. 10 | 11 | Installing ``ohapi`` for development 12 | ------------------------------------ 13 | 14 | To install a development version of this package run the following steps on your 15 | shell: 16 | 17 | .. code-block:: Shell 18 | 19 | git clone git@github.com:OpenHumans/open-humans-api.git 20 | cd open-humans-api 21 | pip install -e. 22 | 23 | 24 | ohapi.tests.test_test 25 | ----------------------------- 26 | 27 | .. automodule:: ohapi.tests.test_test 28 | :exclude-members: parameter_defaults, setUp 29 | :members: 30 | :show-inheritance: 31 | 32 | ohapi.tests.test_api 33 | ----------------------------- 34 | 35 | .. automodule:: ohapi.tests.test_api 36 | :exclude-members: parameter_defaults, setUp 37 | :members: 38 | :show-inheritance: 39 | 40 | 41 | ohapi.tests.test_projects 42 | -------------------------------- 43 | 44 | .. automodule:: ohapi.tests.test_projects 45 | :exclude-members: parameter_defaults, setUp 46 | :members: 47 | :show-inheritance: 48 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_file_remote_info_not_none_valid.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.18.4] 9 | method: GET 10 | uri: https://valid_url/ 11 | response: 12 | body: {string: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do 13 | eiusmod tempor 14 | 15 | incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis 16 | 17 | nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. 18 | 19 | Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore 20 | 21 | eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, 22 | sunt 23 | 24 | in culpa qui officia deserunt mollit anim id est laborum. 25 | 26 | '} 27 | headers: 28 | Accept-Ranges: [bytes] 29 | Content-Length: ['446'] 30 | Content-Type: [binary/octet-stream] 31 | Date: ['Mon, 02 Jul 2018 23:19:40 GMT'] 32 | ETag: ['"cefd9dacdc23ee1faabb1e954d281274"'] 33 | Last-Modified: ['Mon, 02 Jul 2018 23:00:20 GMT'] 34 | Server: [AmazonS3] 35 | x-amz-id-2: [KefPVQo2BBgvgmpCl+u2HFevWKB+SauA8TOSzi/x0yyDSKPJhjSAvaEBbcQO/lMVe2u2AguhaZ4=] 36 | x-amz-request-id: [AEAF554B1245B868] 37 | x-amz-version-id: [rG7EIMjWf.yG6vpWewzaMqzTKfU66ds4] 38 | status: {code: 200, message: OK} 39 | version: 1 40 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_file_remote_info_not_none_matching_file_size.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.18.4] 9 | method: GET 10 | uri: https://valid_url/ 11 | response: 12 | body: {string: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do 13 | eiusmod tempor 14 | 15 | incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis 16 | 17 | nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. 18 | 19 | Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore 20 | 21 | eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, 22 | sunt 23 | 24 | in culpa qui officia deserunt mollit anim id est laborum. 25 | 26 | '} 27 | headers: 28 | Accept-Ranges: [bytes] 29 | Content-Length: ['446'] 30 | Content-Type: [binary/octet-stream] 31 | Date: ['Mon, 02 Jul 2018 23:43:46 GMT'] 32 | ETag: ['"cefd9dacdc23ee1faabb1e954d281274"'] 33 | Last-Modified: ['Mon, 02 Jul 2018 23:00:20 GMT'] 34 | Server: [AmazonS3] 35 | x-amz-id-2: [BtYrHhO/iaplmK5RtnIRh5PqCH/aUGGfML14swW+SVUu4UkNjHvA2PPlujsb5kYvi5L6Ye+AsTw=] 36 | x-amz-request-id: [8B809DB719868935] 37 | x-amz-version-id: [rG7EIMjWf.yG6vpWewzaMqzTKfU66ds4] 38 | status: {code: 200, message: OK} 39 | version: 1 40 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | import sys 5 | 6 | 7 | def readme(): 8 | with open('README.md') as f: 9 | return f.read() 10 | 11 | 12 | # Add backport of futures unless Python version is 3.2 or later. 13 | install_requires = [ 14 | 'click>=6.3', 15 | 'humanfriendly>=1.44.3', 16 | 'requests>=2.9.1', 17 | 'arrow>=0.8.0', 18 | ] 19 | if sys.version_info < (3, 2): 20 | install_requires.append('futures>=3.0.5') 21 | 22 | setup( 23 | name='open-humans-api', 24 | author='Mad Price Ball', 25 | author_email='support@openhumans.org', 26 | 27 | url='https://github.com/OpenHumans/open-humans-api', 28 | 29 | description='Tools for working with Open Humans APIs', 30 | long_description=readme(), 31 | long_description_content_type="text/markdown", 32 | 33 | version='0.2.8', 34 | 35 | license='MIT', 36 | 37 | keywords=['open-humans'], 38 | 39 | classifiers=[ 40 | 'Environment :: Console', 41 | 'Development Status :: 3 - Alpha', 42 | 'Intended Audience :: Developers', 43 | 'Intended Audience :: End Users/Desktop', 44 | 'Intended Audience :: Information Technology', 45 | 'Intended Audience :: System Administrators', 46 | 'License :: OSI Approved :: BSD License', 47 | 'Operating System :: POSIX', 48 | 'Operating System :: MacOS :: MacOS X', 49 | 'Operating System :: Microsoft :: Windows', 50 | 'Programming Language :: Python', 51 | 'Topic :: Utilities', 52 | ], 53 | 54 | packages=['ohapi'], 55 | 56 | entry_points={ 57 | 'console_scripts': [ 58 | 'ohpub-download = ohapi.command_line:public_data_download_cli', 59 | 'ohproj-download = ohapi.command_line:download_cli', 60 | 'ohproj-download-metadata = ohapi.command_line:download_metadata_cli', 61 | 'ohproj-upload = ohapi.command_line:upload_cli', 62 | 'ohproj-upload-metadata = ohapi.command_line:upload_metadata_cli', 63 | 'ohproj-oauth2-token-exchange = ohapi.command_line:oauth_token_exchange_cli', 64 | 'ohproj-oauth2-url = ohapi.command_line:oauth2_auth_url_cli', 65 | 'ohproj-message = ohapi.command_line:message_cli', 66 | 'ohproj-delete = ohapi.command_line:delete_cli', 67 | ] 68 | }, 69 | 70 | install_requires=install_requires, 71 | ) 72 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_file_remote_info_not_none_invalid_metadata.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.18.4] 9 | method: GET 10 | uri: https://valid_url/ 11 | response: 12 | body: {string: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do 13 | eiusmod tempor 14 | 15 | incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis 16 | 17 | nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. 18 | 19 | Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore 20 | 21 | eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, 22 | sunt 23 | 24 | in culpa qui officia deserunt mollit anim id est laborum. 25 | 26 | '} 27 | headers: 28 | Accept-Ranges: [bytes] 29 | Content-Length: ['446'] 30 | Content-Type: [binary/octet-stream] 31 | Date: ['Mon, 02 Jul 2018 23:55:12 GMT'] 32 | ETag: ['"cefd9dacdc23ee1faabb1e954d281274"'] 33 | Last-Modified: ['Mon, 02 Jul 2018 23:00:20 GMT'] 34 | Server: [AmazonS3] 35 | x-amz-id-2: [mdNtjUd/uk2nHMu93GVpNrCNsDQ+YDOZtjP9bdbE9ydZ1NDq3KJB6QpXlffokJroare405SChUk=] 36 | x-amz-request-id: [5D6B83B84F3048B7] 37 | x-amz-version-id: [rG7EIMjWf.yG6vpWewzaMqzTKfU66ds4] 38 | status: {code: 200, message: OK} 39 | - request: 40 | body: project_member_id=PROJECTMEMBERID&metadata=%7B%7D&filename=lorem_ipsum_partial.txt 41 | headers: 42 | Accept: ['*/*'] 43 | Accept-Encoding: ['gzip, deflate'] 44 | Connection: [keep-alive] 45 | Content-Length: ['75'] 46 | Content-Type: [application/x-www-form-urlencoded] 47 | User-Agent: [python-requests/2.18.4] 48 | method: POST 49 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 50 | response: 51 | body: {string: '{"metadata":["\"description\" is a required field of the metadata"]}'} 52 | headers: 53 | Allow: ['POST, OPTIONS'] 54 | Cache-Control: ['no-store, no-cache, max-age=0, must-revalidate'] 55 | Connection: [close] 56 | Content-Language: [en] 57 | Content-Type: [application/json] 58 | Date: ['Mon, 02 Jul 2018 23:55:12 GMT'] 59 | Expires: ['Mon, 02 Jul 2018 23:55:12 GMT'] 60 | Last-Modified: ['Mon, 02 Jul 2018 23:55:12 GMT'] 61 | Server: [Cowboy] 62 | Vary: ['Accept, Accept-Language, Cookie'] 63 | Via: [1.1 vegur] 64 | X-Frame-Options: [SAMEORIGIN] 65 | status: {code: 400, message: Bad Request} 66 | version: 1 67 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_file_remote_info_not_none_invalid_metadata_with_desc.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.18.4] 9 | method: GET 10 | uri: https://valid_url/ 11 | response: 12 | body: {string: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do 13 | eiusmod tempor 14 | 15 | incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis 16 | 17 | nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. 18 | 19 | Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore 20 | 21 | eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, 22 | sunt 23 | 24 | in culpa qui officia deserunt mollit anim id est laborum. 25 | 26 | '} 27 | headers: 28 | Accept-Ranges: [bytes] 29 | Content-Length: ['446'] 30 | Content-Type: [binary/octet-stream] 31 | Date: ['Mon, 02 Jul 2018 23:53:00 GMT'] 32 | ETag: ['"cefd9dacdc23ee1faabb1e954d281274"'] 33 | Last-Modified: ['Mon, 02 Jul 2018 23:00:20 GMT'] 34 | Server: [AmazonS3] 35 | x-amz-id-2: [AHBefsmm+gVj1uQPWRN3Vac8IhmhyA19N6j7g+D853fr0AbdUck7fCbg2i8qplMtznCZGK7G8kg=] 36 | x-amz-request-id: [625DE94143757177] 37 | x-amz-version-id: [rG7EIMjWf.yG6vpWewzaMqzTKfU66ds4] 38 | status: {code: 200, message: OK} 39 | - request: 40 | body: project_member_id=PROJECTMEMBERID&metadata=%7B%22description%22%3A+%22Lorem+ipsum+text%22%7D&filename=lorem_ipsum_partial.txt 41 | headers: 42 | Accept: ['*/*'] 43 | Accept-Encoding: ['gzip, deflate'] 44 | Connection: [keep-alive] 45 | Content-Length: ['118'] 46 | Content-Type: [application/x-www-form-urlencoded] 47 | User-Agent: [python-requests/2.18.4] 48 | method: POST 49 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 50 | response: 51 | body: {string: '{"metadata":["\"tags\" is a required field of the metadata"]}'} 52 | headers: 53 | Allow: ['POST, OPTIONS'] 54 | Cache-Control: ['max-age=0, must-revalidate, no-cache, no-store'] 55 | Connection: [close] 56 | Content-Language: [en] 57 | Content-Type: [application/json] 58 | Date: ['Mon, 02 Jul 2018 23:53:00 GMT'] 59 | Expires: ['Mon, 02 Jul 2018 23:53:00 GMT'] 60 | Last-Modified: ['Mon, 02 Jul 2018 23:53:00 GMT'] 61 | Server: [Cowboy] 62 | Vary: ['Accept, Accept-Language, Cookie'] 63 | Via: [1.1 vegur] 64 | X-Frame-Options: [SAMEORIGIN] 65 | status: {code: 400, message: Bad Request} 66 | version: 1 67 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_file_remote_info_not_none_expired_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.18.4] 9 | method: GET 10 | uri: https://valid_url/ 11 | response: 12 | body: {string: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do 13 | eiusmod tempor 14 | 15 | incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis 16 | 17 | nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. 18 | 19 | Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore 20 | 21 | eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, 22 | sunt 23 | 24 | in culpa qui officia deserunt mollit anim id est laborum. 25 | 26 | '} 27 | headers: 28 | Accept-Ranges: [bytes] 29 | Content-Length: ['446'] 30 | Content-Type: [binary/octet-stream] 31 | Date: ['Mon, 02 Jul 2018 23:42:58 GMT'] 32 | ETag: ['"cefd9dacdc23ee1faabb1e954d281274"'] 33 | Last-Modified: ['Mon, 02 Jul 2018 23:00:20 GMT'] 34 | Server: [AmazonS3] 35 | x-amz-id-2: [y6T7LOjQgtM+VyZnuzEuu5cazD3gfNi9AEINGfeLXoTes+M1SOoRlPSZ+O4wDgQVCwO5jir/ids=] 36 | x-amz-request-id: [BD49E3D03E5744FE] 37 | x-amz-version-id: [rG7EIMjWf.yG6vpWewzaMqzTKfU66ds4] 38 | status: {code: 200, message: OK} 39 | - request: 40 | body: project_member_id=PROJECTMEMBERID&metadata=%7B%22tags%22%3A+%5B%22text%22%5D%2C+%22description%22%3A+%22Lorem+ipsum+text%22%7D&filename=lorem_ipsum_partial.txt 41 | headers: 42 | Accept: ['*/*'] 43 | Accept-Encoding: ['gzip, deflate'] 44 | Connection: [keep-alive] 45 | Content-Length: ['152'] 46 | Content-Type: [application/x-www-form-urlencoded] 47 | User-Agent: [python-requests/2.18.4] 48 | method: POST 49 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 50 | response: 51 | body: {string: '{"detail":"Expired token."}'} 52 | headers: 53 | Allow: ['POST, OPTIONS'] 54 | Cache-Control: ['max-age=0, must-revalidate, no-cache, no-store'] 55 | Connection: [close] 56 | Content-Language: [en] 57 | Content-Type: [application/json] 58 | Date: ['Mon, 02 Jul 2018 23:42:57 GMT'] 59 | Expires: ['Mon, 02 Jul 2018 23:42:58 GMT'] 60 | Last-Modified: ['Mon, 02 Jul 2018 23:42:58 GMT'] 61 | Server: [Cowboy] 62 | Vary: ['Accept, Accept-Language, Cookie'] 63 | Via: [1.1 vegur] 64 | Www-Authenticate: [Bearer realm="api"] 65 | X-Frame-Options: [SAMEORIGIN] 66 | status: {code: 401, message: Unauthorized} 67 | version: 1 68 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_file_remote_info_not_none_invalid_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.18.4] 9 | method: GET 10 | uri: https://valid_url/ 11 | response: 12 | body: {string: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do 13 | eiusmod tempor 14 | 15 | incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis 16 | 17 | nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. 18 | 19 | Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore 20 | 21 | eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, 22 | sunt 23 | 24 | in culpa qui officia deserunt mollit anim id est laborum. 25 | 26 | '} 27 | headers: 28 | Accept-Ranges: [bytes] 29 | Content-Length: ['446'] 30 | Content-Type: [binary/octet-stream] 31 | Date: ['Tue, 03 Jul 2018 05:24:06 GMT'] 32 | ETag: ['"cefd9dacdc23ee1faabb1e954d281274"'] 33 | Last-Modified: ['Tue, 03 Jul 2018 04:51:42 GMT'] 34 | Server: [AmazonS3] 35 | x-amz-id-2: [Qdc1VHEqCZzacL1VgxQ6XUkzbNiYiDk09hEBbUVkyqyEEMrqFnFkdungNlYODVZIMU6991MYlb0=] 36 | x-amz-request-id: [D11A134299A9113F] 37 | x-amz-version-id: [fBMyyoO8_Zp4oEKW2u0ZuSswJEfmgp0w] 38 | status: {code: 200, message: OK} 39 | - request: 40 | body: project_member_id=PROJECTMEMBERID&metadata=%7B%22tags%22%3A+%5B%22text%22%5D%2C+%22description%22%3A+%22Lorem+ipsum+text%22%7D&filename=lorem_ipsum_partial.txt 41 | headers: 42 | Accept: ['*/*'] 43 | Accept-Encoding: ['gzip, deflate'] 44 | Connection: [keep-alive] 45 | Content-Length: ['152'] 46 | Content-Type: [application/x-www-form-urlencoded] 47 | User-Agent: [python-requests/2.18.4] 48 | method: POST 49 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 50 | response: 51 | body: {string: '{"detail":"Invalid token."}'} 52 | headers: 53 | Allow: ['POST, OPTIONS'] 54 | Cache-Control: ['no-store, no-cache, max-age=0, must-revalidate'] 55 | Connection: [close] 56 | Content-Language: [en] 57 | Content-Type: [application/json] 58 | Date: ['Tue, 03 Jul 2018 05:24:04 GMT'] 59 | Expires: ['Tue, 03 Jul 2018 05:24:05 GMT'] 60 | Last-Modified: ['Tue, 03 Jul 2018 05:24:05 GMT'] 61 | Server: [Cowboy] 62 | Vary: ['Accept, Accept-Language, Cookie'] 63 | Via: [1.1 vegur] 64 | Www-Authenticate: [Bearer realm="api"] 65 | X-Frame-Options: [SAMEORIGIN] 66 | status: {code: 401, message: Unauthorized} 67 | version: 1 68 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_update_data_valid_master_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.9.1] 9 | method: GET 10 | uri: https://www.openhumans.org/api/direct-sharing/project/members/?access_token=ACCESSTOKEN 11 | response: 12 | body: {string: '{"count":3,"next":null,"previous":null,"results":[{"created":"date1","project_member_id":"PMI1","message_permission":true,"sources_shared":[],"username":"User1","data":[],"file_count":0},{"created":"date2","project_member_id":"PMI2","message_permission":true,"sources_shared":[],"username":"user2","data":[{}],"file_count":0},{"created":"date3","project_member_id":"PMI3","message_permission":true,"sources_shared":[],"username":"user3","data":[{}],"file_count":0}]}'} 13 | headers: 14 | Allow: ['GET, HEAD, OPTIONS'] 15 | Cache-Control: ['must-revalidate, no-store, no-cache, max-age=0'] 16 | Connection: [close] 17 | Content-Language: [en] 18 | Content-Type: [application/json] 19 | Date: ['Mon, 19 Mar 2018 05:53:05 GMT'] 20 | Expires: ['Mon, 19 Mar 2018 05:53:06 GMT'] 21 | Last-Modified: ['Mon, 19 Mar 2018 05:53:06 GMT'] 22 | Server: [Cowboy] 23 | Vary: ['Accept, Accept-Language, Cookie'] 24 | Via: [1.1 vegur] 25 | X-Frame-Options: [SAMEORIGIN] 26 | status: {code: 200, message: OK} 27 | - request: 28 | body: null 29 | headers: 30 | Accept: ['*/*'] 31 | Accept-Encoding: ['gzip, deflate'] 32 | Connection: [keep-alive] 33 | User-Agent: [python-requests/2.9.1] 34 | method: GET 35 | uri: https://www.openhumans.org/api/direct-sharing/project/members/?access_token=ACCESSTOKEN 36 | response: 37 | body: {string: '{"count":3,"next":null,"previous":null,"results":[{"created":"date1","project_member_id":"PMI1","message_permission":true,"sources_shared":[],"username":"User1","data":[],"file_count":0},{"created":"date2","project_member_id":"PMI2","message_permission":true,"sources_shared":[],"username":"user2","file_count":0,"data":[{}]},{"created":"date3","project_member_id":"PMI3","message_permission":true,"sources_shared":[],"username":"user3","file_count":0,"data":[{}]}]}'} 38 | headers: 39 | Allow: ['GET, HEAD, OPTIONS'] 40 | Cache-Control: ['must-revalidate, no-store, no-cache, max-age=0'] 41 | Connection: [close] 42 | Content-Language: [en] 43 | Content-Type: [application/json] 44 | Date: ['Mon, 19 Mar 2018 05:53:07 GMT'] 45 | Expires: ['Mon, 19 Mar 2018 05:53:07 GMT'] 46 | Last-Modified: ['Mon, 19 Mar 2018 05:53:07 GMT'] 47 | Server: [Cowboy] 48 | Vary: ['Accept, Accept-Language, Cookie'] 49 | Via: [1.1 vegur] 50 | X-Frame-Options: [SAMEORIGIN] 51 | status: {code: 200, message: OK} 52 | version: 1 53 | -------------------------------------------------------------------------------- /ohapi/tests/test_projects.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from ohapi.projects import OHProject 3 | import vcr 4 | 5 | parameter_defaults = { 6 | 'MEMBER_DATA': {"data": [{"basename": 1}]}, 7 | 'TARGET_MEMBER_DIR': 'targetmemberdir', 8 | 'MAX_SIZE': 'max_size', 9 | 'MASTER_ACCESS_TOKEN': 'masteraccesstoken', 10 | 'MASTER_ACCESS_TOKEN_EXPIRED': 'masteraccesstokenexpired', 11 | 'MASTER_ACCESS_TOKEN_INVALID': 'masteraccesstokeninvalid', 12 | } 13 | 14 | """ 15 | _config_params_api.py is not usually present. You can create this to use valid 16 | codes and tokens if you wish to record new cassettes. If present, this file is 17 | used to overwrite `parameter_defaults` with the (hopefully valid, but secret) 18 | items in the file. DO NOT COMMIT IT TO GIT! 19 | 20 | To get started, do: 21 | cp _config_params_api.py.example _config_params_api.py 22 | 23 | Edit _config_params_api.py to define valid secret codes, tokens, etc. 24 | 25 | Run a specific function to (re)create an associated cassette, e.g.: 26 | pytest ohapi/tests/test_projects.py::ProjectsTest::test_get_member_file_data_member_data_none 27 | 28 | (This only makes a new cassette if one doesn't already exist!) 29 | """ 30 | try: 31 | from _config_params_api import params 32 | for param in params: 33 | parameter_defaults[param] = params[param] 34 | except ImportError: 35 | pass 36 | 37 | for param in parameter_defaults: 38 | locals()[param] = parameter_defaults[param] 39 | 40 | 41 | FILTERSET = [('access_token', 'ACCESSTOKEN')] 42 | 43 | my_vcr = vcr.VCR(path_transformer=vcr.VCR.ensure_suffix('.yaml'), 44 | cassette_library_dir='ohapi/cassettes', 45 | filter_headers=[('Authorization', 'XXXXXXXX')], 46 | filter_query_parameters=FILTERSET, 47 | filter_post_data_parameters=FILTERSET) 48 | 49 | 50 | class ProjectsTest(TestCase): 51 | 52 | def setUp(self): 53 | pass 54 | 55 | def test_get_member_file_data_member_data_none(self): 56 | """ 57 | Test for :func:`_get_member_file_data` 58 | 59 | """ 60 | response = OHProject._get_member_file_data(member_data=MEMBER_DATA) 61 | self.assertEqual(response, {1: {'basename': 1}}) 62 | 63 | 64 | class ProjectsTestUpdateData(TestCase): 65 | """ 66 | Tests for :func:`update_data` 67 | 68 | """ 69 | 70 | def setUp(self): 71 | pass 72 | 73 | @my_vcr.use_cassette() 74 | def test_update_data_valid_master_access_token(self): 75 | ohproject = OHProject(master_access_token=MASTER_ACCESS_TOKEN) 76 | response = ohproject.update_data() 77 | self.assertEqual(len(response), 3) 78 | 79 | @my_vcr.use_cassette 80 | def test_update_data_expired_master_access_token(self): 81 | self.assertRaises(Exception, OHProject, MASTER_ACCESS_TOKEN_EXPIRED) 82 | 83 | @my_vcr.use_cassette 84 | def test_update_data_invalid_master_access_token(self): 85 | self.assertRaises(Exception, OHProject, MASTER_ACCESS_TOKEN_INVALID) 86 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_aws_valid_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=VALID_PMI1&filename=foo&metadata=%7B%22description%22%3A+%22Test+data%22%2C+%22tags%22%3A+%5B%22text%22%5D%7D 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['131'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.9.1] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"id":62525,"url":"https://open-humans-production.s3.amazonaws.com/member-files/direct-sharing-91/9e66a62e-61a8-11e8-b07f-0adfe9ac8c63/data1.csv?Signature=hpTZ1tI2qNR%2BjKsHpMAZQ7rDNOk%3D&Expires=1527445347&AWSAccessKeyId=AKIAIKNTFUJJTNS6N7HA"}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-store, max-age=0, must-revalidate, no-cache'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Sun, 27 May 2018 12:22:27 GMT'] 22 | Expires: ['Sun, 27 May 2018 12:22:27 GMT'] 23 | Last-Modified: ['Sun, 27 May 2018 12:22:27 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 201, message: Created} 29 | - request: 30 | body: !!python/object/new:_io.BytesIO 31 | state: !!python/tuple 32 | - !!binary | 33 | ZmlsZW5hbWUJdGFncwlkZXNjcmlwdGlvbgltZDUJY3JlYXRpb25fZGF0ZQ0KMgkxCTEJMQkyDQoz 34 | CTEJMQkyCTENCjQJMQkxCTIJMg0KNQkxCTIJMQkxDQo2CTEJMgkxCTINCjcJMQkyCTIJMQ0KOAkx 35 | CTIJMgkyDQo5CTIJMQkxCTENCjEwCTIJMQkxCTINCjExCTIJMQkyCTENCjEyCTIJMQkyCTINCjEz 36 | CTIJMgkxCTENCjI0CTMJMgkyCTINCg== 37 | - 0 38 | - null 39 | headers: 40 | Accept: ['*/*'] 41 | Accept-Encoding: ['gzip, deflate'] 42 | Connection: [keep-alive] 43 | Content-Length: ['193'] 44 | User-Agent: [python-requests/2.9.1] 45 | method: PUT 46 | uri: https://open-humans-production.s3.amazonaws.com/member-files/direct-sharing-91/9e66a62e-61a8-11e8-b07f-0adfe9ac8c63/data1.csv?AWSAccessKeyId=AKIAIKNTFUJJTNS6N7HA&Expires=1527445347&Signature=hpTZ1tI2qNR%2BjKsHpMAZQ7rDNOk%3D 47 | response: 48 | body: {string: ''} 49 | headers: 50 | Content-Length: ['0'] 51 | Date: ['Sun, 27 May 2018 12:22:30 GMT'] 52 | ETag: ['"a001239b1fc73e870624f30a1bbcfaa5"'] 53 | Server: [AmazonS3] 54 | x-amz-id-2: [AGsm/0GOPc3UL+r4kSqUdMrQ7HQUbsg1CKpdPGuQb5OZJVRfEePau1Sys3Fzb5sExchR+Zcz+dA=] 55 | x-amz-request-id: [6A87A532B72F8C2C] 56 | x-amz-version-id: [EUF6e13CrmKQmirroBg1i0Jbyai7TZYq] 57 | status: {code: 200, message: OK} 58 | - request: 59 | body: file_id=62525&project_member_id=40110289 60 | headers: 61 | Accept: ['*/*'] 62 | Accept-Encoding: ['gzip, deflate'] 63 | Connection: [keep-alive] 64 | Content-Length: ['40'] 65 | Content-Type: [application/x-www-form-urlencoded] 66 | User-Agent: [python-requests/2.9.1] 67 | method: POST 68 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/complete/?access_token=ACCESSTOKEN 69 | response: 70 | body: {string: '{"status":"ok","size":193}'} 71 | headers: 72 | Allow: ['POST, OPTIONS'] 73 | Cache-Control: ['must-revalidate, max-age=0, no-cache, no-store'] 74 | Connection: [close] 75 | Content-Language: [en] 76 | Content-Type: [application/json] 77 | Date: ['Sun, 27 May 2018 12:22:31 GMT'] 78 | Expires: ['Sun, 27 May 2018 12:22:31 GMT'] 79 | Last-Modified: ['Sun, 27 May 2018 12:22:31 GMT'] 80 | Server: [Cowboy] 81 | Vary: ['Accept, Accept-Language, Cookie'] 82 | Via: [1.1 vegur] 83 | X-Frame-Options: [SAMEORIGIN] 84 | status: {code: 200, message: OK} 85 | version: 1 86 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | ![awesome-dog](https://am24.akamaized.net/tms/cnt/uploads/2014/09/dog-youre-awesome.gif) 3 | 4 | First of all: You are awesome for wanting to contribute to this project. 5 | Thank you so much for this! 🎉 🍾 😍. 6 | Making it easy for people to integrate the *Open Humans* API into their own 7 | projects is central to our mission. 8 | 9 | The easiest way to contribute to this is by [opening issues](https://github.com/OpenHumans/open-humans-api/issues) 10 | of things you have noticed. Please check whether a similar issue already exists 11 | before opening a new one. 12 | 13 | ## Contributing code 14 | 15 | ### Install your development environment 16 | If you want to contribute code you should install `open-humans-api` in the editable mode. 17 | 18 | The easiest workflow to set up your development environment is the following. 19 | 20 | 1. Clone this repository into your own *GitHub* account. Then run the following: 21 | 22 | ``` 23 | git clone https://github.com/$YOUR_USER_NAME/open-humans-api.git 24 | cd open-humans-api 25 | pip install -e . 26 | ``` 27 | 28 | This will install the whole package from the cloned repository in editable mode. 29 | If you want to run tests and style checks locally you should also install 30 | the required development packages. From your repository you can do so by running 31 | `pip install -r dev-requirements.txt`. This will install `py.test` and `flake8`. 32 | 33 | ### Submitting a pull request 34 | If you have written some code that you would like us to merge into `OpenHumans/open-humans-api` 35 | [you can start a pull request](https://github.com/OpenHumans/open-humans-api/pulls) (PR). 36 | Some guidelines for these: 37 | - You should submit PRs from your own repository, ideally from a new feature-branch. 38 | To switch to one you can run `git checkout -b your_branch_name`. Do your edits, 39 | commit them, push them to your fork and then you can make a pull request to us. 40 | - If you are still working on a pull request please prefix the name 41 | of the pull request with `WIP` or `[WIP]` to state that it's a *Work In Progress*. 42 | - Once you think your pull request is ready to be merged edit the title and replace 43 | `[WIP]` with `[MRG]`. 44 | - We are using some continuous integration services to automatically evaluate all 45 | pull requests: 46 | - `TravisCI` will run all existing tests over each commit to see whether things have accidentally broken. Your pull request can only be merged if all tests pass 47 | - `HoundCI` runs `flake8` to identify whether your new code breaks the style guide. It will leave comments on each line that does so. Please respond to all the comments `Hound` makes, otherwise your PR can not be merged. 48 | - Lastly, `CodeClimate` will evaluate whether the addition of new code will decrease the overall test coverage. If your code decreases the test coverage it also can't be merged. This means: New functionalities should come with tests that show that they work. 49 | 50 | ### What to contribute 51 | 52 | #### Fixing existing problems 53 | You [read through our issues or created your own one](https://github.com/OpenHumans/open-humans-api/issues) and you happen to know the solution to it? And even better, you want to contribute the solution? We are [happy to accept your pull requests](https://github.com/OpenHumans/open-humans-api/pulls). Ideally pull requests should just tackle a single issue at a time and come with a description of what will be fixed and how. 54 | 55 | #### Contributing new features. 56 | You have an idea for a new feature or an extension of existing features? Maybe open an issue first so that we can all discuss whether it's a good idea to add it! 57 | 58 | #### Contributing documentation. 59 | Software is only as good as its documentation. If you see that our documentation is out of date, ambiguous, or just plain wrong: Please improve it and we'll happily merge it! 60 | 61 | --- 62 | Thanks again for your interest in contributing to the `open-humans-api` client for Python! We appreciate it a lot! 🎉 🍾 😍 63 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_stream_valid.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=PROJECTMEMBERID&metadata=%7B%22tags%22%3A+%5B%22text%22%5D%2C+%22description%22%3A+%22Lorem+ipsum+text%22%7D&filename=lorem_ipsum.txt 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['144'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.18.4] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"id":82295,"url":"https://open-humans-production.s3.amazonaws.com/member-files/direct-sharing-39/c6aa66f2-7e7c-11e8-8bfb-62c08ca290cf/lorem_ipsum.txt?Signature=%2F%2BPBbor7ML%2BknwLMUeSecbVnnKc%3D&Expires=1530615100&AWSAccessKeyId=AKIAIKNTFUJJTNS6N7HA"}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-store, no-cache, max-age=0, must-revalidate'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Tue, 03 Jul 2018 04:51:40 GMT'] 22 | Expires: ['Tue, 03 Jul 2018 04:51:40 GMT'] 23 | Last-Modified: ['Tue, 03 Jul 2018 04:51:40 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 201, message: Created} 29 | - request: 30 | body: !!python/object/new:_io.BytesIO 31 | state: !!python/tuple 32 | - !!binary | 33 | TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIGFkaXBpc2NpbmcgZWxpdCwg 34 | c2VkIGRvIGVpdXNtb2QgdGVtcG9yCmluY2lkaWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu 35 | YSBhbGlxdWEuIFV0IGVuaW0gYWQgbWluaW0gdmVuaWFtLCBxdWlzCm5vc3RydWQgZXhlcmNpdGF0 36 | aW9uIHVsbGFtY28gbGFib3JpcyBuaXNpIHV0IGFsaXF1aXAgZXggZWEgY29tbW9kbyBjb25zZXF1 37 | YXQuCkR1aXMgYXV0ZSBpcnVyZSBkb2xvciBpbiByZXByZWhlbmRlcml0IGluIHZvbHVwdGF0ZSB2 38 | ZWxpdCBlc3NlIGNpbGx1bSBkb2xvcmUKZXUgZnVnaWF0IG51bGxhIHBhcmlhdHVyLiBFeGNlcHRl 39 | dXIgc2ludCBvY2NhZWNhdCBjdXBpZGF0YXQgbm9uIHByb2lkZW50LCBzdW50CmluIGN1bHBhIHF1 40 | aSBvZmZpY2lhIGRlc2VydW50IG1vbGxpdCBhbmltIGlkIGVzdCBsYWJvcnVtLgo= 41 | - 0 42 | - null 43 | headers: 44 | Accept: ['*/*'] 45 | Accept-Encoding: ['gzip, deflate'] 46 | Connection: [keep-alive] 47 | Content-Length: ['446'] 48 | User-Agent: [python-requests/2.18.4] 49 | method: PUT 50 | uri: https://open-humans-production.s3.amazonaws.com/member-files/direct-sharing-39/c6aa66f2-7e7c-11e8-8bfb-62c08ca290cf/lorem_ipsum.txt?AWSAccessKeyId=AKIAIKNTFUJJTNS6N7HA&Expires=1530615100&Signature=%2F%2BPBbor7ML%2BknwLMUeSecbVnnKc%3D 51 | response: 52 | body: {string: ''} 53 | headers: 54 | Content-Length: ['0'] 55 | Date: ['Tue, 03 Jul 2018 04:51:42 GMT'] 56 | ETag: ['"cefd9dacdc23ee1faabb1e954d281274"'] 57 | Server: [AmazonS3] 58 | x-amz-id-2: [Q8u5ylHxSY6bNuLZkijyb9TYXP3Dji12VWRhZCZXc9QCLoyNaMvhrOOpx3caWlcdEsGdqo5fC6w=] 59 | x-amz-request-id: [DC809771BB568515] 60 | x-amz-version-id: [fBMyyoO8_Zp4oEKW2u0ZuSswJEfmgp0w] 61 | status: {code: 200, message: OK} 62 | - request: 63 | body: project_member_id=PROJECTMEMBERID&file_id=FILEID 64 | headers: 65 | Accept: ['*/*'] 66 | Accept-Encoding: ['gzip, deflate'] 67 | Connection: [keep-alive] 68 | Content-Length: ['40'] 69 | Content-Type: [application/x-www-form-urlencoded] 70 | User-Agent: [python-requests/2.18.4] 71 | method: POST 72 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/complete/?access_token=ACCESSTOKEN 73 | response: 74 | body: {string: '{"status":"ok","size":446}'} 75 | headers: 76 | Allow: ['POST, OPTIONS'] 77 | Cache-Control: ['max-age=0, must-revalidate, no-cache, no-store'] 78 | Connection: [close] 79 | Content-Language: [en] 80 | Content-Type: [application/json] 81 | Date: ['Tue, 03 Jul 2018 04:51:42 GMT'] 82 | Expires: ['Tue, 03 Jul 2018 04:51:42 GMT'] 83 | Last-Modified: ['Tue, 03 Jul 2018 04:51:42 GMT'] 84 | Server: [Cowboy] 85 | Vary: ['Accept, Accept-Language, Cookie'] 86 | Via: [1.1 vegur] 87 | X-Frame-Options: [SAMEORIGIN] 88 | status: {code: 200, message: OK} 89 | version: 1 90 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_upload_valid_file_valid_access_token.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: project_member_id=PROJECTMEMBERID&metadata=%7B%22tags%22%3A+%5B%22text%22%5D%2C+%22description%22%3A+%22Lorem+ipsum+text%22%7D&filename=lorem_ipsum.txt 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | Content-Length: ['144'] 9 | Content-Type: [application/x-www-form-urlencoded] 10 | User-Agent: [python-requests/2.18.4] 11 | method: POST 12 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/direct/?access_token=ACCESSTOKEN 13 | response: 14 | body: {string: '{"id":82289,"url":"https://open-humans-production.s3.amazonaws.com/member-files/direct-sharing-39/b1025250-7e4b-11e8-9e37-62c08ca290cf/lorem_ipsum.txt?Signature=F7vG8ZEvdqEpG0c8G%2FNttUr4zOs%3D&Expires=1530594019&AWSAccessKeyId=AKIAIKNTFUJJTNS6N7HA"}'} 15 | headers: 16 | Allow: ['POST, OPTIONS'] 17 | Cache-Control: ['no-store, no-cache, max-age=0, must-revalidate'] 18 | Connection: [close] 19 | Content-Language: [en] 20 | Content-Type: [application/json] 21 | Date: ['Mon, 02 Jul 2018 23:00:18 GMT'] 22 | Expires: ['Mon, 02 Jul 2018 23:00:19 GMT'] 23 | Last-Modified: ['Mon, 02 Jul 2018 23:00:19 GMT'] 24 | Server: [Cowboy] 25 | Vary: ['Accept, Accept-Language, Cookie'] 26 | Via: [1.1 vegur] 27 | X-Frame-Options: [SAMEORIGIN] 28 | status: {code: 201, message: Created} 29 | - request: 30 | body: !!python/object/new:_io.BytesIO 31 | state: !!python/tuple 32 | - !!binary | 33 | TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIGFkaXBpc2NpbmcgZWxpdCwg 34 | c2VkIGRvIGVpdXNtb2QgdGVtcG9yCmluY2lkaWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu 35 | YSBhbGlxdWEuIFV0IGVuaW0gYWQgbWluaW0gdmVuaWFtLCBxdWlzCm5vc3RydWQgZXhlcmNpdGF0 36 | aW9uIHVsbGFtY28gbGFib3JpcyBuaXNpIHV0IGFsaXF1aXAgZXggZWEgY29tbW9kbyBjb25zZXF1 37 | YXQuCkR1aXMgYXV0ZSBpcnVyZSBkb2xvciBpbiByZXByZWhlbmRlcml0IGluIHZvbHVwdGF0ZSB2 38 | ZWxpdCBlc3NlIGNpbGx1bSBkb2xvcmUKZXUgZnVnaWF0IG51bGxhIHBhcmlhdHVyLiBFeGNlcHRl 39 | dXIgc2ludCBvY2NhZWNhdCBjdXBpZGF0YXQgbm9uIHByb2lkZW50LCBzdW50CmluIGN1bHBhIHF1 40 | aSBvZmZpY2lhIGRlc2VydW50IG1vbGxpdCBhbmltIGlkIGVzdCBsYWJvcnVtLgo= 41 | - 0 42 | - null 43 | headers: 44 | Accept: ['*/*'] 45 | Accept-Encoding: ['gzip, deflate'] 46 | Connection: [keep-alive] 47 | Content-Length: ['446'] 48 | User-Agent: [python-requests/2.18.4] 49 | method: PUT 50 | uri: https://open-humans-production.s3.amazonaws.com/member-files/direct-sharing-39/b1025250-7e4b-11e8-9e37-62c08ca290cf/lorem_ipsum.txt?AWSAccessKeyId=AKIAIKNTFUJJTNS6N7HA&Expires=1530594019&Signature=F7vG8ZEvdqEpG0c8G%2FNttUr4zOs%3D 51 | response: 52 | body: {string: ''} 53 | headers: 54 | Content-Length: ['0'] 55 | Date: ['Mon, 02 Jul 2018 23:00:20 GMT'] 56 | ETag: ['"cefd9dacdc23ee1faabb1e954d281274"'] 57 | Server: [AmazonS3] 58 | x-amz-id-2: [exxbMlQa6HoaRoiHzo2ei0tTuLPxHgcdY2Vm6GFIlinAHseIdLPL3MUbQ7u0KF2oP3jxOlO0X0c=] 59 | x-amz-request-id: [E00052228845C084] 60 | x-amz-version-id: [rG7EIMjWf.yG6vpWewzaMqzTKfU66ds4] 61 | status: {code: 200, message: OK} 62 | - request: 63 | body: project_member_id=PROJECTMEMBERID&file_id=FILEID 64 | headers: 65 | Accept: ['*/*'] 66 | Accept-Encoding: ['gzip, deflate'] 67 | Connection: [keep-alive] 68 | Content-Length: ['40'] 69 | Content-Type: [application/x-www-form-urlencoded] 70 | User-Agent: [python-requests/2.18.4] 71 | method: POST 72 | uri: https://www.openhumans.org/api/direct-sharing/project/files/upload/complete/?access_token=ACCESSTOKEN 73 | response: 74 | body: {string: '{"size":446,"status":"ok"}'} 75 | headers: 76 | Allow: ['POST, OPTIONS'] 77 | Cache-Control: ['no-store, no-cache, max-age=0, must-revalidate'] 78 | Connection: [close] 79 | Content-Language: [en] 80 | Content-Type: [application/json] 81 | Date: ['Mon, 02 Jul 2018 23:00:19 GMT'] 82 | Expires: ['Mon, 02 Jul 2018 23:00:20 GMT'] 83 | Last-Modified: ['Mon, 02 Jul 2018 23:00:20 GMT'] 84 | Server: [Cowboy] 85 | Vary: ['Accept, Accept-Language, Cookie'] 86 | Via: [1.1 vegur] 87 | X-Frame-Options: [SAMEORIGIN] 88 | status: {code: 200, message: OK} 89 | version: 1 90 | -------------------------------------------------------------------------------- /ohapi/cassettes/test_download_file_valid_url.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: ['*/*'] 6 | Accept-Encoding: ['gzip, deflate'] 7 | Connection: [keep-alive] 8 | User-Agent: [python-requests/2.18.4] 9 | method: GET 10 | uri: http://www.loremipsum.de/downloads/version1.txt 11 | response: 12 | body: {string: "Lorem ipsum dolor sit amet, consectetaur adipisicing elit, sed 13 | do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad 14 | minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex 15 | ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate 16 | velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat 17 | cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id 18 | est laborum Et harumd und lookum like Greek to me, dereud facilis est er expedit 19 | distinct. Nam liber te conscient to factor tum poen legum odioque civiuda. 20 | Et tam neque pecun modut est neque nonor et imper ned libidig met, consectetur 21 | adipiscing elit, sed ut labore et dolore magna aliquam makes one wonder who 22 | would ever read this stuff? Bis nostrud exercitation ullam mmodo consequet. 23 | Duis aute in voluptate velit esse cillum dolore eu fugiat nulla pariatur. 24 | At vver eos et accusam dignissum qui blandit est praesent luptatum delenit 25 | aigue excepteur sint occae. Et harumd dereud facilis est er expedit distinct. 26 | Nam libe soluta nobis eligent optio est congue nihil impedit doming id Lorem 27 | ipsum dolor sit amet, consectetur adipiscing elit, set eiusmod tempor incidunt 28 | et labore et dolore magna aliquam. Ut enim ad minim veniam, quis nostrud exerc. 29 | Irure dolor in reprehend incididunt ut labore et dolore magna aliqua. Ut enim 30 | ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip 31 | ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate 32 | velit esse molestaie cillum. Tia non ob ea soluad incommod quae egen ium improb 33 | fugiend. Officia deserunt mollit anim id est laborum Et harumd dereud facilis 34 | est er expedit distinct. Nam liber te conscient to factor tum poen legum odioque 35 | civiuda et tam. Neque pecun modut est neque nonor et imper ned libidig met, 36 | consectetur adipiscing elit, sed ut labore et dolore magna aliquam is nostrud 37 | exercitation ullam mmodo consequet. Duis aute in voluptate velit esse cillum 38 | dolore eu fugiat nulla pariatur. At vver eos et accusam dignissum qui blandit 39 | est praesent. Trenz pruca beynocguon doas nog apoply su trenz ucu hugh rasoluguon 40 | monugor or trenz ucugwo jag scannar. Wa hava laasad trenzsa gwo producgs su 41 | IdfoBraid, yop quiel geg ba solaly rasponsubla rof trenzur sala ent dusgrubuguon. 42 | Offoctivo immoriatoly, hawrgasi pwicos asi sirucor.Thas sirutciun applios 43 | tyu thuso itoms ghuso pwicos gosi sirucor in mixent gosi sirucor ic mixent 44 | ples cak ontisi sowios uf Zerm hawr rwivos. Unte af phen neige pheings atoot 45 | Prexs eis phat eit sakem eit vory gast te Plok peish ba useing phen roxas. 46 | Eslo idaffacgad gef trenz beynocguon quiel ba trenz Spraadshaag ent trenz 47 | dreek wirc procassidt program. Cak pwico vux bolug incluros all uf cak sirucor 48 | hawrgasi itoms alung gith cakiw nog pwicos. Plloaso mako nuto uf cakso dodtos 49 | anr koop a cupy uf cak vux noaw yerw phuno. Whag schengos, uf efed, quiel 50 | ba mada su otrenzr swipontgwook proudgs hus yag su ba dagarmidad. Plasa maku 51 | noga wipont trenzsa schengos ent kaap zux copy wipont trenz kipg naar mixent 52 | phona. Cak pwico siructiun ruos nust apoply tyu cak UCU sisulutiun munityuw 53 | uw cak UCU-TGU jot scannow. Trens roxas eis ti Plokeing quert loppe eis yop 54 | prexs. Piy opher hawers, eit yaggles orn ti sumbloat alohe plok. Su havo loasor 55 | cakso tgu pwuructs tyu InfuBwain, ghu gill nug bo suloly sispunsiblo fuw cakiw 56 | salo anr ristwibutiun. Hei muk neme eis loppe. Treas em wankeing ont sime 57 | ploked peish rof phen sumbloat syug si phat phey gavet peish ta paat ein pheeir 58 | sumbloats. Aslu unaffoctor gef cak siructiun gill bo cak spiarshoot anet cak 59 | GurGanglo gur pwucossing pwutwam. Ghat dodtos, ig pany, gill bo maro tyu ucakw 60 | suftgasi pwuructs hod yot tyubo rotowminor. Plloaso mako nuto uf cakso dodtos 61 | anr koop a cupy uf cak vux noaw yerw phuno. Whag schengos, uf efed, quiel 62 | ba mada su otrenzr swipontgwook proudgs hus yag su ba dagarmidad. Plasa maku 63 | noga wipont trenzsa schengos ent kaap zux copy wipont trenz kipg naar mixent 64 | phona. Cak pwico siructiun ruos nust apoply tyu cak UCU sisulutiun munityuw 65 | uw cak UCU-TGU jot scannow. Trens roxas eis ti Plokeing quert loppe eis yop 66 | prexs. Piy opher hawers, eit yaggles orn ti sumbloat alohe plok. Su havo loasor 67 | cakso tgu pwuructs tyu. \n"} 68 | headers: 69 | Accept-Ranges: [bytes] 70 | Connection: [keep-alive] 71 | Content-Length: ['4247'] 72 | Content-Type: [text/plain] 73 | Date: ['Mon, 19 Mar 2018 18:40:21 GMT'] 74 | ETag: ['"1097-3feabd6847680"'] 75 | Keep-Alive: [timeout=15] 76 | Last-Modified: ['Fri, 19 Aug 2005 08:08:42 GMT'] 77 | Server: [Apache] 78 | status: {code: 200, message: OK} 79 | version: 1 80 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/stable/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | import os 16 | import sys 17 | sys.path.insert(0, os.path.abspath('..')) 18 | import mock 19 | MOCK_MODULES = [] 20 | for mod_name in MOCK_MODULES: 21 | sys.modules[mod_name] = mock.Mock() 22 | 23 | # -- Project information ----------------------------------------------------- 24 | 25 | project = 'open-humans-api' 26 | copyright = '2018, Open Humans' 27 | author = 'Open Humans' 28 | 29 | # The short X.Y version 30 | version = '' 31 | # The full version, including alpha/beta/rc tags 32 | release = '' 33 | 34 | 35 | # -- General configuration --------------------------------------------------- 36 | 37 | # If your documentation needs a minimal Sphinx version, state it here. 38 | # 39 | # needs_sphinx = '1.0' 40 | 41 | # Add any Sphinx extension module names here, as strings. They can be 42 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 43 | # ones. 44 | extensions = [ 45 | 'sphinx.ext.autodoc', 46 | 'sphinx_click.ext' 47 | ] 48 | 49 | # Add any paths that contain templates here, relative to this directory. 50 | templates_path = ['_templates'] 51 | 52 | # The suffix(es) of source filenames. 53 | # You can specify multiple suffix as a list of string: 54 | # 55 | # source_suffix = ['.rst', '.md'] 56 | source_suffix = '.rst' 57 | 58 | # The master toctree document. 59 | master_doc = 'index' 60 | 61 | # The language for content autogenerated by Sphinx. Refer to documentation 62 | # for a list of supported languages. 63 | # 64 | # This is also used if you do content translation via gettext catalogs. 65 | # Usually you set "language" from the command line for these cases. 66 | language = None 67 | 68 | # List of patterns, relative to source directory, that match files and 69 | # directories to ignore when looking for source files. 70 | # This pattern also affects html_static_path and html_extra_path . 71 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store','../ohapi/cassettes'] 72 | 73 | # The name of the Pygments (syntax highlighting) style to use. 74 | pygments_style = 'sphinx' 75 | 76 | 77 | # -- Options for HTML output ------------------------------------------------- 78 | 79 | # The theme to use for HTML and HTML Help pages. See the documentation for 80 | # a list of builtin themes. 81 | # 82 | # html_theme = 'alabaster' 83 | 84 | html_theme = "sphinx_rtd_theme" 85 | 86 | # Theme options are theme-specific and customize the look and feel of a theme 87 | # further. For a list of options available for each theme, see the 88 | # documentation. 89 | # 90 | # html_theme_options = {} 91 | 92 | # Add any paths that contain custom static files (such as style sheets) here, 93 | # relative to this directory. They are copied after the builtin static files, 94 | # so a file named "default.css" will overwrite the builtin "default.css". 95 | html_static_path = ['_static'] 96 | 97 | # Custom sidebar templates, must be a dictionary that maps document names 98 | # to template names. 99 | # 100 | # The default sidebars (for documents that don't match any pattern) are 101 | # defined by theme itself. Builtin themes are using these templates by 102 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 103 | # 'searchbox.html']``. 104 | # 105 | # html_sidebars = {} 106 | 107 | 108 | # -- Options for HTMLHelp output --------------------------------------------- 109 | 110 | # Output file base name for HTML help builder. 111 | htmlhelp_basename = 'open-humans-apidoc' 112 | 113 | 114 | # -- Options for LaTeX output ------------------------------------------------ 115 | 116 | latex_elements = { 117 | # The paper size ('letterpaper' or 'a4paper'). 118 | # 119 | # 'papersize': 'letterpaper', 120 | 121 | # The font size ('10pt', '11pt' or '12pt'). 122 | # 123 | # 'pointsize': '10pt', 124 | 125 | # Additional stuff for the LaTeX preamble. 126 | # 127 | # 'preamble': '', 128 | 129 | # Latex figure (float) alignment 130 | # 131 | # 'figure_align': 'htbp', 132 | } 133 | 134 | # Grouping the document tree into LaTeX files. List of tuples 135 | # (source start file, target name, title, 136 | # author, documentclass [howto, manual, or own class]). 137 | latex_documents = [ 138 | (master_doc, 'open-humans-api.tex', 'open-humans-api Documentation', 139 | 'Open Humans', 'manual'), 140 | ] 141 | 142 | 143 | # -- Options for manual page output ------------------------------------------ 144 | 145 | # One entry per manual page. List of tuples 146 | # (source start file, name, description, authors, manual section). 147 | man_pages = [ 148 | (master_doc, 'open-humans-api', 'open-humans-api Documentation', 149 | [author], 1) 150 | ] 151 | 152 | 153 | # -- Options for Texinfo output ---------------------------------------------- 154 | 155 | # Grouping the document tree into Texinfo files. List of tuples 156 | # (source start file, target name, title, author, 157 | # dir menu entry, description, category) 158 | texinfo_documents = [ 159 | (master_doc, 'open-humans-api', 'open-humans-api Documentation', 160 | author, 'open-humans-api', 'One line description of project.', 161 | 'Miscellaneous'), 162 | ] 163 | 164 | 165 | # -- Extension configuration ------------------------------------------------- 166 | -------------------------------------------------------------------------------- /docs/cli.rst: -------------------------------------------------------------------------------- 1 | Command Line Tools 2 | ****************** 3 | 4 | 5 | Example Use Cases 6 | ================= 7 | 8 | Download public files for a given project/user 9 | ---------------------------------------------- 10 | 11 | You can use ``ohpub-download`` on the command line to download all public files 12 | associated with a project. Here is an example that will download all public 13 | 23andMe files: 14 | 15 | .. code-block:: Shell 16 | 17 | mkdir 23andme # create new folder to store the data in 18 | ohpub-download --source direct-sharing-128 --directory 23andme 19 | 20 | To do the same for all public files of a given user: 21 | 22 | .. code-block:: Shell 23 | 24 | mkdir gedankenstuecke # create new folder to store the data in 25 | ohpub-download --username gedankenstuecke --directory gedankenstuecke 26 | 27 | 28 | Download private files from members of your project 29 | --------------------------------------------------- 30 | 31 | If you are running a project yourself you can use a ``master_access_token`` 32 | you can get `from the Open Humans website `_ 33 | to download the files shared with you. 34 | 35 | Here is an example of how to do this: 36 | 37 | .. code-block:: Shell 38 | 39 | mkdir my_downloaded_data # create new folder to store the data in 40 | ohproj-download --master-token my_master_access_token --directory my_downloaded_data 41 | 42 | 43 | Upload files to the accounts of your project members 44 | ---------------------------------------------------- 45 | 46 | Uploading data into your members accounts is two-step procedure using both 47 | ``ohproj-upload-metadata`` as well as ``ohproj-upload``, as you first need to draft 48 | some metadata that will go along with the files. 49 | 50 | Both commands expect a directory that contains sub-directories named after 51 | your members ``project_member_id``. An example directory structure with files could look like this: 52 | 53 | * member_data/ 54 | 55 | * 01234567/ 56 | 57 | * testdata.json 58 | * testdata.txt 59 | 60 | * 12345678/ 61 | 62 | * testdata.json 63 | * testdata.txt 64 | 65 | To upload this data we need to have a ``CSV`` file that contains the metadata. 66 | We can draft one using the following command: 67 | 68 | 69 | .. code-block:: Shell 70 | 71 | ohproj-upload-metadata -d member_data --create-csv member_data_metadata.csv 72 | 73 | The resulting CSV will look like this: 74 | 75 | .. code-block:: shell 76 | 77 | $ cat member_data_metadata.csv => 78 | project_member_id,filename,tags,description,md5,creation_date 79 | 01234567,testdata.txt,,,fa61a92e21a2597900cbde09d8ddbc1a,2016-08-23T15:23:22.277060+00:00 80 | 01234567,testdata.json,json,,577da9879649acaf17226a6461bd19c8,2016-08-23T16:06:16.415039+00:00 81 | 12345678,testdata.txt,,,fa61a92e21a2597900cbde09d8ddbc1a,2016-09-20T10:10:59.863201+00:00 82 | 12345678,testdata.json,json,,577da9879649acaf17226a6461bd19c8,2016-09-20T10:10:59.859201+00:00 83 | 84 | Edit this CSV with a text or spreadsheet editor of your choice to contain the metadata and then save it as a CSV again: 85 | 86 | .. code-block:: shell 87 | 88 | $ cat member_data_metadata.csv => 89 | project_member_id,filename,tags,description,md5,creation_date 90 | 1234567,testdata.txt,"txt, verbose-data",Complete test data in text format.,fa61a92e21a2597900cbde09d8ddbc1a,2016-08-23T15:23:22.277060+00:00 91 | 1234567,testdata.json,"json, metadata",Summary metadata in JSON format.,577da9879649acaf17226a6461bd19c8,2016-08-23T16:06:16.415039+00:00 92 | 12345678,testdata.txt,"txt, verbose-data",Complete test data in text format.,fa61a92e21a2597900cbde09d8ddbc1a,2016-09-20T10:10:59.863201+00:00 93 | 12345678,testdata.json,"json, metadata",Summary test data JSON.,577da9879649acaf17226a6461bd19c8,2016-09-20T10:10:59.859201+00:00 94 | 95 | We filled in the tags as a ``"``-escaped and comma-separated list as well as a text-description. 96 | With this we can now perform the actual upload like this: 97 | 98 | .. code-block:: shell 99 | 100 | ohproj-upload -T YOUR_MASTER_ACCESS_TOKEN --metadata-csv member_data_metadata.csv -d member_data 101 | 102 | This will upload the data from the ``member_data`` directory along with the metadata specified in ``member_data_metadata.csv``. 103 | 104 | Using an OAuth2 ``access_token`` 105 | --------------------------------------------------- 106 | The CLI tools also accept the usage of an OAuth2 ``access_token``s instead of an ``master_access_token``. 107 | This enables you to use the CLI tools in your own OAuth2 applications if you're not using Python for your development. 108 | 109 | Here is an example to download all the data shared by a single member through the command line: 110 | 111 | .. code-block:: shell 112 | 113 | ohproj-download -t personal_access_token -m '12345678' --directory download_directory 114 | 115 | 116 | List of commands and their parameters 117 | ===================================== 118 | 119 | .. click:: ohapi.command_line:public_data_download_cli 120 | :prog: ohpub-download 121 | 122 | .. click:: ohapi.command_line:download_cli 123 | :prog: ohproj-download 124 | 125 | .. click:: ohapi.command_line:download_metadata_cli 126 | :prog: ohproj-download-metadata 127 | 128 | .. click:: ohapi.command_line:upload_metadata_cli 129 | :prog: ohproj-upload-metadata 130 | 131 | .. click:: ohapi.command_line:upload_cli 132 | :prog: ohproj-upload 133 | 134 | .. click:: ohapi.command_line:oauth_token_exchange_cli 135 | :prog: ohproj-oauth2-token-exchange 136 | 137 | .. click:: ohapi.command_line:oauth2_auth_url_cli 138 | :prog: ohproj-oauth2-url 139 | 140 | .. click:: ohapi.command_line:message_cli 141 | :prog: ohproj-message 142 | 143 | .. click:: ohapi.command_line:delete_cli 144 | :prog: ohproj-delete 145 | 146 | .. automodule:: ohapi.command_line 147 | :members: 148 | :undoc-members: 149 | :show-inheritance: 150 | -------------------------------------------------------------------------------- /ohapi/public.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Utility functions to download public data files. 4 | """ 5 | import logging 6 | import os 7 | import re 8 | import signal 9 | 10 | from functools import partial 11 | try: 12 | from urllib import urlencode 13 | except ImportError: 14 | from urllib.parse import urlencode 15 | 16 | import click 17 | import concurrent.futures 18 | import requests 19 | import sys 20 | 21 | from humanfriendly import format_size, parse_size 22 | 23 | from .api import get_page 24 | 25 | 26 | BASE_URL = 'https://www.openhumans.org' 27 | BASE_URL_API = '{}/api/public-data/'.format(BASE_URL) 28 | LIMIT_DEFAULT = 100 29 | 30 | def signal_handler_cb(signal_name, frame): 31 | """ 32 | Exit on Ctrl-C. 33 | """ 34 | os._exit(1) 35 | 36 | 37 | def download_url(result, directory, max_bytes): 38 | """ 39 | Download a file. 40 | 41 | :param result: This field contains a url from which data will be 42 | downloaded. 43 | :param directory: This field is the target directory to which data will be 44 | downloaded. 45 | :param max_bytes: This field is the maximum file size in bytes. 46 | """ 47 | response = requests.get(result['download_url'], stream=True) 48 | 49 | # TODO: make this more robust by parsing the URL 50 | filename = response.url.split('/')[-1] 51 | filename = re.sub(r'\?.*$', '', filename) 52 | filename = '{}-{}'.format(result['user']['id'], filename) 53 | 54 | size = int(response.headers['Content-Length']) 55 | 56 | if size > max_bytes: 57 | logging.info('Skipping {}, {} > {}'.format(filename, format_size(size), 58 | format_size(max_bytes))) 59 | 60 | return 61 | 62 | logging.info('Downloading {} ({})'.format(filename, format_size(size))) 63 | 64 | output_path = os.path.join(directory, filename) 65 | 66 | try: 67 | stat = os.stat(output_path) 68 | 69 | if stat.st_size == size: 70 | logging.info('Skipping "{}"; exists and is the right size'.format( 71 | filename)) 72 | 73 | return 74 | else: 75 | logging.info('Removing "{}"; exists and is the wrong size'.format( 76 | filename)) 77 | 78 | os.remove(output_path) 79 | except OSError: 80 | # TODO: check errno here? 81 | pass 82 | 83 | with open(output_path, 'wb') as f: 84 | total_length = response.headers.get('content-length') 85 | total_length = int(total_length) 86 | dl = 0 87 | for chunk in response.iter_content(chunk_size=8192): 88 | if chunk: 89 | dl += len(chunk) 90 | f.write(chunk) 91 | d = int(50 * dl / total_length) 92 | sys.stdout.write("\r[%s%s]%d%s" % ('.' * d, 93 | '' * (50 - d), 94 | d * 2, 95 | '%')) 96 | sys.stdout.flush 97 | print("\n") 98 | 99 | logging.info('Downloaded {}'.format(filename)) 100 | 101 | 102 | def download(source=None, username=None, directory='.', max_size='128m', 103 | quiet=None, debug=None): 104 | """ 105 | Download public data from Open Humans. 106 | 107 | :param source: This field is the data source from which to download. It's 108 | default value is None. 109 | :param username: This fiels is username of user. It's default value is 110 | None. 111 | :param directory: This field is the target directory to which data is 112 | downloaded. 113 | :param max_size: This field is the maximum file size. It's default value is 114 | 128m. 115 | :param quiet: This field is the logging level. It's default value is 116 | None. 117 | :param debug: This field is the logging level. It's default value is 118 | None. 119 | """ 120 | if debug: 121 | logging.basicConfig(level=logging.DEBUG) 122 | elif quiet: 123 | logging.basicConfig(level=logging.ERROR) 124 | else: 125 | logging.basicConfig(level=logging.INFO) 126 | 127 | logging.debug("Running with source: '{}'".format(source) + 128 | " and username: '{}'".format(username) + 129 | " and directory: '{}'".format(directory) + 130 | " and max-size: '{}'".format(max_size)) 131 | 132 | signal.signal(signal.SIGINT, signal_handler_cb) 133 | 134 | max_bytes = parse_size(max_size) 135 | 136 | options = {} 137 | 138 | if source: 139 | options['source'] = source 140 | 141 | if username: 142 | options['username'] = username 143 | 144 | page = '{}?{}'.format(BASE_URL_API, urlencode(options)) 145 | 146 | results = [] 147 | counter = 1 148 | 149 | logging.info('Retrieving metadata') 150 | 151 | while True: 152 | logging.info('Retrieving page {}'.format(counter)) 153 | 154 | response = get_page(page) 155 | results = results + response['results'] 156 | 157 | if response['next']: 158 | page = response['next'] 159 | else: 160 | break 161 | 162 | counter += 1 163 | 164 | logging.info('Downloading {} files'.format(len(results))) 165 | 166 | download_url_partial = partial(download_url, directory=directory, 167 | max_bytes=max_bytes) 168 | 169 | with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor: 170 | for value in executor.map(download_url_partial, results): 171 | if value: 172 | logging.info(value) 173 | 174 | 175 | def get_members_by_source(base_url=BASE_URL_API): 176 | """ 177 | Function returns which members have joined each activity. 178 | 179 | :param base_url: It is URL: `https://www.openhumans.org/api/public-data`. 180 | """ 181 | url = '{}members-by-source/'.format(base_url) 182 | response = get_page(url) 183 | return response 184 | 185 | 186 | def get_sources_by_member(base_url=BASE_URL_API, limit=LIMIT_DEFAULT): 187 | """ 188 | Function returns which activities each member has joined. 189 | 190 | :param base_url: It is URL: `https://www.openhumans.org/api/public-data`. 191 | :param limit: It is the limit of data send by one request. 192 | """ 193 | url = '{}sources-by-member/'.format(base_url) 194 | page = '{}?{}'.format(url, urlencode({'limit': limit})) 195 | results = [] 196 | while True: 197 | data = get_page(page) 198 | results = results + data['results'] 199 | if data['next']: 200 | page = data['next'] 201 | else: 202 | break 203 | return results 204 | -------------------------------------------------------------------------------- /ohapi/tests/test_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from unittest.mock import mock_open, patch 3 | import arrow 4 | import os 5 | import vcr 6 | from posix import stat_result 7 | import stat 8 | from io import StringIO 9 | from ohapi.utils_fs import (guess_tags, load_metadata_csv, 10 | validate_metadata, characterize_local_files, 11 | read_id_list, download_file, 12 | write_metadata_to_filestream) 13 | from humanfriendly import parse_size 14 | 15 | MAX_FILE_DEFAULT = parse_size('128m') 16 | 17 | parameter_defaults = { 18 | 'CLIENT_ID_VALID': 'validclientid', 19 | 'CLIENT_SECRET_VALID': 'validclientsecret', 20 | 'CODE_VALID': 'validcode', 21 | 'REFRESH_TOKEN_VALID': 'validrefreshtoken', 22 | 'CLIENT_ID_INVALID': 'invalidclientid', 23 | 'CLIENT_SECRET_INVALID': 'invalidclientsecret', 24 | 'CODE_INVALID': 'invalidcode', 25 | 'REFRESH_TOKEN_INVALID': 'invalidrefreshtoken', 26 | 'REDIRECT_URI': 'http://127.0.0.1:5000/authorize_openhumans/', 27 | 'ACCESS_TOKEN': 'accesstoken', 28 | 'ACCESS_TOKEN_EXPIRED': 'accesstokenexpired', 29 | 'ACCESS_TOKEN_INVALID': 'accesstokeninvalid', 30 | 'MASTER_ACCESS_TOKEN': 'masteraccesstoken', 31 | 'INVALID_PMI1': 'invalidprojectmemberid1', 32 | 'INVALID_PMI2': 'invalidprojectmemberid2', 33 | 'VALID_PMI1': 'validprojectmemberid1', 34 | 'VALID_PMI2': 'validprojectmemberid2', 35 | 'SUBJECT': 'testsubject', 36 | 'MESSAGE': 'testmessage', 37 | 38 | } 39 | 40 | try: 41 | from _config_params_api import params 42 | for param in params: 43 | parameter_defaults[param] = params[param] 44 | except ImportError: 45 | pass 46 | 47 | for param in parameter_defaults: 48 | locals()[param] = parameter_defaults[param] 49 | 50 | 51 | FILTERSET = [('access_token', 'ACCESSTOKEN'), ('client_id', 'CLIENTID'), 52 | ('client_secret', 'CLIENTSECRET'), ('code', 'CODE'), 53 | ('refresh_token', 'REFRESHTOKEN'), 54 | ('invalid_access_token', 'INVALIDACCESSTOKEN')] 55 | 56 | my_vcr = vcr.VCR(path_transformer=vcr.VCR.ensure_suffix('.yaml'), 57 | cassette_library_dir='ohapi/cassettes', 58 | filter_headers=[('Authorization', 'XXXXXXXX')], 59 | filter_query_parameters=FILTERSET, 60 | filter_post_data_parameters=FILTERSET) 61 | 62 | 63 | def test_test(): 64 | x = 1 + 2 65 | assert x == 3 66 | 67 | 68 | class UtilsTest(TestCase): 69 | 70 | """ 71 | Tests for :func:`utils_fs` 72 | """ 73 | def setUp(self): 74 | pass 75 | 76 | def test_guess_tags(self): 77 | """ 78 | Tests for :func:`guess_tags` 79 | 80 | """ 81 | fname = "foo.vcf" 82 | self.assertEqual(guess_tags(fname), ['vcf']) 83 | fname = "foo.json.gz" 84 | self.assertEqual(guess_tags(fname), ['json']) 85 | fname = "foo.csv.bz2" 86 | self.assertEqual(guess_tags(fname), ['csv']) 87 | 88 | def test_load_metadata_csv(self): 89 | """ 90 | Tests for :func:`load_metadata_csv` 91 | 92 | """ 93 | metadata_users = load_metadata_csv('ohapi/tests/data/' 94 | 'metadata_proj_member_key.csv') 95 | self.assertEqual(len(metadata_users.keys()), 2) 96 | for key in metadata_users: 97 | self.assertEqual(len(metadata_users[key]), 2) 98 | 99 | metadata_files = load_metadata_csv('ohapi/tests/data/' 100 | 'metadata_proj_file_key_works.csv') 101 | self.assertEqual(len(metadata_files.keys()), 2) 102 | 103 | def test_validate_metadata(self): 104 | """ 105 | Tests for :func:`validate_metadata` 106 | 107 | """ 108 | directory = 'ohapi/tests/data/test_directory/' 109 | metadata = {'file_1.json', 'file_2.json'} 110 | self.assertEqual(validate_metadata(directory, metadata), True) 111 | 112 | def test_read_id_list_filepath_not_given(self): 113 | """ 114 | Tests for :func:`read_id_list` 115 | 116 | """ 117 | response = read_id_list(filepath=None) 118 | self.assertEqual(response, None) 119 | 120 | def test_read_id_list_filepath_given(self): 121 | """ 122 | Tests for :func:`read_id_list` 123 | 124 | """ 125 | filename = 'sample.txt' 126 | filedir = 'ohapi/tests/data/test_id_dir/' 127 | FILEPATH = os.path.join(filedir, filename) 128 | response = read_id_list(filepath=FILEPATH) 129 | self.assertEqual(response, ['12345678']) 130 | 131 | @my_vcr.use_cassette() 132 | def test_download_file_valid_url(self): 133 | """ 134 | Tests for :func:`download_file` 135 | 136 | """ 137 | with patch('ohapi.utils_fs.open', mock_open(), create=True): 138 | FILEPATH = 'ohapi/tests/data/test_download_dir/test_download_file' 139 | DOWNLOAD_URL = 'http://www.loremipsum.de/downloads/version1.txt' 140 | response = download_file( 141 | download_url=DOWNLOAD_URL, target_filepath=FILEPATH) 142 | self.assertEqual(response.status_code, 200) 143 | 144 | def test_mk_metadata_empty_directory(self): 145 | with patch('ohapi.utils_fs.os.path.isdir') as mocked_isdir, \ 146 | patch('ohapi.utils_fs.os.listdir') as mocked_listdir: 147 | mocked_isdir.return_value = True 148 | mocked_listdir.return_value = [] 149 | teststream = StringIO() 150 | write_metadata_to_filestream('test_dir', teststream) 151 | assert(teststream.getvalue() == 'filename,tags,description,' + 152 | 'md5,creation_date\r\n') 153 | 154 | def test_mk_metadata_single_user(self): 155 | with patch('ohapi.utils_fs.os.path.isdir') as mocked_isdir, \ 156 | patch('ohapi.utils_fs.os.listdir') as mocked_listdir: 157 | with patch('ohapi.utils_fs.open', 158 | mock_open(read_data=b'some stuff'), 159 | create=True): 160 | mocked_isdir.return_value = True 161 | mocked_listdir.return_value = ['f1.txt', 'f2.txt'] 162 | mocked_isdir.side_effect = [False, False] 163 | try: 164 | def fake_stat(arg): 165 | faked = list(orig_os_stat('/tmp')) 166 | faked[stat.ST_SIZE] = len("some stuff") 167 | faked[stat.ST_CTIME] = "1497164239.6941652" 168 | return stat_result(faked) 169 | orig_os_stat = os.stat 170 | os.stat = fake_stat 171 | teststream = StringIO() 172 | write_metadata_to_filestream('test_dir', teststream) 173 | content = teststream.getvalue() 174 | assert(len(content) == 197 and content.startswith( 175 | 'filename,tags,description,md5,' + 176 | 'creation_date\r\n') and "f1.txt,,,beb6a43adfb95" + 177 | "0ec6f82ceed19beee21,2017-06-11T06:57:19.694165+" + 178 | "00:00\r\n" in content and "f2.txt,,,beb6a43adfb" + 179 | "950ec6f82ceed19beee21," + 180 | "2017-06-11T06:57:19.694165+00:00\r\n" in content) 181 | finally: 182 | os.stat = orig_os_stat 183 | 184 | def test_mk_metadata_multi_user(self): 185 | with patch('ohapi.utils_fs.open', mock_open(read_data=b'some stuff'), 186 | create=True): 187 | orig_os_stat = os.stat 188 | orig_os_is_dir = os.path.isdir 189 | orig_os_list_dir = os.listdir 190 | try: 191 | def fake_stat(arg): 192 | faked = list(orig_os_stat('/tmp')) 193 | faked[stat.ST_SIZE] = len("some stuff") 194 | faked[stat.ST_CTIME] = "1497164239.6941652" 195 | return stat_result(faked) 196 | 197 | def fake_list_dir(arg): 198 | if arg == 'test_dir': 199 | return ['12345678'] 200 | elif '12345678' in arg: 201 | return ['f1.txt', 'f2.txt'] 202 | 203 | def fake_is_dir(arg): 204 | if ('test_dir') in arg or ('12345678') in arg: 205 | return True 206 | elif ('f1.txt') in arg or ('f2.txt') in arg: 207 | return False 208 | os.listdir = fake_list_dir 209 | os.path.isdir = fake_is_dir 210 | os.stat = fake_stat 211 | teststream = StringIO() 212 | write_metadata_to_filestream('test_dir', teststream) 213 | content = teststream.getvalue() 214 | assert(len(content) == 233 and content.startswith( 215 | 'project_member_id,filename,tags,description,md5,' + 216 | 'creation_date\r\n') and "12345678,f1.txt,,,beb6a43adfb9" + 217 | "50ec6f82ceed19beee21,2017-06-11T06:57:19.694165+" + 218 | "00:00\r\n" in content and "12345678,f2.txt,,,beb6a43adf" + 219 | "b950ec6f82ceed19beee21," + 220 | "2017-06-11T06:57:19.694165+00:00\r\n" in content) 221 | finally: 222 | os.stat = orig_os_stat 223 | os.listdir = orig_os_list_dir 224 | os.path.isdir = orig_os_is_dir 225 | -------------------------------------------------------------------------------- /ohapi/projects.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions to use master_access_tokens to interact with a project 3 | """ 4 | 5 | import logging 6 | import os 7 | 8 | import arrow 9 | from humanfriendly import parse_size 10 | 11 | from .api import delete_file, get_all_results, get_page, upload_aws 12 | from .utils_fs import download_file, validate_metadata 13 | 14 | MAX_SIZE_DEFAULT = '128m' 15 | 16 | 17 | class OHProject: 18 | """ 19 | Work with an Open Humans Project. 20 | """ 21 | def __init__(self, master_access_token): 22 | self.master_access_token = master_access_token 23 | self.project_data = None 24 | self.update_data() 25 | 26 | @staticmethod 27 | def _get_member_file_data(member_data, id_filename=False): 28 | """ 29 | Helper function to get file data of member of a project. 30 | 31 | :param member_data: This field is data related to member in a project. 32 | """ 33 | file_data = {} 34 | for datafile in member_data['data']: 35 | if id_filename: 36 | basename = '{}.{}'.format(datafile['id'], datafile['basename']) 37 | else: 38 | basename = datafile['basename'] 39 | if (basename not in file_data or 40 | arrow.get(datafile['created']) > 41 | arrow.get(file_data[basename]['created'])): 42 | file_data[basename] = datafile 43 | return file_data 44 | 45 | def update_data(self): 46 | """ 47 | Returns data for all users including shared data files. 48 | """ 49 | url = ('https://www.openhumans.org/api/direct-sharing/project/' 50 | 'members/?access_token={}'.format(self.master_access_token)) 51 | results = get_all_results(url) 52 | self.project_data = dict() 53 | for result in results: 54 | self.project_data[result['project_member_id']] = result 55 | if len(result['data']) < result['file_count']: 56 | member_data = get_page(result['exchange_member']) 57 | final_data = member_data['data'] 58 | while member_data['next']: 59 | member_data = get_page(member_data['next']) 60 | final_data = final_data + member_data['data'] 61 | self.project_data[ 62 | result['project_member_id']]['data'] = final_data 63 | return self.project_data 64 | 65 | @classmethod 66 | def download_member_project_data(cls, member_data, target_member_dir, 67 | max_size=MAX_SIZE_DEFAULT, 68 | id_filename=False): 69 | """ 70 | Download files to sync a local dir to match OH member project data. 71 | 72 | :param member_data: This field is data related to member in a project. 73 | :param target_member_dir: This field is the target directory where data 74 | will be downloaded. 75 | :param max_size: This field is the maximum file size. It's default 76 | value is 128m. 77 | """ 78 | logging.debug('Download member project data...') 79 | sources_shared = member_data['sources_shared'] 80 | file_data = cls._get_member_file_data(member_data, 81 | id_filename=id_filename) 82 | for basename in file_data: 83 | # This is using a trick to identify a project's own data in an API 84 | # response, without knowing the project's identifier: if the data 85 | # isn't a shared data source, it must be the project's own data. 86 | if file_data[basename]['source'] in sources_shared: 87 | continue 88 | target_filepath = os.path.join(target_member_dir, basename) 89 | download_file(download_url=file_data[basename]['download_url'], 90 | target_filepath=target_filepath, 91 | max_bytes=parse_size(max_size)) 92 | 93 | @classmethod 94 | def download_member_shared(cls, member_data, target_member_dir, source=None, 95 | max_size=MAX_SIZE_DEFAULT, id_filename=False): 96 | """ 97 | Download files to sync a local dir to match OH member shared data. 98 | 99 | Files are downloaded to match their "basename" on Open Humans. 100 | If there are multiple files with the same name, the most recent is 101 | downloaded. 102 | 103 | :param member_data: This field is data related to member in a project. 104 | :param target_member_dir: This field is the target directory where data 105 | will be downloaded. 106 | :param source: This field is the source from which to download data. 107 | :param max_size: This field is the maximum file size. It's default 108 | value is 128m. 109 | """ 110 | logging.debug('Download member shared data...') 111 | sources_shared = member_data['sources_shared'] 112 | file_data = cls._get_member_file_data(member_data, 113 | id_filename=id_filename) 114 | 115 | logging.info('Downloading member data to {}'.format(target_member_dir)) 116 | for basename in file_data: 117 | 118 | # If not in sources shared, it's the project's own data. Skip. 119 | if file_data[basename]['source'] not in sources_shared: 120 | continue 121 | 122 | # Filter source if specified. Determine target directory for file. 123 | if source: 124 | if source == file_data[basename]['source']: 125 | target_filepath = os.path.join(target_member_dir, basename) 126 | else: 127 | continue 128 | else: 129 | source_data_dir = os.path.join(target_member_dir, 130 | file_data[basename]['source']) 131 | if not os.path.exists(source_data_dir): 132 | os.mkdir(source_data_dir) 133 | target_filepath = os.path.join(source_data_dir, basename) 134 | 135 | download_file(download_url=file_data[basename]['download_url'], 136 | target_filepath=target_filepath, 137 | max_bytes=parse_size(max_size)) 138 | 139 | def download_all(self, target_dir, source=None, project_data=False, 140 | memberlist=None, excludelist=None, 141 | max_size=MAX_SIZE_DEFAULT, id_filename=False): 142 | """ 143 | Download data for all users including shared data files. 144 | 145 | :param target_dir: This field is the target directory to download data. 146 | :param source: This field is the data source. It's default value is 147 | None. 148 | :param project_data: This field is data related to particular project. 149 | It's default value is False. 150 | :param memberlist: This field is list of members whose data will be 151 | downloaded. It's default value is None. 152 | :param excludelist: This field is list of members whose data will be 153 | skipped. It's default value is None. 154 | :param max_size: This field is the maximum file size. It's default 155 | value is 128m. 156 | """ 157 | members = self.project_data.keys() 158 | for member in members: 159 | if not (memberlist is None) and member not in memberlist: 160 | logging.debug('Skipping {}, not in memberlist'.format(member)) 161 | continue 162 | if excludelist and member in excludelist: 163 | logging.debug('Skipping {}, in excludelist'.format(member)) 164 | continue 165 | member_dir = os.path.join(target_dir, member) 166 | if not os.path.exists(member_dir): 167 | os.mkdir(member_dir) 168 | if project_data: 169 | self.download_member_project_data( 170 | member_data=self.project_data[member], 171 | target_member_dir=member_dir, 172 | max_size=max_size, 173 | id_filename=id_filename) 174 | else: 175 | self.download_member_shared( 176 | member_data=self.project_data[member], 177 | target_member_dir=member_dir, 178 | source=source, 179 | max_size=max_size, 180 | id_filename=id_filename) 181 | 182 | @staticmethod 183 | def upload_member_from_dir(member_data, target_member_dir, metadata, 184 | access_token, mode='default', 185 | max_size=MAX_SIZE_DEFAULT): 186 | """ 187 | Upload files in target directory to an Open Humans member's account. 188 | 189 | The default behavior is to overwrite files with matching filenames on 190 | Open Humans, but not otherwise delete files. 191 | 192 | If the 'mode' parameter is 'safe': matching filenames will not be 193 | overwritten. 194 | 195 | If the 'mode' parameter is 'sync': files on Open Humans that are not 196 | in the local directory will be deleted. 197 | 198 | :param member_data: This field is data related to member in a project. 199 | :param target_member_dir: This field is the target directory from where 200 | data will be uploaded. 201 | :param metadata: This field is metadata for files to be uploaded. 202 | :param access_token: This field is user specific access token. 203 | :param mode: This field takes three value default, sync, safe. It's 204 | default value is 'default'. 205 | :param max_size: This field is the maximum file size. It's default 206 | value is 128m. 207 | """ 208 | if not validate_metadata(target_member_dir, metadata): 209 | raise ValueError('Metadata should match directory contents!') 210 | project_data = {f['basename']: f for f in member_data['data'] if 211 | f['source'] not in member_data['sources_shared']} 212 | for filename in metadata: 213 | if filename in project_data and mode == 'safe': 214 | logging.info('Skipping {}, remote exists with matching' 215 | ' name'.format(filename)) 216 | continue 217 | filepath = os.path.join(target_member_dir, filename) 218 | remote_file_info = (project_data[filename] if filename in 219 | project_data else None) 220 | upload_aws(target_filepath=filepath, 221 | metadata=metadata[filename], 222 | access_token=access_token, 223 | project_member_id=member_data['project_member_id'], 224 | remote_file_info=remote_file_info) 225 | if mode == 'sync': 226 | for filename in project_data: 227 | if filename not in metadata: 228 | logging.debug("Deleting {}".format(filename)) 229 | delete_file( 230 | file_basename=filename, 231 | access_token=access_token, 232 | project_member_id=member_data['project_member_id']) 233 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # open-humans-api 2 | [![Build Status](https://travis-ci.org/OpenHumans/open-humans-api.svg?branch=master)](https://travis-ci.org/OpenHumans/open-humans-api) [![Maintainability](https://api.codeclimate.com/v1/badges/f44ae877944131bf59c2/maintainability)](https://codeclimate.com/github/OpenHumans/open-humans-api/maintainability) [![Test Coverage](https://api.codeclimate.com/v1/badges/f44ae877944131bf59c2/test_coverage)](https://codeclimate.com/github/OpenHumans/open-humans-api/test_coverage) 3 | [![Documentation Status](https://readthedocs.org/projects/open-humans-api/badge/?version=latest)](http://open-humans-api.readthedocs.io/en/latest/?badge=latest) 4 | 5 | 6 | 7 | This package aims to provide some tools to facilitate working with the Open 8 | Humans APIs. 9 | 10 | In particular, this package provides some command line tools for data file 11 | downloads and uploads. These tools are listed below. 12 | 13 | ## Installation 14 | 15 | This package is distributed via PyPI. We recommend you install it using 16 | pip, e.g. `pip install open-humans-api`. If you want to learn how to install 17 | this module to further develop it and contribute code please 18 | [read our `CONTRIBUTING.md`](https://github.com/OpenHumans/open-humans-api/blob/master/CONTRIBUTING.md) 19 | which explains all these things. 20 | 21 | ## Command line tools 22 | 23 | Command line tools aim to facilitate one-off operations by users 24 | (for example, one-off data upload by a project). 25 | 26 | These tools might also be helpful for programmers seeking to use the API 27 | in non-Python programmatic contexts. 28 | 29 | ### ohpub-download 30 | 31 | ``` 32 | Usage: ohpub-download [OPTIONS] 33 | 34 | Download public data from Open Humans. 35 | 36 | Options: 37 | -s, --source TEXT the source to download files from 38 | -u, --username TEXT the user to download files from 39 | -d, --directory TEXT the directory for downloaded files 40 | -m, --max-size TEXT the maximum file size to download 41 | -q, --quiet Report ERROR level logging to stdout 42 | --debug Report DEBUG level logging to stdout 43 | --help Show this message and exit. 44 | ``` 45 | 46 | #### Examples 47 | 48 | ``` 49 | # download all 23andMe files to 23andme/ 50 | $ mkdir 23andme 51 | $ ohpub-download --source direct-sharing-128 --directory 23andme 52 | # download all of beau's files to the current directory 53 | $ ohpub-download --username beau 54 | ``` 55 | 56 | ### ohproj-download 57 | 58 | ``` 59 | Usage: ohproj-download [OPTIONS] 60 | 61 | Download data from project members to the target directory. 62 | 63 | Unless this is a member-specific download, directories will be created for 64 | each project member ID. Also, unless a source is specified, all shared 65 | sources are downloaded and data is sorted into subdirectories according to 66 | source. 67 | 68 | Projects can optionally return data to Open Humans member accounts. If 69 | project_data is True (or the "--project-data" flag is used), this data 70 | (the project's own data files, instead of data from other sources) will be 71 | downloaded for each member. 72 | 73 | Options: 74 | -d, --directory TEXT Target directory for downloaded files. [required] 75 | -T, --master-token TEXT Project master access token. 76 | -m, --member TEXT Project member ID. 77 | -t, --access-token TEXT OAuth2 user access token. 78 | -s, --source TEXT Only download files from this source. 79 | --project-data TEXT Download this project's own data. 80 | --max-size TEXT Maximum file size to download. [default: 128m] 81 | -v, --verbose Report INFO level logging to stdout 82 | --debug Report DEBUG level logging to stdout. 83 | --memberlist TEXT Text file with whitelist IDs to retrieve 84 | --excludelist TEXT Text file with blacklist IDs to avoid 85 | --help Show this message and exit. 86 | ``` 87 | 88 | ### ohproj-download-metadata 89 | 90 | ``` 91 | Usage: ohproj-download-metadata [OPTIONS] 92 | 93 | Output CSV with metadata for a project's downloadable files in Open 94 | Humans. 95 | 96 | Options: 97 | -T, --master-token TEXT Project master access token. [required] 98 | -v, --verbose Show INFO level logging 99 | --debug Show DEBUG level logging. 100 | --output-csv TEXT Output project metedata CSV [required] 101 | --help Show this message and exit. 102 | ``` 103 | 104 | ### ohproj-upload-metadata 105 | 106 | ``` 107 | Usage: ohproj-upload-metadata [OPTIONS] 108 | 109 | Draft or review metadata files for uploading files to Open Humans. 110 | 111 | The target directory should either represent files for a single member (no 112 | subdirectories), or contain a subdirectory for each project member ID. 113 | 114 | Options: 115 | -d, --directory TEXT Target directory [required] 116 | --create-csv TEXT Create draft CSV metadata [required] 117 | --max-size TEXT Maximum file size to consider. [default: 128m] 118 | -v, --verbose Show INFO level logging 119 | --debug Show DEBUG level logging. 120 | --help Show this message and exit. 121 | ``` 122 | 123 | #### Example usage: creating metadata for data upload 124 | 125 | Create directory containing data for project members. For example it might 126 | look like the following example (two project members with IDs '01234567' 127 | and '12345678'). 128 | 129 | * member_data/ 130 | * 01234567/ 131 | * testdata.json 132 | * testdata.txt 133 | * 12345678/ 134 | * testdata.json 135 | * testdata.txt 136 | 137 | Draft metadata file: 138 | ``` 139 | $ ohproj-upload-metadata -d member_data --create-csv member_data_metadata.csv 140 | ``` 141 | 142 | Initially it looks like this: 143 | ``` 144 | project_member_id,filename,tags,description,md5,creation_date 145 | 01234567,testdata.txt,,,fa61a92e21a2597900cbde09d8ddbc1a,2016-08-23T15:23:22.277060+00:00 146 | 01234567,testdata.json,json,,577da9879649acaf17226a6461bd19c8,2016-08-23T16:06:16.415039+00:00 147 | 12345678,testdata.txt,,,fa61a92e21a2597900cbde09d8ddbc1a,2016-09-20T10:10:59.863201+00:00 148 | 12345678,testdata.json,json,,577da9879649acaf17226a6461bd19c8,2016-09-20T10:10:59.859201+00:00 149 | ``` 150 | 151 | You can use a spreadsheet editor to edit it. Make sure to save the result as 152 | CSV! For example, it might look like this if you add descriptions and more tags: 153 | ``` 154 | 1234567,testdata.txt,"txt, verbose-data",Complete test data in text format.,fa61a92e21a2597900cbde09d8ddbc1a,2016-08-23T15:23:22.277060+00:00 155 | 1234567,testdata.json,"json, metadata",Summary metadata in JSON format.,577da9879649acaf17226a6461bd19c8,2016-08-23T16:06:16.415039+00:00 156 | 12345678,testdata.txt,"txt, verbose-data",Complete test data in text format.,fa61a92e21a2597900cbde09d8ddbc1a,2016-09-20T10:10:59.863201+00:00 157 | 12345678,testdata.json,"json, metadata",Summary test data JSON.,577da9879649acaf17226a6461bd19c8,2016-09-20T10:10:59.859201+00:00 158 | ``` 159 | 160 | ### ohproj-upload 161 | ``` 162 | Usage: ohproj-upload [OPTIONS] 163 | 164 | Upload files for the project to Open Humans member accounts. 165 | 166 | If using a master access token and not specifying member ID: 167 | 168 | (1) Files should be organized in subdirectories according to project 169 | member ID, e.g.: 170 | 171 | main_directory/01234567/data.json 172 | main_directory/12345678/data.json 173 | main_directory/23456789/data.json 174 | 175 | (2) The metadata CSV should have the following format: 176 | 177 | 1st column: Project member ID 178 | 2nd column: filenames 179 | 3rd & additional columns: Metadata fields (see below) 180 | 181 | If uploading for a specific member: 182 | (1) The local directory should not contain subdirectories. 183 | (2) The metadata CSV should have the following format: 184 | 1st column: filenames 185 | 2nd & additional columns: Metadata fields (see below) 186 | 187 | The default behavior is to overwrite files with matching filenames on Open 188 | Humans, but not otherwise delete files. (Use --safe or --sync to change 189 | this behavior.) 190 | 191 | If included, the following metadata columns should be correctly formatted: 192 | 'tags': should be comma-separated strings 193 | 'md5': should match the file's md5 hexdigest 194 | 'creation_date', 'start_date', 'end_date': ISO 8601 dates or datetimes 195 | 196 | Other metedata fields (e.g. 'description') can be arbitrary strings. 197 | 198 | Options: 199 | -d, --directory TEXT Target directory for downloaded files. [required] 200 | --metadata-csv TEXT CSV file containing file metadata. [required] 201 | -T, --master-token TEXT Project master access token. 202 | -m, --member TEXT Project member ID. 203 | -t, --access-token TEXT OAuth2 user access token. 204 | --safe Do not overwrite files in Open Humans. 205 | --sync Delete files not present in local directories. 206 | --max-size TEXT Maximum file size to download. [default: 128m] 207 | -v, --verbose Report INFO level logging to stdout 208 | --debug Report DEBUG level logging to stdout. 209 | --help Show this message and exit. 210 | ``` 211 | 212 | #### Example usage: uploading data 213 | 214 | For organizing the data files and creating a metadata file, see the example 215 | usage for the `ohproj-metadata` command line tool. 216 | 217 | Uploading that data with a master access token: 218 | ``` 219 | $ ohproj-upload -T MASTER_ACCESS_TOKEN --metadata-csv member_data_metadata.csv -d member_data 220 | ``` 221 | 222 | ### ohproj-oauth2-url 223 | ``` 224 | Usage: ohproj-oauth2-url [OPTIONS] 225 | 226 | Get the OAuth2 URL of specified Open Humans Project 227 | 228 | Specifying Redirect URL is optional but client id is required. 229 | 230 | Options: 231 | -r, --redirect_uri TEXT Redirect URL of the project 232 | -c, --client_id TEXT Client ID of the project 233 | ``` 234 | 235 | ### ohproj-message 236 | ``` 237 | Usage: ohproj-message [OPTIONS] 238 | 239 | Message the project members of an Open Humans Project 240 | 241 | Options: 242 | -s, --subject TEXT Subject of the message 243 | -m, --message_body TEXT Compose message 244 | -at, --access_token TEXT OAuth2 user access token 245 | --all_memebers BOOL Setting this true sends message to all members of the project. By default it is false. 246 | --project_member_ids ID A list of comma separated IDs. Example argument: "ID1, ID2" 247 | -v, --verbose Show INFO level logging. Default value is FALSE 248 | --debug Show DEBUG level logging. Default value is FALSE 249 | ``` 250 | 251 | ### ohproj-delete 252 | ``` 253 | Usage: ohproj-delete [OPTIONS] 254 | 255 | -T, --access_token TEXT Access token of the project 256 | -m, --project_member_id ID Project Member ID 257 | -b, --file_basename TEXT File Basename 258 | -i, --file_id File ID 259 | -all_files BOOL Setting true to all_files deletes all the files in the given project. By default the value is false. 260 | ``` 261 | 262 | 263 | #### Setting up documentation locally 264 | 265 | Navigate to the docs folder. 266 | ``` 267 | $ cd docs 268 | ``` 269 | 270 | Run the make html command 271 | ``` 272 | $ make html 273 | ``` 274 | 275 | The documentation will be in docs_html folder. 276 | ``` 277 | $ cd docs_html 278 | ``` 279 | 280 | Open index.html. 281 | 282 | #### Rebuilding the documentation locally 283 | 284 | Navigate to the docs folder. 285 | ``` 286 | $ cd docs 287 | ``` 288 | 289 | Run the make clean command 290 | ``` 291 | $ make clean 292 | ``` 293 | 294 | Run the make html command 295 | ``` 296 | $ make html 297 | ``` 298 | 299 | The documentation will be in docs_html folder. 300 | ``` 301 | $ cd docs_html 302 | ``` 303 | 304 | Open index.html. 305 | -------------------------------------------------------------------------------- /ohapi/api.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions to use the OAuth2 project API to e.g. message users, download 3 | their files, upload new ones and delete existing files. 4 | """ 5 | 6 | from collections import OrderedDict 7 | import json 8 | import logging 9 | import os 10 | try: 11 | import urllib.parse as urlparse 12 | except ImportError: 13 | import urlparse 14 | 15 | from humanfriendly import format_size, parse_size 16 | import requests 17 | 18 | 19 | MAX_FILE_DEFAULT = parse_size('128m') 20 | OH_BASE_URL = os.getenv('OHAPI_OH_BASE_URL', 'https://www.openhumans.org/') 21 | 22 | 23 | class SettingsError(Exception): 24 | pass 25 | 26 | 27 | def oauth2_auth_url(redirect_uri=None, client_id=None, base_url=OH_BASE_URL): 28 | """ 29 | Returns an OAuth2 authorization URL for a project, given Client ID. This 30 | function constructs an authorization URL for a user to follow. 31 | The user will be redirected to Authorize Open Humans data for our external 32 | application. An OAuth2 project on Open Humans is required for this to 33 | properly work. To learn more about Open Humans OAuth2 projects, go to: 34 | https://www.openhumans.org/direct-sharing/oauth2-features/ 35 | 36 | :param redirect_uri: This field is set to `None` by default. However, if 37 | provided, it appends it in the URL returned. 38 | :param client_id: This field is also set to `None` by default however, 39 | is a mandatory field for the final URL to work. It uniquely identifies 40 | a given OAuth2 project. 41 | :param base_url: It is this URL `https://www.openhumans.org`. 42 | """ 43 | if not client_id: 44 | client_id = os.getenv('OHAPI_CLIENT_ID') 45 | if not client_id: 46 | raise SettingsError( 47 | "Client ID not provided! Provide client_id as a parameter, " 48 | "or set OHAPI_CLIENT_ID in your environment.") 49 | params = OrderedDict([ 50 | ('client_id', client_id), 51 | ('response_type', 'code'), 52 | ]) 53 | if redirect_uri: 54 | params['redirect_uri'] = redirect_uri 55 | 56 | auth_url = urlparse.urljoin( 57 | base_url, '/direct-sharing/projects/oauth2/authorize/?{}'.format( 58 | urlparse.urlencode(params))) 59 | 60 | return auth_url 61 | 62 | 63 | def oauth2_token_exchange(client_id, client_secret, redirect_uri, 64 | base_url=OH_BASE_URL, code=None, refresh_token=None): 65 | """ 66 | Exchange code or refresh token for a new token and refresh token. For the 67 | first time when a project is created, code is required to generate refresh 68 | token. Once the refresh token is obtained, it can be used later on for 69 | obtaining new access token and refresh token. The user must store the 70 | refresh token to obtain the new access token. For more details visit: 71 | https://www.openhumans.org/direct-sharing/oauth2-setup/#setup-oauth2-authorization 72 | 73 | :param client_id: This field is the client id of user. 74 | :param client_secret: This field is the client secret of user. 75 | :param redirect_uri: This is the user redirect uri. 76 | :param base_url: It is this URL `https://www.openhumans.org` 77 | :param code: This field is used to obtain access_token for the first time. 78 | It's default value is none. 79 | :param refresh_token: This field is used to obtain a new access_token when 80 | the token expires. 81 | """ 82 | if not (code or refresh_token) or (code and refresh_token): 83 | raise ValueError("Either code or refresh_token must be specified.") 84 | if code: 85 | data = { 86 | 'grant_type': 'authorization_code', 87 | 'redirect_uri': redirect_uri, 88 | 'code': code, 89 | } 90 | elif refresh_token: 91 | data = { 92 | 'grant_type': 'refresh_token', 93 | 'refresh_token': refresh_token, 94 | } 95 | token_url = urlparse.urljoin(base_url, '/oauth2/token/') 96 | req = requests.post( 97 | token_url, data=data, 98 | auth=requests.auth.HTTPBasicAuth(client_id, client_secret)) 99 | handle_error(req, 200) 100 | data = req.json() 101 | return data 102 | 103 | 104 | def get_page(url): 105 | """ 106 | Get a single page of results. 107 | 108 | :param url: This field is the url from which data will be requested. 109 | """ 110 | response = requests.get(url) 111 | handle_error(response, 200) 112 | data = response.json() 113 | return data 114 | 115 | 116 | def get_all_results(starting_page): 117 | """ 118 | Given starting API query for Open Humans, iterate to get all results. 119 | 120 | :param starting page: This field is the first page, starting from which 121 | results will be obtained. 122 | """ 123 | logging.info('Retrieving all results for {}'.format(starting_page)) 124 | page = starting_page 125 | results = [] 126 | 127 | while True: 128 | logging.debug('Getting data from: {}'.format(page)) 129 | data = get_page(page) 130 | logging.debug('JSON data: {}'.format(data)) 131 | results = results + data['results'] 132 | 133 | if data['next']: 134 | page = data['next'] 135 | else: 136 | break 137 | 138 | return results 139 | 140 | 141 | def exchange_oauth2_member(access_token, base_url=OH_BASE_URL, 142 | all_files=True): 143 | """ 144 | Returns data for a specific user, including shared data files. 145 | 146 | :param access_token: This field is the user specific access_token. 147 | :param base_url: It is this URL `https://www.openhumans.org`. 148 | """ 149 | url = urlparse.urljoin( 150 | base_url, 151 | '/api/direct-sharing/project/exchange-member/?{}'.format( 152 | urlparse.urlencode({'access_token': access_token}))) 153 | member_data = get_page(url) 154 | 155 | returned = member_data.copy() 156 | 157 | # Get all file data if all_files is True. 158 | if all_files: 159 | while member_data['next']: 160 | member_data = get_page(member_data['next']) 161 | returned['data'] = returned['data'] + member_data['data'] 162 | 163 | logging.debug('JSON data: {}'.format(returned)) 164 | return returned 165 | 166 | def delete_file(access_token, project_member_id=None, base_url=OH_BASE_URL, 167 | file_basename=None, file_id=None, all_files=False): 168 | """ 169 | Delete project member files by file_basename, file_id, or all_files. To 170 | learn more about Open Humans OAuth2 projects, go to: 171 | https://www.openhumans.org/direct-sharing/oauth2-features/. 172 | 173 | :param access_token: This field is user specific access_token. 174 | :param project_member_id: This field is the project member id of user. It's 175 | default value is None. 176 | :param base_url: It is this URL `https://www.openhumans.org`. 177 | :param file_basename: This field is the name of the file to delete for the 178 | particular user for the particular project. 179 | :param file_id: This field is the id of the file to delete for the 180 | particular user for the particular project. 181 | :param all_files: This is a boolean field to delete all files for the 182 | particular user for the particular project. 183 | """ 184 | url = urlparse.urljoin( 185 | base_url, '/api/direct-sharing/project/files/delete/?{}'.format( 186 | urlparse.urlencode({'access_token': access_token}))) 187 | if not(project_member_id): 188 | response = exchange_oauth2_member(access_token, base_url=base_url) 189 | project_member_id = response['project_member_id'] 190 | data = {'project_member_id': project_member_id} 191 | if file_basename and not (file_id or all_files): 192 | data['file_basename'] = file_basename 193 | elif file_id and not (file_basename or all_files): 194 | data['file_id'] = file_id 195 | elif all_files and not (file_id or file_basename): 196 | data['all_files'] = True 197 | else: 198 | raise ValueError( 199 | "One (and only one) of the following must be specified: " 200 | "file_basename, file_id, or all_files is set to True.") 201 | response = requests.post(url, data=data) 202 | handle_error(response, 200) 203 | return response 204 | 205 | 206 | # Alternate names for the same functions. 207 | def delete_files(*args, **kwargs): 208 | """ 209 | Alternate name for the :func:`delete_file`. 210 | """ 211 | return delete_file(*args, **kwargs) 212 | 213 | 214 | def message(subject, message, access_token, all_members=False, 215 | project_member_ids=None, base_url=OH_BASE_URL): 216 | """ 217 | Send an email to individual users or in bulk. To learn more about Open 218 | Humans OAuth2 projects, go to: 219 | https://www.openhumans.org/direct-sharing/oauth2-features/ 220 | 221 | :param subject: This field is the subject of the email. 222 | :param message: This field is the body of the email. 223 | :param access_token: This is user specific access token/master token. 224 | :param all_members: This is a boolean field to send email to all members of 225 | the project. 226 | :param project_member_ids: This field is the list of project_member_id. 227 | :param base_url: It is this URL `https://www.openhumans.org`. 228 | """ 229 | url = urlparse.urljoin( 230 | base_url, '/api/direct-sharing/project/message/?{}'.format( 231 | urlparse.urlencode({'access_token': access_token}))) 232 | if not(all_members) and not(project_member_ids): 233 | response = requests.post(url, data={'subject': subject, 234 | 'message': message}) 235 | handle_error(response, 200) 236 | return response 237 | elif all_members and project_member_ids: 238 | raise ValueError( 239 | "One (and only one) of the following must be specified: " 240 | "project_members_id or all_members is set to True.") 241 | else: 242 | r = requests.post(url, data={'all_members': all_members, 243 | 'project_member_ids': project_member_ids, 244 | 'subject': subject, 245 | 'message': message}) 246 | handle_error(r, 200) 247 | return r 248 | 249 | 250 | def _exceeds_size(filesize, max_bytes, file_identifier): 251 | if int(filesize) > max_bytes: 252 | logging.info('Skipping {}, {} > {}'.format( 253 | file_identifier, format_size(filesize), format_size(max_bytes))) 254 | return True 255 | return False 256 | 257 | 258 | def handle_error(r, expected_code): 259 | """ 260 | Helper function to match reponse of a request to the expected status 261 | code 262 | 263 | :param r: This field is the response of request. 264 | :param expected_code: This field is the expected status code for the 265 | function. 266 | """ 267 | code = r.status_code 268 | if code != expected_code: 269 | info = 'API response status code {}:\n{}'.format(code, r.content) 270 | raise Exception(info) 271 | 272 | 273 | def upload_stream(stream, filename, metadata, access_token, datatypes=None, 274 | base_url=OH_BASE_URL, remote_file_info=None, 275 | project_member_id=None, max_bytes=MAX_FILE_DEFAULT, 276 | file_identifier=None): 277 | """ 278 | Upload a file object using the "direct upload" feature, which uploads to 279 | an S3 bucket URL provided by the Open Humans API. To learn more about this 280 | API endpoint see: 281 | * https://www.openhumans.org/direct-sharing/on-site-data-upload/ 282 | * https://www.openhumans.org/direct-sharing/oauth2-data-upload/ 283 | 284 | :param stream: This field is the stream (or file object) to be 285 | uploaded. 286 | :param metadata: This field is the metadata associated with the file. 287 | Description and tags are compulsory fields of metadata. 288 | :param access_token: This is user specific access token/master token. 289 | :param base_url: It is this URL `https://www.openhumans.org`. 290 | :param remote_file_info: This field is for for checking if a file with 291 | matching name and file size already exists. Its default value is none. 292 | :param project_member_id: This field is the list of project member id of 293 | all members of a project. Its default value is None. 294 | :param max_bytes: This field is the maximum file size a user can upload. 295 | Its default value is 128m. 296 | :param max_bytes: If provided, this is used in logging output. Its default 297 | value is None (in which case, filename is used). 298 | """ 299 | if not file_identifier: 300 | file_identifier = filename 301 | 302 | # Determine a stream's size using seek. 303 | # f is a file-like object. 304 | old_position = stream.tell() 305 | stream.seek(0, os.SEEK_END) 306 | filesize = stream.tell() 307 | stream.seek(old_position, os.SEEK_SET) 308 | if filesize == 0: 309 | raise Exception('The submitted file is empty.') 310 | 311 | # Check size, and possibly remote file match. 312 | if _exceeds_size(filesize, max_bytes, file_identifier): 313 | raise ValueError("Maximum file size exceeded") 314 | if remote_file_info: 315 | response = requests.get(remote_file_info['download_url'], stream=True) 316 | remote_size = int(response.headers['Content-Length']) 317 | if remote_size == filesize: 318 | info_msg = ('Skipping {}, remote exists with matching ' 319 | 'file size'.format(file_identifier)) 320 | logging.info(info_msg) 321 | return(info_msg) 322 | 323 | url = urlparse.urljoin( 324 | base_url, 325 | '/api/direct-sharing/project/files/upload/direct/?{}'.format( 326 | urlparse.urlencode({'access_token': access_token}))) 327 | 328 | if not(project_member_id): 329 | response = exchange_oauth2_member(access_token, base_url=base_url) 330 | project_member_id = response['project_member_id'] 331 | 332 | data = {'project_member_id': project_member_id, 333 | 'metadata': json.dumps(metadata), 334 | 'filename': filename} 335 | if datatypes: 336 | data['datatypes'] = json.dumps(datatypes) 337 | r1 = requests.post(url, data=data) 338 | handle_error(r1, 201) 339 | r2 = requests.put(url=r1.json()['url'], data=stream) 340 | handle_error(r2, 200) 341 | done = urlparse.urljoin( 342 | base_url, 343 | '/api/direct-sharing/project/files/upload/complete/?{}'.format( 344 | urlparse.urlencode({'access_token': access_token}))) 345 | 346 | r3 = requests.post(done, data={'project_member_id': project_member_id, 347 | 'file_id': r1.json()['id']}) 348 | handle_error(r3, 200) 349 | logging.info('Upload complete: {}'.format(file_identifier)) 350 | return r3 351 | 352 | 353 | def upload_file(target_filepath, metadata, access_token, datatypes=None, 354 | base_url=OH_BASE_URL, remote_file_info=None, 355 | project_member_id=None, max_bytes=MAX_FILE_DEFAULT): 356 | """ 357 | Upload a file from a local filepath using the "direct upload" API. 358 | To learn more about this API endpoint see: 359 | * https://www.openhumans.org/direct-sharing/on-site-data-upload/ 360 | * https://www.openhumans.org/direct-sharing/oauth2-data-upload/ 361 | 362 | :param target_filepath: This field is the filepath of the file to be 363 | uploaded 364 | :param metadata: This field is a python dictionary with keys filename, 365 | description and tags for single user upload and filename, 366 | project member id, description and tags for multiple user upload. 367 | :param access_token: This is user specific access token/master token. 368 | :param base_url: It is this URL `https://www.openhumans.org`. 369 | :param remote_file_info: This field is for for checking if a file with 370 | matching name and file size already exists. Its default value is none. 371 | :param project_member_id: This field is the list of project member id of 372 | all members of a project. Its default value is None. 373 | :param max_bytes: This field is the maximum file size a user can upload. 374 | It's default value is 128m. 375 | """ 376 | with open(target_filepath, 'rb') as stream: 377 | filename = os.path.basename(target_filepath) 378 | return upload_stream( 379 | stream=stream, 380 | filename=filename, 381 | metadata=metadata, 382 | access_token=access_token, 383 | datatypes=datatypes, 384 | base_url=base_url, 385 | remote_file_info=remote_file_info, 386 | project_member_id=project_member_id, 387 | max_bytes=max_bytes, 388 | file_identifier=target_filepath) 389 | 390 | 391 | def upload_aws(target_filepath, metadata, access_token, base_url=OH_BASE_URL, 392 | remote_file_info=None, project_member_id=None, 393 | max_bytes=MAX_FILE_DEFAULT): 394 | """ 395 | Upload a file from a local filepath using the "direct upload" API. 396 | Equivalent to upload_file. To learn more about this API endpoint see: 397 | * https://www.openhumans.org/direct-sharing/on-site-data-upload/ 398 | * https://www.openhumans.org/direct-sharing/oauth2-data-upload/ 399 | 400 | :param target_filepath: This field is the filepath of the file to be 401 | uploaded 402 | :param metadata: This field is the metadata associated with the file. 403 | Description and tags are compulsory fields of metadata. 404 | :param access_token: This is user specific access token/master token. 405 | :param base_url: It is this URL `https://www.openhumans.org`. 406 | :param remote_file_info: This field is for for checking if a file with 407 | matching name and file size already exists. Its default value is none. 408 | :param project_member_id: This field is the list of project member id of 409 | all members of a project. Its default value is None. 410 | :param max_bytes: This field is the maximum file size a user can upload. 411 | It's default value is 128m. 412 | """ 413 | return upload_file(target_filepath, metadata, access_token, base_url, 414 | remote_file_info, project_member_id, max_bytes) 415 | -------------------------------------------------------------------------------- /ohapi/tests/test_api.py: -------------------------------------------------------------------------------- 1 | import io 2 | from unittest import TestCase 3 | 4 | import pytest 5 | import vcr 6 | 7 | from ohapi.api import ( 8 | SettingsError, oauth2_auth_url, oauth2_token_exchange, 9 | get_page, message, delete_file, upload_file, upload_stream) 10 | 11 | parameter_defaults = { 12 | 'CLIENT_ID_VALID': 'validclientid', 13 | 'CLIENT_SECRET_VALID': 'validclientsecret', 14 | 'CODE_VALID': 'validcode', 15 | 'REFRESH_TOKEN_VALID': 'validrefreshtoken', 16 | 'CLIENT_ID_INVALID': 'invalidclientid', 17 | 'CLIENT_SECRET_INVALID': 'invalidclientsecret', 18 | 'CODE_INVALID': 'invalidcode', 19 | 'REFRESH_TOKEN_INVALID': 'invalidrefreshtoken', 20 | 'REDIRECT_URI': 'http://127.0.0.1:5000/authorize_openhumans/', 21 | 'ACCESS_TOKEN': 'accesstoken', 22 | 'ACCESS_TOKEN_EXPIRED': 'accesstokenexpired', 23 | 'ACCESS_TOKEN_INVALID': 'accesstokeninvalid', 24 | 'MASTER_ACCESS_TOKEN': 'masteraccesstoken', 25 | 'INVALID_PMI1': 'invalidprojectmemberid1', 26 | 'INVALID_PMI2': 'invalidprojectmemberid2', 27 | 'VALID_PMI1': 'validprojectmemberid1', 28 | 'VALID_PMI2': 'validprojectmemberid2', 29 | 'SUBJECT': 'testsubject', 30 | 'MESSAGE': 'testmessage', 31 | 'REMOTE_FILE_INFO': {'download_url': 'https://valid_url/'}, 32 | 'TARGET_FILEPATH': 'testing_extras/lorem_ipsum.txt', 33 | 'TARGET_FILEPATH2': 'testing_extras/lorem_ipsum_partial.txt', 34 | 'TARGET_FILEPATH_EMPTY': 'testing_extras/empty_file.txt', 35 | 'FILE_METADATA': {'tags': ['text'], 'description': 'Lorem ipsum text'}, 36 | 'FILE_METADATA_INVALID': {}, 37 | 'FILE_METADATA_INVALID_WITH_DESC': {'description': 'Lorem ipsum text'}, 38 | 'MAX_BYTES': 'maxbytes' 39 | } 40 | 41 | """ 42 | _config_params_api.py is not usually present. You can create this to use valid 43 | codes and tokens if you wish to record new cassettes. If present, this file is 44 | used to overwrite `parameter_defaults` with the (hopefully valid, but secret) 45 | items in the file. DO NOT COMMIT IT TO GIT! 46 | 47 | To get started, do: 48 | cp _config_params_api.py.example _config_params_api.py 49 | 50 | Edit _config_params_api.py to define valid secret codes, tokens, etc. 51 | 52 | Run a specific function to (re)create an associated cassette, e.g.: 53 | pytest ohapi/tests/test_api.py::APITest::test_oauth2_token_exchange__valid_code 54 | 55 | (This only makes a new cassette if one doesn't already exist!) 56 | """ 57 | try: 58 | from _config_params_api import params 59 | for param in params: 60 | parameter_defaults[param] = params[param] 61 | except ImportError: 62 | pass 63 | 64 | for param in parameter_defaults: 65 | locals()[param] = parameter_defaults[param] 66 | 67 | 68 | FILTERSET = [('access_token', 'ACCESSTOKEN'), ('client_id', 'CLIENTID'), 69 | ('client_secret', 'CLIENTSECRET'), ('code', 'CODE'), 70 | ('refresh_token', 'REFRESHTOKEN'), 71 | ('invalid_access_token', 'INVALIDACCESSTOKEN'), 72 | ('project_member_id', 'PROJECTMEMBERID'), 73 | ('file_id', 'FILEID')] 74 | 75 | 76 | my_vcr = vcr.VCR(path_transformer=vcr.VCR.ensure_suffix('.yaml'), 77 | cassette_library_dir='ohapi/cassettes', 78 | filter_headers=[('Authorization', 'XXXXXXXX')], 79 | filter_query_parameters=FILTERSET, 80 | filter_post_data_parameters=FILTERSET,) 81 | 82 | 83 | class APITestOAuthTokenExchange(TestCase): 84 | """ 85 | Tests for :func:`oauth2_auth_url`. 86 | """ 87 | 88 | def setUp(self): 89 | pass 90 | 91 | def test_oauth2_auth_url__no_client_id(self): 92 | with pytest.raises(SettingsError): 93 | oauth2_auth_url() 94 | 95 | def test_oauth2_auth_url__with_client_id(self): 96 | auth_url = oauth2_auth_url(client_id='abcd1234') 97 | assert auth_url == ( 98 | 'https://www.openhumans.org/direct-sharing/projects/' 99 | 'oauth2/authorize/?client_id=abcd1234&response_type=code') 100 | 101 | def test_oauth2_auth_url__with_client_id_and_redirect_uri(self): 102 | auth_url = oauth2_auth_url(client_id='abcd1234', 103 | redirect_uri='http://127.0.0.1:5000/auth/') 104 | assert auth_url == ( 105 | 'https://www.openhumans.org/direct-sharing/projects/' 106 | 'oauth2/authorize/?client_id=abcd1234&response_type=code' 107 | '&redirect_uri=http%3A%2F%2F127.0.0.1%3A5000%2Fauth%2F') 108 | 109 | @my_vcr.use_cassette() 110 | def test_oauth2_token_exchange__valid_code(self): 111 | data = oauth2_token_exchange( 112 | code=CODE_VALID, client_id=CLIENT_ID_VALID, 113 | client_secret=CLIENT_SECRET_VALID, redirect_uri=REDIRECT_URI) 114 | assert data == { 115 | 'access_token': 'returnedaccesstoken', 116 | 'expires_in': 36000, 117 | 'refresh_token': 'returnedrefreshtoken', 118 | 'scope': 'american-gut read wildlife open-humans write ' 119 | 'pgp go-viral', 120 | 'token_type': 'Bearer'} 121 | 122 | @my_vcr.use_cassette() 123 | def test_oauth2_token_exchange__invalid_code(self): 124 | with self.assertRaises(Exception): 125 | data = oauth2_token_exchange( 126 | code=CODE_VALID, client_id=CLIENT_ID_VALID, 127 | client_secret=CLIENT_SECRET_VALID, redirect_uri=REDIRECT_URI) 128 | 129 | @my_vcr.use_cassette() 130 | def test_oauth2_token_exchange__invalid_client(self): 131 | with self.assertRaises(Exception): 132 | data = oauth2_token_exchange( 133 | code=CODE_INVALID, client_id=CLIENT_ID_INVALID, 134 | client_secret=CLIENT_SECRET_VALID, redirect_uri=REDIRECT_URI) 135 | 136 | @my_vcr.use_cassette() 137 | def test_oauth2_token_exchange__invalid_secret(self): 138 | with self.assertRaises(Exception): 139 | data = oauth2_token_exchange( 140 | code=CODE_VALID, client_id=CLIENT_ID_VALID, 141 | client_secret=CLIENT_SECRET_INVALID, 142 | redirect_uri=REDIRECT_URI) 143 | 144 | @my_vcr.use_cassette() 145 | def test_oauth2_token_exchange__valid_refresh(self): 146 | data = oauth2_token_exchange( 147 | refresh_token=REFRESH_TOKEN_VALID, client_id=CLIENT_ID_VALID, 148 | client_secret=CLIENT_SECRET_VALID, redirect_uri=REDIRECT_URI) 149 | assert data == { 150 | 'access_token': 'newaccesstoken', 151 | 'expires_in': 36000, 152 | 'refresh_token': 'newrefreshtoken', 153 | 'scope': 'american-gut read wildlife open-humans write ' 154 | 'pgp go-viral', 155 | 'token_type': 'Bearer'} 156 | 157 | @my_vcr.use_cassette() 158 | def test_oauth2_token_exchange__invalid_refresh(self): 159 | with self.assertRaises(Exception): 160 | data = oauth2_token_exchange( 161 | refresh_token=REFRESH_TOKEN_INVALID, 162 | client_id=CLIENT_ID_VALID, 163 | client_secret=CLIENT_SECRET_VALID, 164 | redirect_uri=REDIRECT_URI) 165 | 166 | 167 | class APITestGetPage(TestCase): 168 | """ 169 | Tests for :func:`get_page`. 170 | """ 171 | 172 | def setUp(self): 173 | pass 174 | 175 | @my_vcr.use_cassette() 176 | def test_get_page_with_results(self): 177 | url = ('https://www.openhumans.org/api/direct-sharing/project/' 178 | 'exchange-member/?' 179 | 'access_token={}'.format(ACCESS_TOKEN)) 180 | response = get_page(url) 181 | self.assertEqual(response['project_member_id'], 'PMI') 182 | self.assertEqual(response['message_permission'], True) 183 | self.assertEqual(response['data'], []) 184 | self.assertEqual(response['username'], 'test_user') 185 | self.assertEqual(response['sources_shared'], []) 186 | self.assertEqual(response['created'], 'created_date_time') 187 | 188 | @my_vcr.use_cassette() 189 | def test_get_page_invalid_access_token(self): 190 | try: 191 | url = ('https://www.openhumans.org/api/direct-sharing/project/' 192 | 'exchange-member/?access_token={}'.format("invalid_token")) 193 | self.assertRaises(Exception, get_page, url) 194 | except Exception: 195 | pass 196 | 197 | 198 | class APITestMessage(TestCase): 199 | """ 200 | Tests for :func:`message`. 201 | """ 202 | 203 | def setUp(self): 204 | pass 205 | 206 | @my_vcr.use_cassette() 207 | def test_message_valid_access_token(self): 208 | response = message(subject=SUBJECT, message=MESSAGE, 209 | access_token=ACCESS_TOKEN) 210 | self.assertEqual(response.status_code, 200) 211 | 212 | @my_vcr.use_cassette() 213 | def test_message_expired_access_token(self): 214 | with self.assertRaises(Exception): 215 | response = message(subject=SUBJECT, message=MESSAGE, 216 | access_token=ACCESS_TOKEN_EXPIRED) 217 | assert response.json() == {"detail": "Expired token."} 218 | 219 | @my_vcr.use_cassette() 220 | def test_message_invalid_access_token(self): 221 | with self.assertRaises(Exception): 222 | response = message(subject=SUBJECT, message=MESSAGE, 223 | access_token=ACCESS_TOKEN_INVALID) 224 | assert response.json() == {"detail": "Invalid token."} 225 | 226 | @my_vcr.use_cassette() 227 | def test_message_all_members_true_project_member_id_none(self): 228 | response = message(all_members=True, subject=SUBJECT, message=MESSAGE, 229 | access_token=ACCESS_TOKEN) 230 | self.assertEqual(response.status_code, 200) 231 | 232 | @my_vcr.use_cassette() 233 | def test_message_all_members_true_project_member_id_not_none(self): 234 | self.assertRaises(Exception, message, all_members=True, 235 | project_member_ids=['abcdef', 'sdf'], 236 | subject=SUBJECT, message=MESSAGE, 237 | access_token=ACCESS_TOKEN) 238 | 239 | @my_vcr.use_cassette() 240 | def test_message_all_members_false_projectmemberid_has_invalid_char(self): 241 | with self.assertRaises(Exception): 242 | response = message(project_member_ids=['abcdef1', 'test'], 243 | subject=SUBJECT, message=MESSAGE, 244 | access_token=MASTER_ACCESS_TOKEN) 245 | assert response.json() == {"errors": 246 | {"project_member_ids": 247 | ["Project member IDs are always 8" + 248 | " digits long."]}} 249 | 250 | @my_vcr.use_cassette() 251 | def test_message_all_members_false_projectmemberid_has_invalid_digit(self): 252 | with self.assertRaises(Exception): 253 | response = message(project_member_ids=[INVALID_PMI1, 254 | INVALID_PMI2], 255 | subject=SUBJECT, message=MESSAGE, 256 | access_token=MASTER_ACCESS_TOKEN) 257 | assert response.json() == {"errors": 258 | {"project_member_ids": 259 | ["Invalid project member ID(s):" + 260 | " invalidPMI2"]}} 261 | 262 | @my_vcr.use_cassette() 263 | def test_message_all_members_false_project_member_id_not_none_valid(self): 264 | response = message(project_member_ids=[VALID_PMI1, VALID_PMI2], 265 | subject=SUBJECT, message=MESSAGE, 266 | access_token=ACCESS_TOKEN) 267 | self.assertEqual(response.status_code, 200) 268 | 269 | 270 | class APITestDeleteFile(TestCase): 271 | """ 272 | Tests for :func:`delete_file`. 273 | """ 274 | 275 | def setUp(self): 276 | pass 277 | 278 | @my_vcr.use_cassette() 279 | def test_delete_file__invalid_access_token(self): 280 | with self.assertRaises(Exception): 281 | response = delete_file( 282 | access_token=ACCESS_TOKEN_INVALID, 283 | project_member_id='59319749', 284 | all_files=True) 285 | assert response.json() == {"detail": "Invalid token."} 286 | 287 | @my_vcr.use_cassette() 288 | def test_delete_file_project_member_id_given(self): 289 | response = delete_file(access_token=ACCESS_TOKEN, 290 | project_member_id='59319749', all_files=True) 291 | self.assertEqual(response.status_code, 200) 292 | 293 | @my_vcr.use_cassette() 294 | def test_delete_file_project_member_id_invalid(self): 295 | with self.assertRaises(Exception): 296 | response = delete_file(access_token=ACCESS_TOKEN, all_files=True, 297 | project_member_id='1234') 298 | self.assertEqual(response.status_code, 400) 299 | 300 | @my_vcr.use_cassette() 301 | def test_delete_file__expired_access_token(self): 302 | with self.assertRaises(Exception): 303 | response = delete_file(access_token=ACCESS_TOKEN_EXPIRED, 304 | all_files=True, 305 | project_member_id='59319749') 306 | assert response.json() == {"detail": "Expired token."} 307 | 308 | @my_vcr.use_cassette() 309 | def test_delete_file__valid_access_token(self): 310 | response = delete_file( 311 | access_token=ACCESS_TOKEN, project_member_id='59319749', 312 | all_files=True) 313 | self.assertEqual(response.status_code, 200) 314 | 315 | 316 | class APITestUpload(TestCase): 317 | """ 318 | Tests for :func:`upload_file`. 319 | 320 | File and stream upload testing use "lorem_ipsum.txt" and other files 321 | in the testing_extra directory. 322 | """ 323 | 324 | def setUp(self): 325 | pass 326 | 327 | @my_vcr.use_cassette() 328 | def test_upload_valid_file_valid_access_token(self): 329 | response = upload_file( 330 | target_filepath=TARGET_FILEPATH, 331 | metadata=FILE_METADATA, 332 | access_token=ACCESS_TOKEN, 333 | project_member_id=VALID_PMI1) 334 | self.assertEqual(response.status_code, 200) 335 | assert response.json() == {'size': 446, 'status': 'ok'} 336 | 337 | @my_vcr.use_cassette() 338 | def test_upload_large_file_valid_access_token(self): 339 | self.assertRaisesRegexp( 340 | Exception, 'Maximum file size exceeded', upload_file, 341 | target_filepath=TARGET_FILEPATH, 342 | metadata=FILE_METADATA, 343 | access_token=ACCESS_TOKEN, 344 | project_member_id=VALID_PMI1, 345 | max_bytes=0) 346 | 347 | @my_vcr.use_cassette() 348 | def test_upload_file_invalid_access_token(self): 349 | self.assertRaises( 350 | Exception, 'Invalid token', upload_file, 351 | target_filepath=TARGET_FILEPATH, 352 | metadata=FILE_METADATA, 353 | access_token=ACCESS_TOKEN_INVALID, 354 | project_member_id=VALID_PMI1) 355 | 356 | @my_vcr.use_cassette() 357 | def test_upload_file_expired_access_token(self): 358 | self.assertRaisesRegexp( 359 | Exception, 'Expired token', upload_file, 360 | target_filepath=TARGET_FILEPATH, 361 | metadata=FILE_METADATA, 362 | access_token=ACCESS_TOKEN_EXPIRED, 363 | project_member_id=VALID_PMI1) 364 | 365 | @my_vcr.use_cassette() 366 | def test_upload_file_invalid_metadata_with_description(self): 367 | self.assertRaisesRegexp( 368 | Exception, 'tags.+ is a required field', upload_file, 369 | target_filepath=TARGET_FILEPATH, 370 | metadata=FILE_METADATA_INVALID_WITH_DESC, 371 | access_token=ACCESS_TOKEN, 372 | project_member_id=VALID_PMI1) 373 | 374 | @my_vcr.use_cassette() 375 | def test_upload_file_invalid_metadata_without_description(self): 376 | self.assertRaisesRegexp( 377 | Exception, 'description.+ is a required field of the metadata', 378 | upload_file, 379 | target_filepath=TARGET_FILEPATH, 380 | metadata=FILE_METADATA_INVALID, 381 | access_token=ACCESS_TOKEN, 382 | project_member_id=VALID_PMI1) 383 | 384 | @my_vcr.use_cassette() 385 | def test_upload_file_empty(self): 386 | self.assertRaisesRegexp( 387 | Exception, 'The submitted file is empty.', 388 | upload_file, 389 | target_filepath=TARGET_FILEPATH_EMPTY, 390 | metadata=FILE_METADATA, 391 | access_token=ACCESS_TOKEN, 392 | project_member_id=VALID_PMI1) 393 | 394 | def test_upload_file_remote_info_not_none_valid(self): 395 | """ 396 | Test assumes remote_file_info['download_url'] matches 'lorum_ipsum.txt' 397 | """ 398 | with my_vcr.use_cassette('ohapi/cassettes/test_upload_file_' + 399 | 'remote_info_not_none_valid.yaml') as cass: 400 | upload_file(target_filepath=TARGET_FILEPATH, 401 | metadata=FILE_METADATA, 402 | access_token=ACCESS_TOKEN, 403 | project_member_id=VALID_PMI1, 404 | remote_file_info=REMOTE_FILE_INFO) 405 | self.assertEqual(cass.responses[0][ 406 | "status"]["code"], 200) 407 | self.assertEqual(cass.responses[0][ 408 | "headers"]["Content-Length"], ['446']) 409 | 410 | @my_vcr.use_cassette() 411 | def test_upload_file_remote_info_not_none_invalid_access_token(self): 412 | """ 413 | Test assumes remote_file_info['download_url'] matches 'lorum_ipsum.txt' 414 | """ 415 | # Note: alternate file needed to trigger an attempted upload. 416 | self.assertRaisesRegexp( 417 | Exception, 'Invalid token', upload_file, 418 | target_filepath=TARGET_FILEPATH2, 419 | metadata=FILE_METADATA, 420 | access_token=ACCESS_TOKEN_INVALID, 421 | project_member_id=VALID_PMI1, 422 | remote_file_info=REMOTE_FILE_INFO) 423 | 424 | @my_vcr.use_cassette() 425 | def test_upload_file_remote_info_not_none_expired_access_token(self): 426 | # Note: alternate file needed to trigger an attempted upload. 427 | self.assertRaisesRegexp( 428 | Exception, 'Expired token', upload_file, 429 | target_filepath=TARGET_FILEPATH2, 430 | metadata=FILE_METADATA, 431 | access_token=ACCESS_TOKEN_EXPIRED, 432 | project_member_id=VALID_PMI1, 433 | remote_file_info=REMOTE_FILE_INFO) 434 | 435 | @my_vcr.use_cassette() 436 | def test_upload_file_empty_remote_info_not_none(self): 437 | self.assertRaisesRegexp( 438 | Exception, 'The submitted file is empty.', upload_file, 439 | target_filepath=TARGET_FILEPATH_EMPTY, 440 | metadata=FILE_METADATA, 441 | access_token=ACCESS_TOKEN, 442 | project_member_id=VALID_PMI1, 443 | remote_file_info=REMOTE_FILE_INFO) 444 | 445 | @my_vcr.use_cassette() 446 | def test_upload_file_remote_info_not_none_matching_file_size(self): 447 | result = upload_file( 448 | target_filepath=TARGET_FILEPATH, 449 | metadata=FILE_METADATA, 450 | access_token=ACCESS_TOKEN, 451 | project_member_id=VALID_PMI1, 452 | remote_file_info=REMOTE_FILE_INFO) 453 | self.assertRegexpMatches( 454 | result, 'remote exists with matching file size') 455 | 456 | @my_vcr.use_cassette() 457 | def test_upload_file_remote_info_not_none_invalid_metadata_with_desc(self): 458 | # Note: alternate file needed to trigger an attempted upload. 459 | self.assertRaisesRegexp( 460 | Exception, 'tags.+ is a required field of the metadata', 461 | upload_file, 462 | target_filepath=TARGET_FILEPATH2, 463 | metadata=FILE_METADATA_INVALID_WITH_DESC, 464 | access_token=ACCESS_TOKEN, 465 | project_member_id=VALID_PMI1, 466 | remote_file_info=REMOTE_FILE_INFO) 467 | 468 | @my_vcr.use_cassette() 469 | def test_upload_file_remote_info_not_none_invalid_metadata(self): 470 | self.assertRaisesRegexp( 471 | Exception, 'description.+ is a required field of the metadata', 472 | upload_file, 473 | target_filepath=TARGET_FILEPATH2, 474 | metadata=FILE_METADATA_INVALID, 475 | access_token=ACCESS_TOKEN, 476 | project_member_id=VALID_PMI1, 477 | remote_file_info=REMOTE_FILE_INFO) 478 | 479 | @my_vcr.use_cassette() 480 | def test_upload_stream_valid(self): 481 | stream = None 482 | with open(TARGET_FILEPATH, 'rb') as testfile: 483 | testdata = testfile.read() 484 | stream = io.BytesIO(testdata) 485 | response = upload_stream( 486 | stream=stream, 487 | filename=TARGET_FILEPATH.split('/')[-1], 488 | metadata=FILE_METADATA, 489 | access_token=ACCESS_TOKEN, 490 | project_member_id=VALID_PMI1) 491 | self.assertEqual(response.status_code, 200) 492 | assert response.json() == {'size': 446, 'status': 'ok'} 493 | -------------------------------------------------------------------------------- /ohapi/utils_fs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions to sync and work with Open Humans data in a local filesystem. 3 | """ 4 | import csv 5 | import hashlib 6 | import logging 7 | import os 8 | import re 9 | 10 | import arrow 11 | from humanfriendly import format_size, parse_size 12 | from .api import _exceeds_size 13 | import requests 14 | 15 | 16 | MAX_FILE_DEFAULT = parse_size('128m') 17 | 18 | 19 | def strip_zip_suffix(filename): 20 | """ 21 | Helper function to strip suffix from filename. 22 | 23 | :param filename: This field is the name of file. 24 | """ 25 | if filename.endswith('.gz'): 26 | return filename[:-3] 27 | elif filename.endswith('.bz2'): 28 | return filename[:-4] 29 | else: 30 | return filename 31 | 32 | 33 | def guess_tags(filename): 34 | """ 35 | Function to get potential tags for files using the file names. 36 | 37 | :param filename: This field is the name of file. 38 | """ 39 | tags = [] 40 | stripped_filename = strip_zip_suffix(filename) 41 | if stripped_filename.endswith('.vcf'): 42 | tags.append('vcf') 43 | if stripped_filename.endswith('.json'): 44 | tags.append('json') 45 | if stripped_filename.endswith('.csv'): 46 | tags.append('csv') 47 | return tags 48 | 49 | 50 | def characterize_local_files(filedir, max_bytes=MAX_FILE_DEFAULT): 51 | """ 52 | Collate local file info as preperation for Open Humans upload. 53 | 54 | Note: Files with filesize > max_bytes are not included in returned info. 55 | 56 | :param filedir: This field is target directory to get files from. 57 | :param max_bytes: This field is the maximum file size to consider. Its 58 | default value is 128m. 59 | """ 60 | file_data = {} 61 | logging.info('Characterizing files in {}'.format(filedir)) 62 | for filename in os.listdir(filedir): 63 | filepath = os.path.join(filedir, filename) 64 | file_stats = os.stat(filepath) 65 | creation_date = arrow.get(file_stats.st_ctime).isoformat() 66 | file_size = file_stats.st_size 67 | if file_size <= max_bytes: 68 | file_md5 = hashlib.md5() 69 | with open(filepath, "rb") as f: 70 | for chunk in iter(lambda: f.read(4096), b""): 71 | file_md5.update(chunk) 72 | md5 = file_md5.hexdigest() 73 | file_data[filename] = { 74 | 'tags': guess_tags(filename), 75 | 'description': '', 76 | 'md5': md5, 77 | 'creation_date': creation_date, 78 | } 79 | return file_data 80 | 81 | 82 | def validate_metadata(target_dir, metadata): 83 | """ 84 | Check that the files listed in metadata exactly match files in target dir. 85 | 86 | :param target_dir: This field is the target directory from which to 87 | match metadata 88 | :param metadata: This field contains the metadata to be matched. 89 | """ 90 | if not os.path.isdir(target_dir): 91 | print("Error: " + target_dir + " is not a directory") 92 | return False 93 | file_list = os.listdir(target_dir) 94 | for filename in file_list: 95 | if filename not in metadata: 96 | print("Error: " + filename + " present at" + target_dir + 97 | " not found in metadata file") 98 | return False 99 | for filename in metadata: 100 | if filename not in file_list: 101 | print("Error: " + filename + " present in metadata file " + 102 | " not found on disk at: " + target_dir) 103 | return False 104 | return True 105 | 106 | 107 | def load_metadata_csv_single_user(csv_in, header, tags_idx): 108 | """ 109 | Return the metadata as requested for a single user. 110 | 111 | :param csv_in: This field is the csv file to return metadata from. 112 | :param header: This field contains the headers in the csv file 113 | :param tags_idx: This field contains the index of the tags in the csv 114 | file. 115 | """ 116 | metadata = {} 117 | n_headers = len(header) 118 | for index, row in enumerate(csv_in, 2): 119 | if row[0] == "": 120 | raise ValueError('Error: In row number ' + str(index) + ':' + 121 | ' "filename" must not be empty.') 122 | if row[0] == 'None' and [x == 'NA' for x in row[1:]]: 123 | break 124 | if len(row) != n_headers: 125 | raise ValueError('Error: In row number ' + str(index) + ':' + 126 | ' Number of columns (' + str(len(row)) + 127 | ') doesnt match Number of headings (' + 128 | str(n_headers) + ')') 129 | metadata[row[0]] = { 130 | header[i]: row[i] for i in range(1, len(header)) if 131 | i != tags_idx 132 | } 133 | metadata[row[0]]['tags'] = [t.strip() for t in 134 | row[tags_idx].split(',') if 135 | t.strip()] 136 | return metadata 137 | 138 | 139 | def load_metadata_csv_multi_user(csv_in, header, tags_idx): 140 | """ 141 | Return the metadata as requested for multiple users. 142 | 143 | :param csv_in: This field is the csv file to return metadata from. 144 | :param header: This field contains the headers in the csv file 145 | :param tags_idx: This field contains the index of the tags in the csv 146 | file. 147 | """ 148 | metadata = {} 149 | n_headers = len(header) 150 | for index, row in enumerate(csv_in, 2): 151 | if row[0] == "": 152 | raise ValueError('Error: In row number ' + str(index) + ':' + 153 | ' "project_member_id" must not be empty.') 154 | if row[1] == "": 155 | raise ValueError('Error: In row number ' + str(index) + ':' + 156 | ' "filename" must not be empty.') 157 | if row[0] not in metadata: 158 | metadata[row[0]] = {} 159 | if row[1] == 'None' and all([x == 'NA' for x in row[2:]]): 160 | continue 161 | if len(row) != n_headers: 162 | raise ValueError('Error: In row number ' + str(index) + ':' + 163 | ' Number of columns (' + str(len(row)) + 164 | ') doesnt match Number of headings (' + 165 | str(n_headers) + ')') 166 | metadata[row[0]][row[1]] = { 167 | header[i]: row[i] for i in range(2, len(header)) if 168 | i != tags_idx 169 | } 170 | metadata[row[0]][row[1]]['tags'] = [t.strip() for t in 171 | row[tags_idx].split(',') if 172 | t.strip()] 173 | return metadata 174 | 175 | 176 | def load_metadata_csv(input_filepath): 177 | """ 178 | Return dict of metadata. 179 | 180 | Format is either dict (filenames are keys) or dict-of-dicts (project member 181 | IDs as top level keys, then filenames as keys). 182 | 183 | :param input_filepath: This field is the filepath of the csv file. 184 | """ 185 | with open(input_filepath) as f: 186 | csv_in = csv.reader(f) 187 | header = next(csv_in) 188 | if 'tags' in header: 189 | tags_idx = header.index('tags') 190 | else: 191 | raise ValueError('"tags" is a compulsory column in metadata file.') 192 | if header[0] == 'project_member_id': 193 | if header[1] == 'filename': 194 | metadata = load_metadata_csv_multi_user(csv_in, header, 195 | tags_idx) 196 | else: 197 | raise ValueError('The second column must be "filename"') 198 | elif header[0] == 'filename': 199 | metadata = load_metadata_csv_single_user(csv_in, header, tags_idx) 200 | else: 201 | raise ValueError('Incorrect Formatting of metadata. The first' + 202 | ' column for single user upload should be' + 203 | ' "filename". For multiuser uploads the first ' + 204 | 'column should be "project member id" and the' + 205 | ' second column should be "filename"') 206 | return metadata 207 | 208 | 209 | def print_error(e): 210 | """ 211 | Helper function to print error. 212 | 213 | :param e: This field is the error to be printed. 214 | """ 215 | print(" ".join([str(arg) for arg in e.args])) 216 | 217 | 218 | def validate_date(date, project_member_id, filename): 219 | """ 220 | Check if date is in ISO 8601 format. 221 | 222 | :param date: This field is the date to be checked. 223 | :param project_member_id: This field is the project_member_id corresponding 224 | to the date provided. 225 | :param filename: This field is the filename corresponding to the date 226 | provided. 227 | """ 228 | try: 229 | arrow.get(date) 230 | except Exception: 231 | return False 232 | return True 233 | 234 | 235 | def is_single_file_metadata_valid(file_metadata, project_member_id, filename): 236 | """ 237 | Check if metadata fields like project member id, description, tags, md5 and 238 | creation date are valid for a single file. 239 | 240 | :param file_metadata: This field is metadata of file. 241 | :param project_member_id: This field is the project member id corresponding 242 | to the file metadata provided. 243 | :param filename: This field is the filename corresponding to the file 244 | metadata provided. 245 | """ 246 | if project_member_id is not None: 247 | if not project_member_id.isdigit() or len(project_member_id) != 8: 248 | raise ValueError( 249 | 'Error: for project member id: ', project_member_id, 250 | ' and filename: ', filename, 251 | ' project member id must be of 8 digits from 0 to 9') 252 | if 'description' not in file_metadata: 253 | raise ValueError( 254 | 'Error: for project member id: ', project_member_id, 255 | ' and filename: ', filename, 256 | ' "description" is a required field of the metadata') 257 | 258 | if not isinstance(file_metadata['description'], str): 259 | raise ValueError( 260 | 'Error: for project member id: ', project_member_id, 261 | ' and filename: ', filename, 262 | ' "description" must be a string') 263 | 264 | if 'tags' not in file_metadata: 265 | raise ValueError( 266 | 'Error: for project member id: ', project_member_id, 267 | ' and filename: ', filename, 268 | ' "tags" is a required field of the metadata') 269 | 270 | if not isinstance(file_metadata['tags'], list): 271 | raise ValueError( 272 | 'Error: for project member id: ', project_member_id, 273 | ' and filename: ', filename, 274 | ' "tags" must be an array of strings') 275 | 276 | if 'creation_date' in file_metadata: 277 | if not validate_date(file_metadata['creation_date'], project_member_id, 278 | filename): 279 | raise ValueError( 280 | 'Error: for project member id: ', project_member_id, 281 | ' and filename: ', filename, 282 | ' Dates must be in ISO 8601 format') 283 | 284 | if 'md5' in file_metadata: 285 | if not re.match(r'[a-f0-9]{32}$', file_metadata['md5'], 286 | flags=re.IGNORECASE): 287 | raise ValueError( 288 | 'Error: for project member id: ', project_member_id, 289 | ' and filename: ', filename, 290 | ' Invalid MD5 specified') 291 | 292 | return True 293 | 294 | 295 | def review_metadata_csv_single_user(filedir, metadata, csv_in, n_headers): 296 | """ 297 | Check validity of metadata for single user. 298 | 299 | :param filedir: This field is the filepath of the directory whose csv 300 | has to be made. 301 | :param metadata: This field is the metadata generated from the 302 | load_metadata_csv function. 303 | :param csv_in: This field returns a reader object which iterates over the 304 | csv. 305 | :param n_headers: This field is the number of headers in the csv. 306 | """ 307 | try: 308 | if not validate_metadata(filedir, metadata): 309 | return False 310 | for filename, file_metadata in metadata.items(): 311 | is_single_file_metadata_valid(file_metadata, None, filename) 312 | except ValueError as e: 313 | print_error(e) 314 | return False 315 | return True 316 | 317 | 318 | def validate_subfolders(filedir, metadata): 319 | """ 320 | Check that all folders in the given directory have a corresponding 321 | entry in the metadata file, and vice versa. 322 | 323 | :param filedir: This field is the target directory from which to 324 | match metadata 325 | :param metadata: This field contains the metadata to be matched. 326 | """ 327 | if not os.path.isdir(filedir): 328 | print("Error: " + filedir + " is not a directory") 329 | return False 330 | subfolders = os.listdir(filedir) 331 | for subfolder in subfolders: 332 | if subfolder not in metadata: 333 | print("Error: folder " + subfolder + 334 | " present on disk but not in metadata") 335 | return False 336 | for subfolder in metadata: 337 | if subfolder not in subfolders: 338 | print("Error: folder " + subfolder + 339 | " present in metadata but not on disk") 340 | return False 341 | return True 342 | 343 | 344 | def review_metadata_csv_multi_user(filedir, metadata, csv_in, n_headers): 345 | """ 346 | Check validity of metadata for multi user. 347 | 348 | :param filedir: This field is the filepath of the directory whose csv 349 | has to be made. 350 | :param metadata: This field is the metadata generated from the 351 | load_metadata_csv function. 352 | :param csv_in: This field returns a reader object which iterates over the 353 | csv. 354 | :param n_headers: This field is the number of headers in the csv. 355 | """ 356 | try: 357 | if not validate_subfolders(filedir, metadata): 358 | return False 359 | for project_member_id, member_metadata in metadata.items(): 360 | if not validate_metadata(os.path.join 361 | (filedir, project_member_id), 362 | member_metadata): 363 | return False 364 | for filename, file_metadata in member_metadata.items(): 365 | is_single_file_metadata_valid(file_metadata, project_member_id, 366 | filename) 367 | 368 | except ValueError as e: 369 | print_error(e) 370 | return False 371 | return True 372 | 373 | 374 | def review_metadata_csv(filedir, input_filepath): 375 | """ 376 | Check validity of metadata fields. 377 | 378 | :param filedir: This field is the filepath of the directory whose csv 379 | has to be made. 380 | :param outputfilepath: This field is the file path of the output csv. 381 | :param max_bytes: This field is the maximum file size to consider. Its 382 | default value is 128m. 383 | """ 384 | try: 385 | metadata = load_metadata_csv(input_filepath) 386 | except ValueError as e: 387 | print_error(e) 388 | return False 389 | 390 | with open(input_filepath) as f: 391 | csv_in = csv.reader(f) 392 | header = next(csv_in) 393 | n_headers = len(header) 394 | if header[0] == 'filename': 395 | res = review_metadata_csv_single_user(filedir, metadata, 396 | csv_in, n_headers) 397 | return res 398 | if header[0] == 'project_member_id': 399 | res = review_metadata_csv_multi_user(filedir, metadata, 400 | csv_in, n_headers) 401 | return res 402 | 403 | 404 | def write_metadata_to_filestream(filedir, filestream, 405 | max_bytes=MAX_FILE_DEFAULT): 406 | """ 407 | Make metadata file for all files in a directory(helper function) 408 | 409 | :param filedir: This field is the filepath of the directory whose csv 410 | has to be made. 411 | :param filestream: This field is a stream for writing to the csv. 412 | :param max_bytes: This field is the maximum file size to consider. Its 413 | default value is 128m. 414 | """ 415 | csv_out = csv.writer(filestream) 416 | subdirs = [os.path.join(filedir, i) for i in os.listdir(filedir) if 417 | os.path.isdir(os.path.join(filedir, i))] 418 | if subdirs: 419 | logging.info('Making metadata for subdirs of {}'.format(filedir)) 420 | if not all([re.match('^[0-9]{8}$', os.path.basename(d)) 421 | for d in subdirs]): 422 | raise ValueError("Subdirs not all project member ID format!") 423 | csv_out.writerow(['project_member_id', 'filename', 'tags', 424 | 'description', 'md5', 'creation_date']) 425 | for subdir in subdirs: 426 | file_info = characterize_local_files( 427 | filedir=subdir, max_bytes=max_bytes) 428 | proj_member_id = os.path.basename(subdir) 429 | if not file_info: 430 | csv_out.writerow([proj_member_id, 'None', 431 | 'NA', 'NA', 'NA', 'NA']) 432 | continue 433 | for filename in file_info: 434 | csv_out.writerow([proj_member_id, 435 | filename, 436 | ', '.join(file_info[filename]['tags']), 437 | file_info[filename]['description'], 438 | file_info[filename]['md5'], 439 | file_info[filename]['creation_date'], 440 | ]) 441 | else: 442 | csv_out.writerow(['filename', 'tags', 443 | 'description', 'md5', 'creation_date']) 444 | file_info = characterize_local_files( 445 | filedir=filedir, max_bytes=max_bytes) 446 | for filename in file_info: 447 | csv_out.writerow([filename, 448 | ', '.join(file_info[filename]['tags']), 449 | file_info[filename]['description'], 450 | file_info[filename]['md5'], 451 | file_info[filename]['creation_date'], 452 | ]) 453 | 454 | 455 | def mk_metadata_csv(filedir, outputfilepath, max_bytes=MAX_FILE_DEFAULT): 456 | """ 457 | Make metadata file for all files in a directory. 458 | 459 | :param filedir: This field is the filepath of the directory whose csv 460 | has to be made. 461 | :param outputfilepath: This field is the file path of the output csv. 462 | :param max_bytes: This field is the maximum file size to consider. Its 463 | default value is 128m. 464 | """ 465 | with open(outputfilepath, 'w') as filestream: 466 | write_metadata_to_filestream(filedir, filestream, max_bytes) 467 | 468 | 469 | def download_file(download_url, target_filepath, max_bytes=MAX_FILE_DEFAULT): 470 | """ 471 | Download a file. 472 | 473 | :param download_url: This field is the url from which data will be 474 | downloaded. 475 | :param target_filepath: This field is the path of the file where 476 | data will be downloaded. 477 | :param max_bytes: This field is the maximum file size to download. Its 478 | default value is 128m. 479 | """ 480 | response = requests.get(download_url, stream=True) 481 | size = int(response.headers['Content-Length']) 482 | 483 | if _exceeds_size(size, max_bytes, target_filepath) is True: 484 | return response 485 | 486 | logging.info('Downloading {} ({})'.format( 487 | target_filepath, format_size(size))) 488 | 489 | if os.path.exists(target_filepath): 490 | stat = os.stat(target_filepath) 491 | if stat.st_size == size: 492 | logging.info('Skipping, file exists and is the right ' 493 | 'size: {}'.format(target_filepath)) 494 | return response 495 | else: 496 | logging.info('Replacing, file exists and is the wrong ' 497 | 'size: {}'.format(target_filepath)) 498 | os.remove(target_filepath) 499 | 500 | with open(target_filepath, 'wb') as f: 501 | for chunk in response.iter_content(chunk_size=8192): 502 | if chunk: 503 | f.write(chunk) 504 | 505 | logging.info('Download complete: {}'.format(target_filepath)) 506 | return response 507 | 508 | 509 | def read_id_list(filepath): 510 | """ 511 | Get project member id from a file. 512 | 513 | :param filepath: This field is the path of file to read. 514 | """ 515 | if not filepath: 516 | return None 517 | id_list = [] 518 | with open(filepath) as f: 519 | for line in f: 520 | line = line.rstrip() 521 | if not re.match('^[0-9]{8}$', line): 522 | raise('Each line in whitelist or blacklist is expected ' 523 | 'to contain an eight digit ID, and nothing else.') 524 | else: 525 | id_list.append(line) 526 | return id_list 527 | --------------------------------------------------------------------------------