├── hapiclient ├── test │ ├── __init__.py │ ├── compare.log │ ├── pytest.ini │ ├── data │ │ ├── test_dataset.pkl │ │ └── test_parameter.pkl │ ├── test_datetime2hapitime.py │ ├── test_hapitime_reformat.py │ ├── test_hapitime2datetime.py │ ├── test_hapitime2datetime.log │ ├── test_chunking.py │ ├── compare.py │ ├── test_chunking.log │ └── test_hapi.py ├── __init__.py ├── hapitime.py ├── util.py └── hapi.py ├── _config.yml ├── MANIFEST.in ├── .gitignore ├── CITATION.cff ├── CODE_OF_CONDUCT.md ├── tox.ini ├── .zenodo.json ├── .travis.yml ├── LICENSE.txt ├── setup.py ├── misc └── version.py ├── CHANGES.txt ├── hapi_demo.py ├── README.md └── Makefile /hapiclient/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /hapiclient/test/compare.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-minimal -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt 2 | recursive-include hapiclient/ *.py -------------------------------------------------------------------------------- /hapiclient/test/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | short: Short-running tests 4 | long: Long-running test -------------------------------------------------------------------------------- /hapiclient/test/data/test_dataset.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hapi-server/client-python/HEAD/hapiclient/test/data/test_dataset.pkl -------------------------------------------------------------------------------- /hapiclient/test/data/test_parameter.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hapi-server/client-python/HEAD/hapiclient/test/data/test_parameter.pkl -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | hapi-data/ 2 | *.pyc 3 | tmp/ 4 | misc/tmp/ 5 | .spyproject/ 6 | *~ 7 | .DS_Store 8 | hapiclient.egg-info/ 9 | dist/ 10 | MANIFEST 11 | .pytest_cache/ 12 | .ipynb_checkpoints/ 13 | .vscode/ 14 | .idea/ 15 | anaconda3/ 16 | build/ -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 0.2.1 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - family-names: Robert S. 5 | given-names: Weigel 6 | orcid: https://orcid.org/0000-0002-9521-5228 7 | title: hapi-server/client-python: 8 | version: v0.2.1 9 | date-released: 2021-10-06 10 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | We welcome and encourage contributions to this project in the form of pull requests, submission of issues, and comments on issues. 2 | 3 | Contributors are expected 4 | 5 | 1. to be respectful and constructive; and 6 | 2. to enforce 1. 7 | 8 | This code of conduct applies to all project-related communication that takes place on or at mailing lists, forums, social media, conferences, meetings, and social events. -------------------------------------------------------------------------------- /hapiclient/__init__.py: -------------------------------------------------------------------------------- 1 | # Allow "from hapiclient import hapi" 2 | from hapiclient.hapi import hapi 3 | 4 | # Allow "from hapiclient import hapitime2datetime" 5 | from hapiclient.hapitime import hapitime2datetime 6 | 7 | # Allow "from hapiclient import datetime2hapitime" 8 | from hapiclient.hapitime import datetime2hapitime 9 | 10 | # Allow "from hapiclient import HAPIError" 11 | from hapiclient.util import HAPIError 12 | 13 | __version__ = '0.2.7b1' 14 | 15 | import sys 16 | if sys.version_info[0] < 3: 17 | # Python 2.7 18 | reload(sys) 19 | sys.setdefaultencoding('utf8') 20 | 21 | -------------------------------------------------------------------------------- /hapiclient/test/test_datetime2hapitime.py: -------------------------------------------------------------------------------- 1 | def test_datetime2hapitime(): 2 | 3 | import datetime 4 | from hapiclient import datetime2hapitime 5 | 6 | dts = [datetime.datetime(2000,1,1),datetime.datetime(2000,1,2)]; 7 | hapi_times = datetime2hapitime(dts) 8 | assert hapi_times[0] == '2000-01-01T00:00:00.000000Z' 9 | assert hapi_times[1] == '2000-01-02T00:00:00.000000Z' 10 | 11 | dt = datetime.datetime(2000,1,1) 12 | hapi_time = datetime2hapitime(dt) 13 | assert hapi_time == '2000-01-01T00:00:00.000000Z' 14 | 15 | 16 | if __name__ == '__main__': 17 | test_datetime2hapitime() 18 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py35,py36,py37,py38,py39,py310,py311 3 | skip_missing_interpreters = true 4 | 5 | [testenv:repo-test] 6 | commands = 7 | pip install pytest deepdiff 8 | 9 | [testenv:long-test] 10 | deps = 11 | pytest 12 | deepdiff 13 | commands = 14 | python -m pytest -v -m 'long' hapiclient/test/test_hapi.py 15 | 16 | [testenv:short-test] 17 | deps = 18 | pytest 19 | deepdiff 20 | commands = 21 | python -m pytest -v -m 'short' hapiclient/test/test_hapi.py 22 | python -m pytest -v hapiclient/test/test_chunking.py 23 | python -m pytest -v hapiclient/test/test_hapitime2datetime.py 24 | python -m pytest -v hapiclient/test/test_datetime2hapitime.py 25 | python -m pytest -v hapiclient/test/test_hapitime_reformat.py 26 | -------------------------------------------------------------------------------- /.zenodo.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Python client for HAPI", 3 | "license": "other-open", 4 | "title": "hapi-server/client-python:", 5 | "version": "v0.2.1", 6 | "upload_type": "software", 7 | "publication_date": "2021-10-06", 8 | "creators": [ 9 | { 10 | "affiliation": "George Mason University", 11 | "name": "Bob Weigel" 12 | } 13 | ], 14 | "access_right": "open", 15 | "related_identifiers": [ 16 | { 17 | "scheme": "url", 18 | "identifier": "https://github.com/hapi-server/client-python/tree/v0.2.1", 19 | "relation": "isSupplementTo" 20 | }, 21 | { 22 | "scheme": "doi", 23 | "identifier": "10.5281/zenodo.5553155", 24 | "relation": "isVersionOf" 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | jobs: 3 | include: 4 | - name: "Python 3.8 on Linux" 5 | os: linux 6 | # if: branch = master 7 | dist: xenial 8 | language: python 9 | python: 3.8 10 | before_install: 11 | - sudo rm /usr/bin/python 12 | - sudo ln -s /usr/bin/python3 /usr/bin/python 13 | - python3 -m pip install tox-travis 14 | 15 | - name: "Python 3.8 on macOS" 16 | os: osx 17 | # if: branch = master 18 | osx_image: xcode11.6 # Python 3.8.0 running on macOS 10.14.6 19 | language: shell # 'language: python' is an error on Travis CI macOS 20 | before_install: 21 | - python3 --version 22 | - python3 -m pip install --upgrade pip 23 | - sudo pip install tox-travis 24 | 25 | - name: "Python 3.8 on Windows" 26 | os: windows # Windows 10.0.17134 N/A Build 17134 27 | # if: branch = master 28 | language: shell # 'language: python' is an error on Travis CI Windows 29 | before_install: 30 | - choco install python --version 3.8.8 31 | - python --version 32 | - python -m pip install --upgrade pip 33 | - python -m pip install tox-travis 34 | env: PATH=/c/Python38:/c/Python38/Scripts:$PATH 35 | 36 | install: pip install --upgrade pip 37 | 38 | script: 39 | - tox -e short-test 40 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2019- R.S. Weigel 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from setuptools import setup, find_packages 3 | 4 | install_requires = ["isodate>=0.6.0","urllib3","joblib"] 5 | 6 | if sys.version_info >= (3, 6): 7 | install_requires.append('pandas>=0.23') 8 | install_requires.append('numpy>=1.14.3') 9 | elif sys.version_info < (3, 6) and sys.version_info > (3, ): 10 | install_requires.append("pandas<1.0") 11 | install_requires.append("numpy<1.17") 12 | install_requires.append("kiwisolver<=1.2") 13 | else: 14 | install_requires.append("pandas>=0.23,<0.25") 15 | install_requires.append("numpy<1.17") 16 | install_requires.append("pyparsing<3") 17 | install_requires.append("kiwisolver<=1.1") 18 | 19 | if sys.argv[1] == 'develop': 20 | install_requires.append("deepdiff") 21 | if sys.version_info < (3, 6): 22 | install_requires.append("pytest<5.0.0") 23 | else: 24 | # Should not be needed, as per 25 | # https://docs.pytest.org/en/stable/py27-py34-deprecation.html 26 | # Perhaps old version of pip causes this? 27 | install_requires.append("pytest") 28 | 29 | # version is modified by misc/version.py (executed from Makefile). Do not edit. 30 | setup( 31 | name='hapiclient', 32 | version='0.2.7b1', 33 | author='Bob Weigel', 34 | author_email='rweigel@gmu.edu', 35 | packages=find_packages(), 36 | url='http://pypi.python.org/pypi/hapiclient/', 37 | license='LICENSE.txt', 38 | description='Interface to Heliophysics data server API', 39 | long_description=open('README.md').read(), 40 | long_description_content_type='text/markdown', 41 | install_requires=install_requires 42 | ) 43 | -------------------------------------------------------------------------------- /misc/version.py: -------------------------------------------------------------------------------- 1 | """ 2 | Reads CHANGES.txt and finds last version. Updates version strings in Makefile, 3 | setup.py, hapi.py, and hapiclient.py to match this version. 4 | 5 | TODO: Find better way to manage this. 6 | TODO: Update copyright year in LICENSE.txt 7 | """ 8 | 9 | import os 10 | import re 11 | 12 | overwrite = True 13 | 14 | # Get last version in CHANGES.txt 15 | print("Finding version information from CHANGES.txt") 16 | fin = open("CHANGES.txt") 17 | version = '0.0.0' 18 | for line in fin: 19 | (repl, n) = re.subn(r"^v(.*):.*", r"\1", line) 20 | if n > 0: 21 | version = repl 22 | fin.close() 23 | version = version.rstrip() 24 | print("Using version = " + version) 25 | 26 | fnames = ["Makefile", "setup.py", "hapiclient/hapi.py", "hapiclient/__init__.py"] 27 | regexes = ["VERSION=(.*)", "version=(.*)", "Version: (.*)", "__version__ = (.*)"] 28 | replaces = ["VERSION=" + version, "version='" + version + "',", "Version: " + version, "__version__ = '" + version + "'"] 29 | for i in range(len(fnames)): 30 | updated = False 31 | lines = '' 32 | fin = open(fnames[i]) 33 | print("Scanning " + fnames[i]) 34 | for lineo in fin: 35 | line1 = re.sub(regexes[i], replaces[i], lineo) 36 | if lineo != line1: 37 | print("Original: " + lineo.rstrip()) 38 | print("Modified: " + line1.rstrip()) 39 | updated = True 40 | lines = lines + line1 41 | fin.close() 42 | 43 | if not updated: 44 | print(" Version in file was already up-to-date.") 45 | continue 46 | 47 | with open(fnames[i] + ".tmp", "w") as fout: 48 | fout.write(lines) 49 | print("Wrote " + fnames[i] + ".tmp") 50 | 51 | if overwrite: 52 | os.rename(fnames[i] + ".tmp", fnames[i]) 53 | print(" Renamed " + fnames[i] + ".tmp" + " to " + fnames[i]) 54 | -------------------------------------------------------------------------------- /hapiclient/test/test_hapitime_reformat.py: -------------------------------------------------------------------------------- 1 | import os 2 | from hapiclient.hapitime import hapitime_reformat 3 | 4 | # See comments in test_hapitime2datetime.py for execution options. 5 | 6 | def test_hapitime_reformat(): 7 | 8 | 9 | dts = [ 10 | "1989Z", 11 | 12 | "1989-01Z", 13 | 14 | "1989-001Z", 15 | "1989-01-01Z", 16 | 17 | "1989-001T00Z", 18 | "1989-01-01T00Z", 19 | 20 | "1989-001T00:00Z", 21 | "1989-01-01T00:00Z", 22 | 23 | "1989-01-01T00:00:00.0Z", 24 | "1989-001T00:00:00.0Z", 25 | 26 | "1989-01-01T00:00:00.00Z", 27 | "1989-001T00:00:00.00Z", 28 | 29 | "1989-01-01T00:00:00.000Z", 30 | "1989-001T00:00:00.000Z", 31 | 32 | "1989-01-01T00:00:00.0000Z", 33 | "1989-001T00:00:00.0000Z", 34 | 35 | "1989-01-01T00:00:00.00000Z", 36 | "1989-001T00:00:00.00000Z", 37 | 38 | "1989-01-01T00:00:00.000000Z", 39 | "1989-001T00:00:00.000000Z", 40 | 41 | "1989-01-01T00:00:00.0000000Z", 42 | "1989-001T00:00:00.0000000Z", 43 | 44 | "1989-01-01T00:00:00.00000000Z", 45 | "1989-001T00:00:00.00000000Z", 46 | 47 | "1989-01-01T00:00:00.000000000Z", 48 | "1989-001T00:00:00.000000000Z" 49 | ] 50 | 51 | 52 | for i in range(len(dts)): 53 | if "T" in dts[i]: 54 | dts.append("1989-001T" + dts[i].split("T")[1]) 55 | 56 | logging = open(os.path.realpath(os.path.splitext(__file__)[0]) + ".log", "a") 57 | 58 | # truncating 59 | for i in range(len(dts)): 60 | form_to_match = dts[i] 61 | for j in range(i + 1, len(dts)): 62 | given_form = dts[j] 63 | given_form_modified = hapitime_reformat(form_to_match, given_form, logging=logging) 64 | assert given_form_modified == form_to_match 65 | 66 | # padding 67 | dts = list(reversed(dts)) 68 | for i in range(len(dts)): 69 | form_to_match = dts[i] 70 | for j in range(i + 1, len(dts)): 71 | given_form = dts[j] 72 | given_form_modified = hapitime_reformat(form_to_match, given_form, logging=logging) 73 | assert given_form_modified == form_to_match 74 | 75 | if __name__ == '__main__': 76 | test_hapitime_reformat() 77 | -------------------------------------------------------------------------------- /hapiclient/test/test_hapitime2datetime.py: -------------------------------------------------------------------------------- 1 | # To test from command line, use 2 | # pytest -v test_hapitime2datetime.py 3 | 4 | # To test a single function on the command line, use, e.g., 5 | # pytest -v test_hapitime2datetime.py -k test_parse_string_input 6 | 7 | # To use in program, use, e.g., 8 | # from hapiclient.test.test_hapitime2datetime import test_parse_string_input 9 | # test_parse_string_input() 10 | 11 | import os 12 | import numpy as np 13 | 14 | from hapiclient import hapitime2datetime 15 | from hapiclient.util import log 16 | 17 | # Create empty file 18 | logfile = os.path.splitext(__file__)[0] + ".log" 19 | with open(logfile, "w") as f: pass 20 | logging = open(logfile, "a") 21 | 22 | expected = '1970-01-01T00:00:00.000000Z' 23 | 24 | def test_api(): 25 | 26 | log("API test.", {'logging': logging}) 27 | 28 | Time = '1970-01-01T00:00:00.000Z' 29 | a = hapitime2datetime(Time,logging=logging) 30 | assert a[0].tzinfo.__class__.__name__ == 'UTC' 31 | assert a[0].strftime("%Y-%m-%dT%H:%M:%S.%fZ") == expected 32 | 33 | Time = ['1970-01-01T00:00:00.000Z'] 34 | a = hapitime2datetime(Time,logging=logging) 35 | assert a[0].tzinfo.__class__.__name__ == 'UTC' 36 | assert a[0].strftime("%Y-%m-%dT%H:%M:%S.%fZ") == expected 37 | 38 | Time = np.array(['1970-01-01T00:00:00.000Z']) 39 | a = hapitime2datetime(Time,logging=logging) 40 | assert a[0].tzinfo.__class__.__name__ == 'UTC' 41 | assert a[0].strftime("%Y-%m-%dT%H:%M:%S.%fZ") == expected 42 | 43 | Time = b'1970-01-01T00:00:00.000Z' 44 | a = hapitime2datetime(Time,logging=logging) 45 | assert a[0].tzinfo.__class__.__name__ == 'UTC' 46 | assert a[0].strftime("%Y-%m-%dT%H:%M:%S.%fZ") == expected 47 | 48 | Time = [b'1970-01-01T00:00:00.000Z'] 49 | a = hapitime2datetime(Time,logging=logging) 50 | assert a[0].tzinfo.__class__.__name__ == 'UTC' 51 | assert a[0].strftime("%Y-%m-%dT%H:%M:%S.%fZ") == expected 52 | 53 | Time = np.array([b'1970-01-01T00:00:00.000Z']) 54 | a = hapitime2datetime(Time,logging=logging) 55 | assert a[0].tzinfo.__class__.__name__ == 'UTC' 56 | assert a[0].strftime("%Y-%m-%dT%H:%M:%S.%fZ") == expected 57 | 58 | 59 | def test_parsing(): 60 | 61 | log("Parse test.", {'logging': logging}) 62 | 63 | dts = [ 64 | "1989Z", 65 | 66 | "1989-01Z", 67 | 68 | "1989-001Z", 69 | "1989-01-01Z", 70 | 71 | "1989-001T00Z", 72 | "1989-01-01T00Z", 73 | 74 | "1989-001T00:00Z", 75 | "1989-01-01T00:00Z", 76 | 77 | "1989-001T00:00:00.Z", 78 | "1989-01-01T00:00:00.Z", 79 | 80 | "1989-01-01T00:00:00.0Z", 81 | "1989-001T00:00:00.0Z", 82 | 83 | "1989-01-01T00:00:00.00Z", 84 | "1989-001T00:00:00.00Z", 85 | 86 | "1989-01-01T00:00:00.000Z", 87 | "1989-001T00:00:00.000Z", 88 | 89 | "1989-01-01T00:00:00.0000Z", 90 | "1989-001T00:00:00.0000Z", 91 | 92 | "1989-01-01T00:00:00.00000Z", 93 | "1989-001T00:00:00.00000Z", 94 | 95 | "1989-01-01T00:00:00.000000Z", 96 | "1989-001T00:00:00.000000Z" 97 | ] 98 | 99 | expected = '1989-01-01T00:00:00.000000Z' 100 | 101 | for i in range(len(dts)): 102 | a = hapitime2datetime(dts[i],logging=logging) 103 | assert a[0].tzinfo.__class__.__name__ == 'UTC' 104 | assert a[0].strftime("%Y-%m-%dT%H:%M:%S.%fZ") == expected 105 | 106 | 107 | def test_error_conditions(logging=logging): 108 | from hapiclient import HAPIError 109 | 110 | 111 | Time = "1999" 112 | log("Checking that hapitime2datetime(" + str(Time) + ") throws HAPIError", {'logging': logging}) 113 | try: 114 | hapitime2datetime(Time, logging=logging) 115 | except HAPIError: 116 | pass 117 | else: 118 | assert False, "HAPIError not raised for hapitime2datetime(" + str(Time) + ")." 119 | 120 | 121 | if __name__ == '__main__': 122 | test_api() 123 | test_parsing() 124 | test_error_conditions() 125 | -------------------------------------------------------------------------------- /hapiclient/test/test_hapitime2datetime.log: -------------------------------------------------------------------------------- 1 | test_api(): API test. 2 | hapitime2datetime(): Pandas processing time = 0.0015s, first time = 1970-01-01T00:00:00.000Z 3 | hapitime2datetime(): Pandas processing time = 0.0006s, first time = 1970-01-01T00:00:00.000Z 4 | hapitime2datetime(): Pandas processing time = 0.0005s, first time = 1970-01-01T00:00:00.000Z 5 | hapitime2datetime(): Pandas processing time = 0.0005s, first time = 1970-01-01T00:00:00.000Z 6 | hapitime2datetime(): Pandas processing time = 0.0005s, first time = 1970-01-01T00:00:00.000Z 7 | hapitime2datetime(): Pandas processing time = 0.0005s, first time = 1970-01-01T00:00:00.000Z 8 | test_parsing(): Parse test. 9 | hapitime2datetime(): Pandas processing failed, first time = 1989Z 10 | hapitime2datetime(): Manual processing time = 0.0015s, Input = 1989Z, fmt = %YZ 11 | hapitime2datetime(): Pandas processing failed, first time = 1989-01Z 12 | hapitime2datetime(): Manual processing time = 0.0008s, Input = 1989-01Z, fmt = %Y-%mZ 13 | hapitime2datetime(): Pandas processing failed, first time = 1989-001Z 14 | hapitime2datetime(): Manual processing time = 0.0008s, Input = 1989-001Z, fmt = %Y-%jZ 15 | hapitime2datetime(): Pandas processing failed, first time = 1989-01-01Z 16 | hapitime2datetime(): Manual processing time = 0.0009s, Input = 1989-01-01Z, fmt = %Y-%m-%dZ 17 | hapitime2datetime(): Pandas processing failed, first time = 1989-001T00Z 18 | hapitime2datetime(): Manual processing time = 0.0009s, Input = 1989-001T00Z, fmt = %Y-%jT%HZ 19 | hapitime2datetime(): Pandas processing time = 0.0015s, first time = 1989-01-01T00Z 20 | hapitime2datetime(): Pandas processing failed, first time = 1989-001T00:00Z 21 | hapitime2datetime(): Manual processing time = 0.0010s, Input = 1989-001T00:00Z, fmt = %Y-%jT%H:%MZ 22 | hapitime2datetime(): Pandas processing time = 0.0012s, first time = 1989-01-01T00:00Z 23 | hapitime2datetime(): Pandas processing failed, first time = 1989-001T00:00:00.Z 24 | hapitime2datetime(): Manual processing time = 0.0010s, Input = 1989-001T00:00:00.Z, fmt = %Y-%jT%H:%M:%S.Z 25 | hapitime2datetime(): Pandas processing time = 0.0004s, first time = 1989-01-01T00:00:00.Z 26 | hapitime2datetime(): Pandas processing time = 0.0005s, first time = 1989-01-01T00:00:00.0Z 27 | hapitime2datetime(): Pandas processing failed, first time = 1989-001T00:00:00.0Z 28 | hapitime2datetime(): Manual processing time = 0.0010s, Input = 1989-001T00:00:00.0Z, fmt = %Y-%jT%H:%M:%S.%fZ 29 | hapitime2datetime(): Pandas processing time = 0.0005s, first time = 1989-01-01T00:00:00.00Z 30 | hapitime2datetime(): Pandas processing failed, first time = 1989-001T00:00:00.00Z 31 | hapitime2datetime(): Manual processing time = 0.0006s, Input = 1989-001T00:00:00.00Z, fmt = %Y-%jT%H:%M:%S.%fZ 32 | hapitime2datetime(): Pandas processing time = 0.0006s, first time = 1989-01-01T00:00:00.000Z 33 | hapitime2datetime(): Pandas processing failed, first time = 1989-001T00:00:00.000Z 34 | hapitime2datetime(): Manual processing time = 0.0005s, Input = 1989-001T00:00:00.000Z, fmt = %Y-%jT%H:%M:%S.%fZ 35 | hapitime2datetime(): Pandas processing time = 0.0005s, first time = 1989-01-01T00:00:00.0000Z 36 | hapitime2datetime(): Pandas processing failed, first time = 1989-001T00:00:00.0000Z 37 | hapitime2datetime(): Manual processing time = 0.0005s, Input = 1989-001T00:00:00.0000Z, fmt = %Y-%jT%H:%M:%S.%fZ 38 | hapitime2datetime(): Pandas processing time = 0.0005s, first time = 1989-01-01T00:00:00.00000Z 39 | hapitime2datetime(): Pandas processing failed, first time = 1989-001T00:00:00.00000Z 40 | hapitime2datetime(): Manual processing time = 0.0005s, Input = 1989-001T00:00:00.00000Z, fmt = %Y-%jT%H:%M:%S.%fZ 41 | hapitime2datetime(): Pandas processing time = 0.0006s, first time = 1989-01-01T00:00:00.000000Z 42 | hapitime2datetime(): Pandas processing failed, first time = 1989-001T00:00:00.000000Z 43 | hapitime2datetime(): Manual processing time = 0.0005s, Input = 1989-001T00:00:00.000000Z, fmt = %Y-%jT%H:%M:%S.%fZ 44 | test_error_conditions(): Checking that hapitime2datetime(1999) throws HAPIError 45 | -------------------------------------------------------------------------------- /CHANGES.txt: -------------------------------------------------------------------------------- 1 | v0.0.1: 2018-09-19 -- Initial package release. 2 | v0.0.2: 2018-09-19 -- Initial package release. 3 | v0.0.3: 2018-09-19 -- Initial package release. 4 | v0.0.4: 2018-09-19 -- Initial package release. 5 | v0.0.5: 6 | 2018-09-26 -- Set pickle protocol to 2 for python 2/3 compatability 7 | v0.0.6: 8 | 2018-09-04 -- Move hapitime2datetime to hapi.py 9 | 2018-10-04 -- Make hapitime2datetime output datetimes and add tests 10 | 2018-10-04 -- Improve dateticks.py 11 | 2018-10-04 -- Add hapi_demo.ipynb 12 | 2018-10-04 -- Other documentation improvements 13 | v0.0.7: 14 | 2018-10-06 -- Improve documentation 15 | 2018-10-06 -- Add reader tests 16 | 2018-10-23 -- Improve dateticks and tests 17 | 2018-10-23 -- Add OO interface for plots; add options 18 | 2018-11-27 -- Major refactoring and clean-up 19 | 2018-11-28 -- Add gallery and autoplot plotting options 20 | v0.0.8: 21 | 2018-02-14 -- Same as v0.0.7 but packaging fix 22 | v0.0.9b0: 23 | 2018-03-06 -- Updates to plot API 24 | v0.0.9b1: 25 | 2018-03-06 -- load(res) -> load(res.json()) in util.jsonparse(). 26 | 2018-03-06 -- Add tests for more Python versions. 27 | 2018-05-15 -- SSL issue for urlopen. 28 | 2018-05-17 -- Fixes for heatmap and Matplotlib 3.1. 29 | v0.1.0: 2018-05-21 -- Tagged and pip-installable version 30 | v0.1.1b: 31 | 2018-06-06 -- Beta for next version 32 | 2018-06-07 -- Plot improvements 33 | 2019-01-07 -- Address issue #6 (parameter order) 34 | v0.1.1: 2019-01-08 -- Tagged and pip-installable version 35 | v0.1.2b: 36 | 2019-01-08 -- Beta for next version 37 | 2019-01-16 -- Address issues #5 and #7 38 | v0.1.2: 2019-01-16 -- Tagged and pip-installable version 39 | v0.1.3b: 40 | 2019-01-17 -- Correction for subset() when only time param requested 41 | 2020-05-15 -- Correct interpretation of size 42 | 2020-05-15 -- Support HAPI 2.1 unit and label arrays 43 | 2020-07-27 -- Many minor plot fixes 44 | v0.1.3: 2020-07-27 -- Tagged and pip-installable version 45 | v0.1.4: 46 | 2020-07-27 -- datetick.py/OS-X fixes 47 | 2020-07-27 -- Python 2.7 48 | v0.1.5b0: 49 | 2020-07-30 -- Use urllib3 50 | v0.1.5b1: 51 | 2020-07-30 -- Use threadsafe plots 52 | v0.1.5b2: 53 | 2020-07-31 -- Handle all NaN timeseries 54 | v0.1.5b4: 55 | 2020-12-11 -- Chunked requests 56 | v0.1.5: 57 | 2020-12-14 -- urllib3 and chunked requests 58 | v0.1.6: 59 | 2020-12-15 -- Closes #25, pythonshell() Windows issue; e576f7e 60 | v0.1.7: 61 | 2020-12-23 -- 797ce42 Make error() work when pythonshell() == 'shell' 62 | 2020-12-23 -- 84daa48 Chunking bugfixes 63 | v0.1.8: 64 | 2020-12-23 -- 1cfbfc3 Add P1D chunk tests 65 | 2020-12-23 -- f41599c Add P1Y chunk tests 66 | 2020-12-23 -- ff8e1ab Refactor hapi.py; clean up util.py 67 | v0.1.9b0: 68 | 2020-12-24 -- Remove hapiplot. Addresses #31. 69 | 2020-12-28 -- 1c1d32f Fix time name conflict. 70 | v0.1.9b1: 71 | 2020-01-19 -- 53669fc Add missing hapitime.py file 72 | 2020-01-19 -- Deleted v0.1.8 pip package b/c affected by above 73 | 2020-01-20 -- 9f8b82f Use atol instead of rtol in readcompare.py 74 | 2020-01-20 -- d3625e3 Fix for wrong error message (#24) 75 | v0.1.9b2: 76 | 2021-05-25 -- 23b659c Simplify dtype/length code 77 | v0.1.9b3: 78 | 2021-05-25 -- 8161460 Refactor Makefile 79 | v0.1.9b4: 80 | 2021-05-26 -- 2cabeb4 README.rst -> README.md 81 | 2021-05-26 -- b825a54 Fix README file extension 82 | v0.2.0: 83 | 2021-06-16 -- 1e891a9 Tox and Travis update 84 | v0.2.1: 85 | 2021-06-16 -- Fix for long_description_content_type 86 | v0.2.2: 87 | 2021-06-16 -- Move import of joblib. 88 | 2022-01-17 -- Full Unicode support and additional tests. 89 | v0.2.3: 90 | 2022-03-01 -- bf0cc88 Add allow_missing_Z in hapitime2datetime. 91 | v0.2.4: 92 | 2022-03-07 -- 88b0eb2 try/except on file read 93 | 2022-03-29 -- 0042abb Fix for method=''. Closes #47. 94 | v0.2.5: 95 | 2022-06-30 -- 10df388 Update docstring for hapi() 96 | 2022-07-12 -- 460c30b Final Unicode/Windows fixes 97 | v0.2.6b: 98 | 2022-08-08 -- 508fb20-3f1168c Add datetime2hapitime 99 | 2022-08-08 -- 8bfef2f Allow start and stop to be None; https://github.com/hapi-server/client-python/issues/10 100 | 2022-09-21 -- bc70f3c CSV read failure; https://github.com/hapi-server/client-python/issues/62 101 | 2022-09-22 -- 373fdca Catch other CSV read failures 102 | v0.2.6b1: 103 | 2023-02-04 -- 2ef7cff Unicode fix; Log string fix 104 | 2024-05-11 -- fba4ad3 Testing Updates 105 | v0.2.6: 106 | 2023-05-24 -- Use token for PyPi in Makefile 107 | v0.2.7b1: 108 | 2025-03-01 -- https://github.com/hapi-server/client-python/issues/76 109 | 2025-03-01 -- https://github.com/hapi-server/client-python/issues/78 -------------------------------------------------------------------------------- /hapi_demo.py: -------------------------------------------------------------------------------- 1 | # Basic demo of hapiclient. Install package using 2 | # pip install hapiclient --upgrade 3 | # from command line. 4 | 5 | # Note: 6 | # In IPython, enter 7 | # %matplotlib qt 8 | # on command line to open plots in new window. Enter 9 | # %matplotlib inline 10 | # to revert. 11 | 12 | # For more extensive demos and examples, see 13 | # https://colab.research.google.com/drive/11Zy99koiE90JKJ4u_KPTaEBMQFzbfU3P?usp=sharing 14 | 15 | def main(): 16 | 17 | demos = [omniweb, sscweb, cdaweb, cassini, lisird] 18 | 19 | for demo in demos: 20 | try: 21 | demo() 22 | except Exception as e: 23 | print("\033[0;31mError:\033[0m " + str(e)) 24 | 25 | 26 | def testdata(): 27 | 28 | from hapiclient import hapi 29 | 30 | server = 'http://hapi-server.org/servers/TestData2.0/hapi' 31 | dataset = 'dataset1' 32 | parameters = 'scalar' 33 | start = '1970-01-01T00:00:00' 34 | stop = '1970-01-02T00:01:00' 35 | parameters = 'scalar,vector' 36 | opts = {'logging': True, 'usecache': True} 37 | 38 | data, meta = hapi(server, dataset, parameters, start, stop, **opts) 39 | 40 | # Plot all parameters 41 | hapiplot(data, meta) 42 | 43 | def omniweb(): 44 | 45 | from hapiclient import hapi 46 | 47 | server = 'https://cdaweb.gsfc.nasa.gov/hapi' 48 | dataset = 'OMNI2_H0_MRG1HR' 49 | start = '2003-09-01T00:00:00' 50 | stop = '2003-12-01T00:00:00' 51 | parameters = 'DST1800' 52 | opts = {'logging': True, 'usecache': False} 53 | 54 | # Get data 55 | data, meta = hapi(server, dataset, parameters, start, stop, **opts) 56 | 57 | # Plot all parameters 58 | hapiplot(data, meta) 59 | 60 | 61 | def sscweb(): 62 | 63 | from hapiclient import hapi 64 | from hapiplot import hapiplot 65 | 66 | # SSCWeb data 67 | server = 'http://hapi-server.org/servers/SSCWeb/hapi' 68 | dataset = 'ace' 69 | start = '2001-01-01T05:00:00' 70 | stop = '2001-01-01T10:00:00' 71 | parameters = 'X_GSE,Y_GSE,Z_GSE' 72 | opts = {'logging': True, 'usecache': True} 73 | data, meta = hapi(server, dataset, parameters, start, stop, **opts) 74 | 75 | hapiplot(data, meta, **opts) 76 | 77 | 78 | def cdaweb(): 79 | 80 | from hapiclient import hapi 81 | from hapiplot import hapiplot 82 | 83 | # CDAWeb data - Magnitude and BGSEc from dataset AC_H0_MFI 84 | server = 'https://cdaweb.gsfc.nasa.gov/hapi' 85 | dataset = 'AC_H0_MFI' 86 | start = '2001-01-01T05:00:00' 87 | stop = '2001-01-01T10:00:00' 88 | parameters = 'Magnitude,BGSEc' 89 | opts = {'logging': True, 'usecache': True} 90 | data, meta = hapi(server, dataset, parameters, start, stop, **opts) 91 | hapiplot(data, meta, **opts) 92 | 93 | # CDAWeb metadata for AC_H0_MFI 94 | server = 'https://cdaweb.gsfc.nasa.gov/hapi' 95 | dataset = 'AC_H0_MFI' 96 | meta = hapi(server, dataset, **opts) 97 | print('Parameters in %s' % dataset) 98 | for i in range(0, len(meta['parameters'])): 99 | print(' %s' % meta['parameters'][i]['name']) 100 | print('') 101 | 102 | # CDAWeb metadata for all datasets 103 | server = 'https://cdaweb.gsfc.nasa.gov/hapi' 104 | meta = hapi(server, **opts) 105 | print('%d CDAWeb datasets' % len(meta['catalog'])) 106 | for i in range(0, 3): 107 | print(' %d. %s' % (i, meta['catalog'][i]['id'])) 108 | print(' ...') 109 | print(' %d. %s' % (len(meta['catalog']), meta['catalog'][-1]['id'])) 110 | print('') 111 | 112 | # List all servers 113 | servers = hapi(logging=True) # servers is an array of URLs 114 | print('') 115 | 116 | 117 | def cassini(): 118 | 119 | from hapiclient import hapi 120 | from hapiplot import hapiplot 121 | 122 | server = 'http://datashop.elasticbeanstalk.com/hapi'; 123 | dataset = 'CHEMS_PHA_BOX_FLUXES_FULL_TIME_RES'; 124 | parameters = 'HPlus_BEST_T1'; 125 | start = '2004-07-01T04:00:00Z'; 126 | stop = '2004-07-01T06:00:00Z'; 127 | opts = {'usecache': True, 'logging': True} 128 | 129 | data, meta = hapi(server, dataset, parameters, start, stop, **opts) 130 | 131 | popts = {'logging': False, 'logy': True, 'logz': True} 132 | hapiplot(data, meta, **popts) 133 | 134 | 135 | def lisird(): 136 | 137 | from hapiclient import hapi 138 | from hapiplot import hapiplot 139 | 140 | server = 'http://lasp.colorado.edu/lisird/hapi'; 141 | dataset = 'sme_ssi'; 142 | parameters = 'irradiance'; 143 | start = '1981-10-09T00:00:00.000Z'; 144 | stop = '1981-10-14T00:00:00.000Z'; 145 | 146 | opts = {'usecache': True, 'logging': True} 147 | data, meta = hapi(server, dataset, parameters, start, stop, **opts) 148 | 149 | hapiplot(data, meta) 150 | 151 | 152 | if __name__ == '__main__': 153 | try: 154 | from hapiplot import hapiplot 155 | except: 156 | print('Package hapiplot is not installed. Will not plot results.') 157 | main() 158 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![DOI](https://zenodo.org/badge/93170857.svg)](https://zenodo.org/badge/latestdoi/93170857) 2 | [![Build Status](https://app.travis-ci.com/hapi-server/client-python.svg?branch=master)](https://app.travis-ci.com/hapi-server/client-python) 3 | 4 | **HAPI Client for Python** 5 | 6 | Basic usage examples for various HAPI servers are given in [hapi_demo.py](https://github.com/hapi-server/client-python/blob/master/hapi_demo.py) and the Examples section of a Jupyter Notebook hosted on Google Colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/hapi-server/client-python-notebooks/blob/master/hapi_demo.ipynb#examples). 7 | 8 | # Installation 9 | 10 | ```bash 11 | pip install hapiclient --upgrade 12 | # or 13 | pip install 'git+https://github.com/hapi-server/client-python' --upgrade 14 | ``` 15 | 16 | The optional [hapiplot package](https://github.com/hapi-server/plot-python) provides basic preview plotting capabilities of data from a HAPI server. The [Plotting section](https://colab.research.google.com/github/hapi-server/client-python-notebooks/blob/master/hapi_demo.ipynb#plotting) of the `hapiclient` Jupyter Notebook shows how to plot the output of `hapiclient` using many other plotting libraries. 17 | 18 | To install `hapiplot`, use 19 | 20 | ```bash 21 | python -m pip install hapiplot --upgrade 22 | # or 23 | python -m pip install 'git+https://github.com/hapi-server/plot-python' --upgrade 24 | ``` 25 | 26 | See the [Appendix](#appendix) for a fail-safe installation method. 27 | 28 | # Basic Example 29 | 30 | ```python 31 | # Get Dst index from CDAWeb HAPI server 32 | from hapiclient import hapi 33 | 34 | # See http://hapi-server.org/servers/ for a list of 35 | # other HAPI servers and datasets. 36 | server = 'https://cdaweb.gsfc.nasa.gov/hapi' 37 | dataset = 'OMNI2_H0_MRG1HR' 38 | start = '2003-09-01T00:00:00' 39 | stop = '2003-12-01T00:00:00' 40 | parameters = 'DST1800' 41 | opts = {'logging': True} 42 | 43 | # Get data 44 | data, meta = hapi(server, dataset, parameters, start, stop, **opts) 45 | print(meta) 46 | print(data) 47 | 48 | # Plot all parameters 49 | from hapiplot import hapiplot 50 | hapiplot(data, meta) 51 | ``` 52 | 53 | # Documentation 54 | 55 | Basic usage examples for various HAPI servers are given in [hapi_demo.py](https://github.com/hapi-server/client-python/blob/master/hapi_demo.py>) and the [Examples section](https://colab.research.google.com/github/hapi-server/client-python-notebooks/blob/master/hapi_demo.ipynb#examples) of a Jupyter Notebook hosted on Google Colab. 56 | 57 | See http://hapi-server.org/servers/ for a list of HAPI servers and datasets. 58 | 59 | All of the features are extensively demonstrated in [hapi_demo.ipynb](https://colab.research.google.com/github/hapi-server/client-python-notebooks/blob/master/hapi_demo.ipynb#data-model), a Jupyter Notebook that can be viewed an executed on Google Colab. 60 | 61 | # Metadata Model 62 | 63 | See also the examples in the [Metadata Model section](https://colab.research.google.com/github/hapi-server/client-python-notebooks/blob/master/hapi_demo.ipynb) of the `hapiclient` Jupyter Notebook. 64 | 65 | The HAPI client metadata model is intentionally minimal and closely follows that of the [HAPI metadata model](https://github.com/hapi-server/data-specification). We expect that another library will be developed that allows high-level search and grouping of information from HAPI servers. See also [issue #106](https://github.com/hapi-server/data-specification/issues/106). 66 | 67 | # Data Model and Time Format 68 | 69 | See also the examples in the [Data Model section](https://colab.research.google.com/github/hapi-server/client-python-notebooks/blob/master/hapi_demo.ipynb) of the `hapiclient` Jupyter Notebook. The examples include 70 | 71 | 1. Fast and well-tested conversion from ISO 8601 timestamp strings to Python `datetime` objects 72 | 2. Putting the content of `data` in a Pandas `DataFrame` object 73 | 3. Creating an Astropy NDArray 74 | 75 | A request for data of the form 76 | ``` 77 | data, meta = hapi(server, dataset, parameters, start, stop) 78 | ``` 79 | 80 | returns the [Numpy N-D array](https://docs.scipy.org/doc/numpy-1.15.1/user/quickstart.html) `data` and a Python dictionary `meta` from a HAPI-compliant data server `server`. The structure of `data` and `meta` mirrors the structure of a response from a HAPI server. 81 | 82 | The HAPI client data model is intentionally basic. There is an ongoing discussion of a data model for Heliophysics data among the [PyHC community](https://heliopython.org/). When this data model is complete, a function that converts `data` and `meta` to that data model will be included in the `hapiclient` package. 83 | 84 | # Development 85 | 86 | ```bash 87 | git clone https://github.com/hapi-server/client-python 88 | cd client-python; python -m pip install -e . 89 | ``` 90 | 91 | or, create an isolated Anaconda installation (downloads and installs latest Miniconda3) using 92 | 93 | ``` bash 94 | make install PYTHON=python3.6 95 | # Execute command displayed to activate isolated environment 96 | ``` 97 | 98 | The command `pip install -e .` creates symlinks so that the local package is 99 | used instead of an installed package. You may need to execute `python -m pip uninstall hapiclient` to ensure the local package is used. To check the version installed, use `python -m pip list | grep hapiclient`. 100 | 101 | To run tests before a commit, execute 102 | 103 | ```bash 104 | make repository-test 105 | ``` 106 | 107 | To run an individual unit test in a Python session, use, e.g., 108 | 109 | ```python 110 | from hapiclient.test.test_hapi import test_reader_short 111 | test_reader_short() 112 | ``` 113 | 114 | # Contact 115 | 116 | Submit bug reports and feature requests on the [repository issue 117 | tracker](https://github.com/hapi-server/client-python/issues>). 118 | -------------------------------------------------------------------------------- /hapiclient/test/test_chunking.py: -------------------------------------------------------------------------------- 1 | import os 2 | import isodate 3 | 4 | from datetime import datetime 5 | from hapiclient import hapi 6 | from hapiclient.hapitime import hapitime2datetime, hapitime_reformat 7 | from hapiclient.test.compare import equal 8 | 9 | # See comments in test_hapitime2datetime.py for execution options. 10 | 11 | compare_logging = True 12 | hapi_logging = False 13 | 14 | # Create empty file 15 | logfile = os.path.splitext(__file__)[0] + ".log" 16 | with open(logfile, "w") as f: pass 17 | 18 | def xprint(msg): 19 | print(msg) 20 | f = open(logfile, "a") 21 | f.write(msg + "\n") 22 | f.close() 23 | 24 | server = "http://hapi-server.org/servers/TestData2.0/hapi" 25 | 26 | def compare(data1, data2, meta1, meta2, opts1, opts2): 27 | if compare_logging: 28 | xprint('_'*80) 29 | xprint('request : %s, %s, %s, %s, %s' % \ 30 | (meta1['x_dataset'], meta1['x_parameters'], meta1['x_time.min'], meta1['x_time.max'], meta1['cadence'])) 31 | xprint('options 1: %s' % opts1) 32 | xprint('options 2: %s' % opts2) 33 | xprint('x_totalTime1 = %6.4f s' % (meta1['x_totalTime'])) 34 | xprint('x_totalTime2 = %6.4f s' % (meta2['x_totalTime'])) 35 | if False and 'x_downloadTimes' in meta2: 36 | print(meta1['x_downloadTime']) 37 | print(meta1['x_readTime']) 38 | print(meta2['x_downloadTimes']) 39 | print(meta2['x_readTimes']) 40 | print(meta2['x_trimTime']) 41 | print(meta2['x_catTime']) 42 | 43 | assert equal(data1, data2) 44 | 45 | 46 | def cat(d1, d2): 47 | # Python 2-compatable dict concatenation. 48 | # https://treyhunner.com/2016/02/how-to-merge-dictionaries-in-python/ 49 | d12 = d1.copy() 50 | d12.update(d2) 51 | return d12 52 | 53 | 54 | opts0 = {'logging': hapi_logging, 'usecache': False, 'cache': True, 'format': 'csv'} 55 | 56 | # Test dict. 57 | # Key indicates chunk size to use for chunk test 58 | td = { 59 | "P1Y": { 60 | "__comment": "dataset3 has cadence of P1D", 61 | "server": server, 62 | "dataset": "dataset3", 63 | "parameters": "scalar", 64 | "start": "1971-01-01T01:50:00Z", 65 | "stop": "1975-08-03T06:50:00Z" 66 | }, 67 | "P1M": { 68 | "__comment": "dataset3 has cadence of P1D", 69 | "server": server, 70 | "dataset": "dataset3", 71 | "parameters": "scalar", 72 | "start": "1971-01-01T01:50:00Z", 73 | "stop": "1971-08-03T06:50:00Z" 74 | }, 75 | "P1D": 76 | { 77 | "__comment": "dataset2 has cadence of P1H", 78 | "server": server, 79 | "dataset": "dataset2", 80 | "parameters": "scalar", 81 | "start": "1970-01-01T00:00:00.000Z", 82 | "stop": "1970-01-10T00:00:00.000Z" 83 | }, 84 | "PT1H": { 85 | "__comment": "dataset1 has cadence of PT1S", 86 | "server": server, 87 | "dataset": "dataset1", 88 | "parameters": "scalar", 89 | "start": "1970-01-01T00:00:00.000Z", 90 | "stop": "1970-01-01T05:00:00.000Z" 91 | } 92 | } 93 | 94 | 95 | def test_chunks(): 96 | 97 | # Test of dt_chunk and n_chunks keyword arguments 98 | for key in td: 99 | 100 | s = td[key]['server'] 101 | d = td[key]['dataset'] 102 | p = td[key]['parameters'] 103 | 104 | if '-dev' in s: 105 | import warnings 106 | warnings.warn("Change server from dev to production for these tests") 107 | 108 | 109 | # Reference result. No splitting will be performed. 110 | opts1 = cat(opts0, {'dt_chunk': None}) 111 | data1, meta1 = hapi(s, d, p, td[key]['start'], td[key]['stop'], **opts1) 112 | 113 | opts = opts0 114 | data, meta = hapi(s, d, p, td[key]['start'], td[key]['stop'], **opts) 115 | compare(data1, data, meta1, meta, opts1, opts) 116 | 117 | opts = cat(opts0, {'n_chunks': 2}) 118 | data, meta = hapi(s, d, p, td[key]['start'], td[key]['stop'], **opts) 119 | compare(data1, data, meta1, meta, opts1, opts) 120 | 121 | opts = cat(opts0, {'dt_chunk': key}) 122 | data, meta = hapi(s, d, p, td[key]['start'], td[key]['stop'], **opts) 123 | compare(data1, data, meta1, meta, opts1, opts) 124 | 125 | opts = cat(opts0, {'parallel': True, 'n_chunks': 2}) 126 | data, meta = hapi(s, d, p, td[key]['start'], td[key]['stop'], **opts) 127 | compare(data1, data, meta1, meta, opts1, opts) 128 | 129 | opts = cat(opts0, {'parallel': True, 'dt_chunk': key}) 130 | data, meta = hapi(s, d, p, td[key]['start'], td[key]['stop'], **opts) 131 | compare(data1, data, meta1, meta, opts1, opts) 132 | 133 | 134 | def test_chunk_threshold(): 135 | 136 | # Test case when time span is less than minimum threshold for automatic 137 | # chunking. Code executed should be the same in both cases. This test does 138 | # not check that same code was executed, however. 139 | 140 | key = 'PT1H' 141 | chunk = 'P1D' 142 | 143 | s = td[key]['server'] 144 | d = td[key]['dataset'] 145 | p = td[key]['parameters'] 146 | 147 | # Default chunk size for 1-second data is P1D. No chunking performed if 148 | # stop-start < PT1D/2. 149 | start = td[key]['start'] 150 | stop = hapitime2datetime(start) + isodate.parse_duration(chunk)/3 151 | stop = datetime.isoformat(stop[0])[0:19] + "Z" 152 | 153 | # Reference result 154 | opts1 = cat(opts0, {'dt_chunk': None}) 155 | data1, meta1 = hapi(s, d, p, start, stop, **opts1) 156 | 157 | opts = cat(opts0, {'dt_chunk': 'infer'}) 158 | data, meta = hapi(s, d, p, start, stop, **opts) 159 | 160 | compare(data1, data, meta1, meta, opts1, opts) 161 | 162 | 163 | def test_timeformats(): 164 | 165 | # Data are efficiently subsetted using a NumPy operation of the form 166 | # data['Time'][ data['Time'] >= bytes('2000-01-01T00', 'UTF-8')] 167 | # Internally, the start and stop strings must be converted to 168 | # the ISO 8601 format of the data. 169 | 170 | # The following tests the possible combinations of start, stop, and 171 | # data time formats. 172 | 173 | key = "PT1H" 174 | 175 | s = td[key]['server'] 176 | d = td[key]['dataset'] 177 | p = td[key]['parameters'] 178 | 179 | start_ymd = td[key]['start'] 180 | start_doy = hapitime_reformat('1989-272T00:00:00.000Z',start_ymd) 181 | 182 | stop_ymd = td[key]['stop'] 183 | stop_doy = hapitime_reformat('1989-272T00:00:00.000Z',stop_ymd) 184 | 185 | # Reference result. 186 | # start: %Y-%m-%d, stop: %Y-%m-%d, data: %Y-%m-%d 187 | opts1 = {'usecache': False, 'cache': False, 'dt_chunk': None} 188 | data1, meta1 = hapi(s, d, p, start_ymd, stop_ymd, **opts1) 189 | 190 | opts = {'usecache': False, 'cache': False, 'dt_chunk': 'infer'} 191 | 192 | # start: %Y-%m-%d, stop: %Y-%j, data: %Y-%m-%d 193 | data, meta = hapi(s, d, p, start_ymd, stop_doy, **opts) 194 | compare(data1, data, meta1, meta, opts1, opts) 195 | 196 | # start: %Y-%j, stop: %Y-%m-%d, data: %Y-%m-%d 197 | data, meta = hapi(s, d, p, start_doy, stop_ymd, **opts) 198 | compare(data1, data, meta1, meta, opts1, opts) 199 | 200 | # start: %Y-%j, stop: %Y-%j, data: %Y-%m-%d 201 | data, meta = hapi(s, d, p, start_doy, stop_doy, **opts) 202 | compare(data1, data, meta1, meta, opts1, opts) 203 | 204 | ############################################################################ 205 | # Switch to use a server that serves data in %Y-%j format. Change to 206 | # TestData server when it has a data set served in %Y-%j format. 207 | ############################################################################ 208 | 209 | key = "P1D" 210 | tdx = { 211 | "P1D": { 212 | "__comment": "Timeformat is YYYY-DOY", 213 | "server": 'http://hapi-server.org/servers/SSCWeb/hapi', 214 | "dataset": 'ace', 215 | "parameters": 'X_GSM', 216 | "start": "2000-01-01T00:00:00.000Z", 217 | "stop": "2000-01-02T00:00:00.000Z" 218 | } 219 | } 220 | 221 | s = tdx[key]['server'] 222 | d = tdx[key]['dataset'] 223 | p = tdx[key]['parameters'] 224 | 225 | start_ymd = tdx[key]['start'] 226 | start_doy = hapitime_reformat('1989-272T00:00:00.000Z',start_ymd) 227 | 228 | stop_ymd = tdx[key]['stop'] 229 | stop_doy = hapitime_reformat('1989-272T00:00:00.000Z',stop_ymd) 230 | 231 | # Reference result 232 | # start: %Y-%m-%d, stop: %Y-%m-%d, data: %Y-%j 233 | data1, meta1 = hapi(s, d, p, start_ymd, stop_ymd, **opts1) 234 | 235 | # start: %Y-%m-%d, stop: %Y-%j, data: %Y-%j 236 | data, meta = hapi(s, d, p, start_ymd, stop_doy, **opts) 237 | compare(data1, data, meta1, meta, opts1, opts) 238 | 239 | # start: %Y-%j, stop: %Y-%m-%d, data: %Y-%j 240 | data, meta = hapi(s, d, p, start_doy, stop_ymd, **opts) 241 | compare(data1, data, meta1, meta, opts1, opts) 242 | 243 | # start: %Y-%j, stop: %Y-%j, data: %Y-%j 244 | data, meta = hapi(s, d, p, start_doy, stop_doy, **opts) 245 | compare(data1, data, meta1, meta, opts1, opts) 246 | 247 | 248 | if __name__ == '__main__': 249 | test_chunks() 250 | test_chunk_threshold() 251 | test_timeformats() 252 | -------------------------------------------------------------------------------- /hapiclient/test/compare.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import shutil 4 | 5 | from hapiclient import hapi 6 | 7 | debug = False 8 | 9 | def comparisonOK(a, b, nolength=False, a_name="First", b_name="Second"): 10 | 11 | if a.dtype != b.dtype: 12 | if debug: 13 | print('---- dts differ.') 14 | unicode_length_mismatch = False 15 | for i in range(len(a.dtype)): 16 | if a.dtype[i].str != b.dtype[i].str: 17 | if debug: 18 | print("--- {}".format(a_name)) 19 | print("--- {}".format(a.dtype[i].str)) 20 | print("--- {}".format(b_name)) 21 | print("--- {}".format(b.dtype[i].str)) 22 | # When length is given, the HAPI spec states that it should be 23 | # the number of bytes required to store any value of the parameter. 24 | # Greek characters require two bytes when encoded as UTF-8, so 25 | # length=2 if the parameter values are all single Greek character. 26 | # However, when NumPy is used to determine the data type of a 27 | # single UTF-8 encoded character because length is not given, 28 | # it uses "U1" (see hapi.py/parse_missing_length() and search 29 | # for ".astype('file read & parse file') 194 | xprint('_____________________________________________________________') 195 | else: 196 | xprint('___________________________________________________________') 197 | xprint('Method total d/l->buff parse buff') 198 | xprint('___________________________________________________________') 199 | 200 | 201 | opts['format'] = 'binary' 202 | opts['method'] = '' 203 | 204 | data0, meta = hapi(server, dataset, parameters, start, stop, **opts) 205 | xprint('binary %8.4f %8.4f %8.4f' % \ 206 | (meta['x_totalTime'], meta['x_downloadTime'], meta['x_readTime'])) 207 | 208 | opts['format'] = 'csv' 209 | 210 | opts['method'] = 'pandas' # Default CSV read method. 211 | data, meta = hapi(server, dataset, parameters, start, stop, **opts) 212 | xprint('csv; pandas %8.4f %8.4f %8.4f' % \ 213 | (meta['x_totalTime'], meta['x_downloadTime'], meta['x_readTime'])) 214 | 215 | allpass = allpass and comparisonOK(data0, data, a_name='binary', b_name='pandas') 216 | 217 | opts['method'] = 'pandasnolength' 218 | data, meta = hapi(server, dataset, parameters, start, stop, **opts) 219 | xprint('csv; pandas; no len. %8.4f %8.4f %8.4f' % \ 220 | (meta['x_totalTime'], meta['x_downloadTime'], meta['x_readTime'])) 221 | 222 | allpass = allpass and comparisonOK(data0, data, nolength=True, a_name='binary', b_name='csv; pandas; no len.') 223 | 224 | 225 | opts['method'] = 'numpy' 226 | data, meta = hapi(server, dataset, parameters, start, stop, **opts) 227 | xprint('csv; numpy %8.4f %8.4f %8.4f' % \ 228 | (meta['x_totalTime'], meta['x_downloadTime'], meta['x_readTime'])) 229 | 230 | allpass = allpass and comparisonOK(data0, data, a_name='binary', b_name='csv; numpy') 231 | 232 | opts['method'] = 'numpynolength' 233 | data, meta = hapi(server, dataset, parameters, start, stop, **opts) 234 | xprint('csv; numpy; no len. %8.4f %8.4f %8.4f' % \ 235 | (meta['x_totalTime'], meta['x_downloadTime'], meta['x_readTime'])) 236 | 237 | 238 | allpass = allpass and comparisonOK(data0, data, nolength=True, a_name='binary', b_name='csv; numpy; no len.') 239 | 240 | return allpass 241 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Test hapi() data read functions using repository code: 2 | # make repository-test PYTHON=_PYTHON_ # Test using _PYTHON_ (e.g, python3.6) 3 | # make repository-test-all # Test on all versions in $(PYTHONVERS) var below 4 | # 5 | # Beta releases: 6 | # 1. Run make repository-test-all 7 | # 2. For non-doc/formatting changes, update version in CHANGES.txt. 8 | # 3. run `make version-update` if version changed in CHANGES.txt. 9 | # 4. Commit and push 10 | # 11 | # Making a local package: 12 | # 1. Update CHANGES.txt to have a new version line 13 | # 2. make package 14 | # 3. make package-test-all 15 | # 16 | # Upload package to pypi.org 17 | # 0. Remove the "b" in the version in CHANGES.txt 18 | # 1. make release 19 | # 2. Wait ~5 minutes and execute 20 | # 3. make release-test-all 21 | # (Will fail until new version is available at pypi.org for pip install. 22 | # Sometimes takes ~5 minutes even though web page is immediately 23 | # updated.) 24 | # 4. After package is finalized, create new version number in CHANGES.txt ending 25 | # with "b0" in setup.py and then run 26 | # make version-update 27 | # git commit -a -m "Update version for next release" 28 | # This will update the version information in the repository to indicate it 29 | # is now in a pre-release state. 30 | # 5. Manually create a release at https://github.com/hapi-server/client-python/releases 31 | # (could do this automatically using https://stackoverflow.com/questions/21214562/how-to-release-versions-on-github-through-the-command-line) 32 | # Notes: 33 | # 1. make repository-test tests with Anaconda virtual environment 34 | # make package-test and release-test tests with native Python virtual 35 | # environment. 36 | # 2. Switch to using tox and conda-tox 37 | # 3. 'pip install --editable . does not install develop dependencies, so 38 | # 'python setup.py develop' is used. Won't need figure out when 2. is finished. 39 | 40 | URL=https://upload.pypi.org/ 41 | REP=pypi 42 | 43 | # Default Python version to use for tests 44 | PYTHON=python3.8 45 | PYTHON_VER=$(subst python,,$(PYTHON)) 46 | 47 | # Python versions to test 48 | PYTHONVERS=python3.13 python3.12 python3.11 python3.10 python3.9 python3.8 python3.7 python3.6 python3.5 49 | 50 | # VERSION is updated in "make version-update" step and derived 51 | # from CHANGES.txt. Do not edit. 52 | VERSION=0.2.7b1 53 | SHELL:= /bin/bash 54 | #SHELL:= /c/Windows/system32/cmd 55 | 56 | LONG_TESTS=false 57 | 58 | CONDA=$(PWD)/anaconda3 59 | 60 | ifeq ($(OS),Windows_NT) 61 | CONDA=C:/Users/weigel/git/client-python/anaconda3 62 | TMP=tmp/ 63 | endif 64 | CONDA_ACTIVATE=source $(CONDA)/etc/profile.d/conda.sh; conda activate 65 | 66 | ################################################################################ 67 | install: $(CONDA)/envs/$(PYTHON) 68 | $(CONDA_ACTIVATE) $(PYTHON); pip install --editable . 69 | @printf "\n--------------------------------------------------------------------------------\n" 70 | @printf "To use created Anaconda environment, execute\n $(CONDA_ACTIVATE) $(PYTHON)" 71 | @printf "\n--------------------------------------------------------------------------------\n" 72 | 73 | test: 74 | make repository-test-all 75 | 76 | #################################################################### 77 | # Tox notes 78 | # 79 | # Ideally local tests would use same commands as .tox.ini and .travis.yml. 80 | # 81 | # To use tox -e short-test, it seems we need to install and activate 82 | # each version of python. So using tox locally does not seem to make 83 | # things much simpler than `make repository-test`, which installs 84 | # conda and virtual enviornments and runs tests. 85 | # 86 | # However, Travis tests use tox-travis and it seems creation of 87 | # virtual environment is done automatically. 88 | # 89 | #repository-test-all-tox: 90 | # tox -e short-test 91 | #repository-test-tox: 92 | # # Does not work 93 | # tox -e py$(subst .,,$(PYTHON_VER)) short-test 94 | #################################################################### 95 | 96 | # Test contents in repository using different Python versions 97 | repository-test-all: 98 | @make clean 99 | #rm -rf $(CONDA) 100 | @ for version in $(PYTHONVERS) ; do \ 101 | make repository-test PYTHON=$$version ; \ 102 | done 103 | 104 | repository-test: 105 | @make clean 106 | make condaenv PYTHON=$(PYTHON) 107 | $(CONDA_ACTIVATE) $(PYTHON); pip install pytest deepdiff; pip install . 108 | 109 | ifeq (LONG_TESTS,true) 110 | $(CONDA_ACTIVATE) $(PYTHON); python -m pytest --tb=short -v -m 'long' hapiclient/test/test_hapi.py 111 | else 112 | $(CONDA_ACTIVATE) $(PYTHON); python -m pytest --tb=short -v -m 'short' hapiclient/test/test_hapi.py 113 | endif 114 | 115 | $(CONDA_ACTIVATE) $(PYTHON); python -m pytest -v hapiclient/test/test_chunking.py 116 | $(CONDA_ACTIVATE) $(PYTHON); python -m pytest -v hapiclient/test/test_hapitime2datetime.py 117 | $(CONDA_ACTIVATE) $(PYTHON); python -m pytest -v hapiclient/test/test_datetime2hapitime.py 118 | $(CONDA_ACTIVATE) $(PYTHON); python -m pytest -v hapiclient/test/test_hapitime_reformat.py 119 | ################################################################################ 120 | 121 | ################################################################################ 122 | # Anaconda install 123 | CONDA_PKG=Miniconda3-latest-Linux-x86_64.sh 124 | CONDA_PKG_PATH=/tmp/$(CONDA_PKG) 125 | ifeq ($(shell uname -s),Darwin) 126 | CONDA_PKG=Miniconda3-latest-MacOSX-x86_64.sh 127 | CONDA_PKG_PATH=/tmp/$(CONDA_PKG) 128 | endif 129 | ifeq ($(OS),Windows_NT) 130 | CONDA_PKG=Miniconda3-latest-Windows-x86_64.exe 131 | CONDA_PKG_PATH=C:/tmp/$(CONDA_PKG) 132 | endif 133 | 134 | activate: 135 | @echo "On command line enter:" 136 | @echo "$(CONDA_ACTIVATE) $(PYTHON)" 137 | 138 | condaenv: 139 | make $(CONDA)/envs/$(PYTHON) PYTHON=$(PYTHON) 140 | 141 | $(CONDA)/envs/$(PYTHON): $(CONDA) 142 | ifeq ($(OS),Windows_NT) 143 | $(CONDA_ACTIVATE); \ 144 | $(CONDA)/Scripts/conda \ 145 | create -y --name $(PYTHON) python=$(PYTHON_VER) 146 | else 147 | $(CONDA_ACTIVATE); \ 148 | $(CONDA)/bin/conda \ 149 | create -y --name $(PYTHON) python=$(PYTHON_VER) 150 | endif 151 | 152 | $(CONDA): $(CONDA_PKG_PATH) 153 | ifeq ($(OS),Windows_NT) 154 | # Not working; path is not set 155 | #start "$(CONDA_PKG_PATH)" /S /D=$(CONDA) 156 | echo "!!! Install miniconda3 into $(CONDA) manually by executing 'start $(PWD)/anaconda3'. Then re-execute make command." 157 | exit 1 158 | else 159 | test -d anaconda3 || bash $(CONDA_PKG_PATH) -b -p $(CONDA) 160 | endif 161 | 162 | $(CONDA_PKG_PATH): 163 | curl https://repo.anaconda.com/miniconda/$(CONDA_PKG) > $(CONDA_PKG_PATH) 164 | ################################################################################ 165 | 166 | ################################################################################ 167 | # Packaging 168 | package: 169 | make clean 170 | make version-update 171 | python setup.py sdist 172 | 173 | package-test-all: 174 | @ for version in $(PYTHONVERS) ; do \ 175 | make repository-test PYTHON=$$version ; \ 176 | done 177 | 178 | env-$(PYTHON): 179 | rm -rf env-$(PYTHON) 180 | $(CONDA_ACTIVATE) $(PYTHON); \ 181 | conda install -y virtualenv; \ 182 | $(PYTHON) -m virtualenv env-$(PYTHON) 183 | 184 | package-test: 185 | make package 186 | make env-$(PYTHON) 187 | make package-venv-test PACKAGE='dist/hapiclient-$(VERSION).tar.gz' 188 | 189 | package-venv-test: 190 | cp hapi_demo.py /tmp # TODO: Explain why needed. 191 | cp hapi_demo.py /tmp 192 | source env-$(PYTHON)/bin/activate && \ 193 | pip install pytest deepdiff ipython && \ 194 | pip uninstall -y hapiplot && \ 195 | pip install --pre hapiplot && \ 196 | pip uninstall -y hapiclient && \ 197 | pip install --pre '$(PACKAGE)' \ 198 | --index-url $(URL)/simple \ 199 | --extra-index-url https://pypi.org/simple && \ 200 | env-$(PYTHON)/bin/pytest -v -m 'short' hapiclient/test/test_hapi.py 201 | env-$(PYTHON)/bin/ipython /tmp/hapi_demo.py 202 | 203 | ################################################################################ 204 | # Release a package to pypi.org 205 | release: 206 | make package 207 | make version-tag 208 | make release-upload 209 | 210 | release-upload: 211 | pip install twine 212 | twine upload \ 213 | -r $(REP) dist/hapiclient-$(VERSION).tar.gz -u __token__ \ 214 | && echo Uploaded to $(subst upload.,,$(URL))/project/hapiclient/ 215 | 216 | release-test-all: 217 | @ for version in $(PYTHONVERS) ; do \ 218 | make release-test PYTHON=$$version ; \ 219 | done 220 | 221 | release-test: 222 | make env-$(PYTHON) 223 | make release-venv-test PYTHON=$(PYTHON) 224 | 225 | release-venv-test: 226 | cp hapi_demo.py /tmp # TODO: Explain why needed. 227 | cp hapi_demo.py /tmp 228 | source env-$(PYTHON)/bin/activate && \ 229 | pip install pytest deepdiff ipython && \ 230 | pip uninstall -y hapiplot && \ 231 | pip install --pre hapiplot && \ 232 | pip uninstall -y hapiclient && \ 233 | pip install hapiclient && \ 234 | env-$(PYTHON)/bin/pytest -v -m 'short' hapiclient/test/test_hapi.py && \ 235 | env-$(PYTHON)/bin/ipython /tmp/hapi_demo.py 236 | 237 | ################################################################################ 238 | # Update version based on content of CHANGES.txt 239 | version-update: 240 | python misc/version.py 241 | 242 | version-tag: 243 | - git commit -a -m "Last $(VERSION) commit" 244 | git push 245 | git tag -a v$(VERSION) -m "Version "$(VERSION) 246 | git push --tags 247 | ################################################################################ 248 | 249 | ################################################################################ 250 | # Install package in local directory (symlinks made to local dir) 251 | install-pip: 252 | pip install 'hapiclient==$(VERSION)' --index-url $(URL)/simple 253 | conda list | grep hapiclient 254 | pip list | grep hapiclient 255 | ################################################################################ 256 | 257 | ################################################################################ 258 | # Recreate reference response files. Use this if server response changes 259 | # Run pytest twice because first run creates test files that 260 | # subsequent tests use for comparison. 261 | test-clean: 262 | rm -f hapiclient/test/data/* 263 | pytest -v hapiclient/test/test_hapi.py 264 | pytest -v hapiclient/test/test_hapi.py 265 | ################################################################################ 266 | 267 | clean: 268 | - @find . -name __pycache__ | xargs rm -rf {} 269 | - @find . -name *.pyc | xargs rm -rf {} 270 | - @find . -name *.DS_Store | xargs rm -rf {} 271 | - @find . -type d -name __pycache__ | xargs rm -rf {} 272 | - @find . -name *.pyc | xargs rm -rf {} 273 | - @rm -f *~ 274 | - @rm -f \#*\# 275 | - @rm -rf env 276 | - @rm -rf dist 277 | - @rm -f MANIFEST 278 | - @rm -rf .pytest_cache/ 279 | - @rm -rf hapiclient.egg-info/ 280 | - @rm -rf /c/tools/miniconda3/envs/python3.6/Scripts/wheel.exe* 281 | - @rm -rf /c/tools/miniconda3/envs/python3.6/vcruntime140.dll.* 282 | -------------------------------------------------------------------------------- /hapiclient/test/test_chunking.log: -------------------------------------------------------------------------------- 1 | ________________________________________________________________________________ 2 | request : dataset3, scalar, 1971-01-01T01:50:00Z, 1975-08-03T06:50:00Z, P1D 3 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 4 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv'} 5 | x_totalTime1 = 0.2720 s 6 | x_totalTime2 = 0.2840 s 7 | ________________________________________________________________________________ 8 | request : dataset3, scalar, 1971-01-01T01:50:00Z, 1975-08-03T06:50:00Z, P1D 9 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 10 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'n_chunks': 2} 11 | x_totalTime1 = 0.2720 s 12 | x_totalTime2 = 0.7074 s 13 | ________________________________________________________________________________ 14 | request : dataset3, scalar, 1971-01-01T01:50:00Z, 1975-08-03T06:50:00Z, P1D 15 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 16 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': 'P1Y'} 17 | x_totalTime1 = 0.2720 s 18 | x_totalTime2 = 1.7951 s 19 | ________________________________________________________________________________ 20 | request : dataset3, scalar, 1971-01-01T01:50:00Z, 1975-08-03T06:50:00Z, P1D 21 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 22 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'parallel': True, 'n_chunks': 2} 23 | x_totalTime1 = 0.2720 s 24 | x_totalTime2 = 0.4590 s 25 | ________________________________________________________________________________ 26 | request : dataset3, scalar, 1971-01-01T01:50:00Z, 1975-08-03T06:50:00Z, P1D 27 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 28 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'parallel': True, 'dt_chunk': 'P1Y'} 29 | x_totalTime1 = 0.2720 s 30 | x_totalTime2 = 1.0061 s 31 | ________________________________________________________________________________ 32 | request : dataset3, scalar, 1971-01-01T01:50:00Z, 1971-08-03T06:50:00Z, P1D 33 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 34 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv'} 35 | x_totalTime1 = 0.4959 s 36 | x_totalTime2 = 0.2389 s 37 | ________________________________________________________________________________ 38 | request : dataset3, scalar, 1971-01-01T01:50:00Z, 1971-08-03T06:50:00Z, P1D 39 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 40 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'n_chunks': 2} 41 | x_totalTime1 = 0.4959 s 42 | x_totalTime2 = 0.5793 s 43 | ________________________________________________________________________________ 44 | request : dataset3, scalar, 1971-01-01T01:50:00Z, 1971-08-03T06:50:00Z, P1D 45 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 46 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': 'P1M'} 47 | x_totalTime1 = 0.4959 s 48 | x_totalTime2 = 2.4138 s 49 | ________________________________________________________________________________ 50 | request : dataset3, scalar, 1971-01-01T01:50:00Z, 1971-08-03T06:50:00Z, P1D 51 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 52 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'parallel': True, 'n_chunks': 2} 53 | x_totalTime1 = 0.4959 s 54 | x_totalTime2 = 0.4608 s 55 | ________________________________________________________________________________ 56 | request : dataset3, scalar, 1971-01-01T01:50:00Z, 1971-08-03T06:50:00Z, P1D 57 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 58 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'parallel': True, 'dt_chunk': 'P1M'} 59 | x_totalTime1 = 0.4959 s 60 | x_totalTime2 = 1.5068 s 61 | ________________________________________________________________________________ 62 | request : dataset2, scalar, 1970-01-01T00:00:00.000Z, 1970-01-10T00:00:00.000Z, PT1H 63 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 64 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv'} 65 | x_totalTime1 = 0.2835 s 66 | x_totalTime2 = 0.2982 s 67 | ________________________________________________________________________________ 68 | request : dataset2, scalar, 1970-01-01T00:00:00.000Z, 1970-01-10T00:00:00.000Z, PT1H 69 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 70 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'n_chunks': 2} 71 | x_totalTime1 = 0.2835 s 72 | x_totalTime2 = 0.6436 s 73 | ________________________________________________________________________________ 74 | request : dataset2, scalar, 1970-01-01T00:00:00.000Z, 1970-01-10T00:00:00.000Z, PT1H 75 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 76 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': 'P1D'} 77 | x_totalTime1 = 0.2835 s 78 | x_totalTime2 = 3.0275 s 79 | ________________________________________________________________________________ 80 | request : dataset2, scalar, 1970-01-01T00:00:00.000Z, 1970-01-10T00:00:00.000Z, PT1H 81 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 82 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'parallel': True, 'n_chunks': 2} 83 | x_totalTime1 = 0.2835 s 84 | x_totalTime2 = 0.3745 s 85 | ________________________________________________________________________________ 86 | request : dataset2, scalar, 1970-01-01T00:00:00.000Z, 1970-01-10T00:00:00.000Z, PT1H 87 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 88 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'parallel': True, 'dt_chunk': 'P1D'} 89 | x_totalTime1 = 0.2835 s 90 | x_totalTime2 = 1.7074 s 91 | ________________________________________________________________________________ 92 | request : dataset1, scalar, 1970-01-01T00:00:00.000Z, 1970-01-01T05:00:00.000Z, PT1S 93 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 94 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv'} 95 | x_totalTime1 = 0.3290 s 96 | x_totalTime2 = 0.5350 s 97 | ________________________________________________________________________________ 98 | request : dataset1, scalar, 1970-01-01T00:00:00.000Z, 1970-01-01T05:00:00.000Z, PT1S 99 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 100 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'n_chunks': 2} 101 | x_totalTime1 = 0.3290 s 102 | x_totalTime2 = 0.4166 s 103 | ________________________________________________________________________________ 104 | request : dataset1, scalar, 1970-01-01T00:00:00.000Z, 1970-01-01T05:00:00.000Z, PT1S 105 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 106 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': 'PT1H'} 107 | x_totalTime1 = 0.3290 s 108 | x_totalTime2 = 1.2239 s 109 | ________________________________________________________________________________ 110 | request : dataset1, scalar, 1970-01-01T00:00:00.000Z, 1970-01-01T05:00:00.000Z, PT1S 111 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 112 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'parallel': True, 'n_chunks': 2} 113 | x_totalTime1 = 0.3290 s 114 | x_totalTime2 = 0.5489 s 115 | ________________________________________________________________________________ 116 | request : dataset1, scalar, 1970-01-01T00:00:00.000Z, 1970-01-01T05:00:00.000Z, PT1S 117 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 118 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'parallel': True, 'dt_chunk': 'PT1H'} 119 | x_totalTime1 = 0.3290 s 120 | x_totalTime2 = 1.2479 s 121 | ________________________________________________________________________________ 122 | request : dataset1, scalar, 1970-01-01T00:00:00.000Z, 1970-01-01T08:00:00Z, PT1S 123 | options 1: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': None} 124 | options 2: {'logging': False, 'usecache': False, 'cache': True, 'format': 'csv', 'dt_chunk': 'infer'} 125 | x_totalTime1 = 0.3765 s 126 | x_totalTime2 = 0.3114 s 127 | ________________________________________________________________________________ 128 | request : dataset1, scalar, 1970-01-01T00:00:00.000Z, 1970-01-01T05:00:00.000Z, PT1S 129 | options 1: {'usecache': False, 'cache': False, 'dt_chunk': None} 130 | options 2: {'usecache': False, 'cache': False, 'dt_chunk': 'infer'} 131 | x_totalTime1 = 0.3379 s 132 | x_totalTime2 = 0.2895 s 133 | ________________________________________________________________________________ 134 | request : dataset1, scalar, 1970-01-01T00:00:00.000Z, 1970-01-01T05:00:00.000Z, PT1S 135 | options 1: {'usecache': False, 'cache': False, 'dt_chunk': None} 136 | options 2: {'usecache': False, 'cache': False, 'dt_chunk': 'infer'} 137 | x_totalTime1 = 0.3379 s 138 | x_totalTime2 = 0.3622 s 139 | ________________________________________________________________________________ 140 | request : dataset1, scalar, 1970-01-01T00:00:00.000Z, 1970-01-01T05:00:00.000Z, PT1S 141 | options 1: {'usecache': False, 'cache': False, 'dt_chunk': None} 142 | options 2: {'usecache': False, 'cache': False, 'dt_chunk': 'infer'} 143 | x_totalTime1 = 0.3379 s 144 | x_totalTime2 = 0.4851 s 145 | ________________________________________________________________________________ 146 | request : ace, X_GSM, 2000-01-01T00:00:00.000Z, 2000-01-02T00:00:00.000Z, PT720S 147 | options 1: {'usecache': False, 'cache': False, 'dt_chunk': None} 148 | options 2: {'usecache': False, 'cache': False, 'dt_chunk': 'infer'} 149 | x_totalTime1 = 1.1988 s 150 | x_totalTime2 = 2.2726 s 151 | ________________________________________________________________________________ 152 | request : ace, X_GSM, 2000-01-01T00:00:00.000Z, 2000-01-02T00:00:00.000Z, PT720S 153 | options 1: {'usecache': False, 'cache': False, 'dt_chunk': None} 154 | options 2: {'usecache': False, 'cache': False, 'dt_chunk': 'infer'} 155 | x_totalTime1 = 1.1988 s 156 | x_totalTime2 = 2.1587 s 157 | ________________________________________________________________________________ 158 | request : ace, X_GSM, 2000-01-01T00:00:00.000Z, 2000-01-02T00:00:00.000Z, PT720S 159 | options 1: {'usecache': False, 'cache': False, 'dt_chunk': None} 160 | options 2: {'usecache': False, 'cache': False, 'dt_chunk': 'infer'} 161 | x_totalTime1 = 1.1988 s 162 | x_totalTime2 = 2.2047 s 163 | -------------------------------------------------------------------------------- /hapiclient/test/test_hapi.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Above line can be removed when Python 2.7 support is dropped. 3 | import os 4 | import pytest 5 | import pickle 6 | import shutil 7 | 8 | import numpy as np 9 | 10 | from deepdiff import DeepDiff 11 | from hapiclient.hapi import hapi 12 | from hapiclient.test import compare 13 | 14 | # To run tests on a specific function, edit the function calls in the 15 | # if __name__ == '__main__' block and then execute 16 | # 17 | # python test_hapi.py 18 | # 19 | # See comments in test_hapitime2datetime.py for other execution options. 20 | 21 | logging = False 22 | serverbad = 'http://hapi-server.org/servers/TestData/xhapi' 23 | server = 'http://hapi-server.org/servers/TestData2.0/hapi' 24 | 25 | def writepickle(fname, var): 26 | print("!!!!!!!!!!!!!!") 27 | print("Writing " + fname) 28 | print("!!!!!!!!!!!!!!") 29 | with open(fname, 'wb') as pickle_file: 30 | pickle.dump(var, pickle_file, protocol=2) 31 | pickle_file.close() 32 | 33 | 34 | def readpickle(fname): 35 | with open(fname, 'rb') as pickle_file: 36 | var = pickle.load(pickle_file) 37 | pickle_file.close() 38 | return var 39 | 40 | 41 | @pytest.mark.short 42 | def test_hapi(): 43 | """Test that a call with no parameters returns something.""" 44 | assert hapi() is not None 45 | 46 | 47 | @pytest.mark.short 48 | def test_server_list(): 49 | """Test that specifying a server returns something.""" 50 | assert hapi(server) is not None 51 | 52 | 53 | @pytest.mark.short 54 | def test_catalog(): 55 | """Request for catalog returns correct status and first dataset""" 56 | meta = hapi(server) 57 | assert meta['status'] == {'code': 1200, 'message': 'OK'} and meta['catalog'][0]['id'] == 'dataset1' 58 | 59 | 60 | @pytest.mark.short 61 | def test_dataset(): 62 | """Request for dataset returns correct dataset metadata""" 63 | meta = hapi(server,'dataset1') 64 | pklFile = 'test_dataset.pkl' 65 | pklFile = os.path.join(os.path.dirname(os.path.realpath(__file__)),'data',pklFile) 66 | if not os.path.isfile(pklFile): 67 | writepickle(pklFile, meta) 68 | assert True 69 | return 70 | else: 71 | metatest = readpickle(pklFile) 72 | assert DeepDiff(meta,metatest) == {} 73 | 74 | 75 | @pytest.mark.short 76 | def test_parameter(): 77 | """Request for dataset returns correct parameter metadata""" 78 | meta = hapi(server,'dataset1') 79 | pklFile = 'test_parameter.pkl' 80 | pklFile = os.path.join(os.path.dirname(os.path.realpath(__file__)),'data',pklFile) 81 | if not os.path.isfile(pklFile): 82 | writepickle(pklFile,meta) 83 | assert True 84 | return 85 | else: 86 | metatest = readpickle(pklFile) 87 | assert DeepDiff(meta,metatest) == {} 88 | 89 | 90 | @pytest.mark.short 91 | def test_bad_server_url(): 92 | """Correct error when given bad URL""" 93 | with pytest.raises(Exception): 94 | hapi(serverbad, {'logging': logging}) 95 | 96 | 97 | @pytest.mark.short 98 | def test_bad_dataset_name(): 99 | """Correct error when given nonexistent dataset name""" 100 | with pytest.raises(Exception): 101 | hapi(server,'dataset1x') 102 | 103 | 104 | @pytest.mark.short 105 | def test_bad_parameter(): 106 | """Correct error when given nonexistent parameter name""" 107 | with pytest.raises(Exception): 108 | hapi(server,'dataset1','scalarx') 109 | 110 | 111 | @pytest.mark.short 112 | def test_reader_short(): 113 | 114 | dataset = 'dataset1' 115 | run = 'short' 116 | 117 | opts = {'logging': logging, 'cachedir': '/tmp/hapi-data', 'usecache': False} 118 | 119 | opts['cache'] = False 120 | 121 | # Read one parameter 122 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 123 | assert compare.read(server, dataset, 'scalar', run, opts) 124 | 125 | # Read two parameters 126 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 127 | assert compare.read(server, dataset, 'scalar,vector', run, opts) 128 | 129 | # Read all parameters 130 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 131 | assert compare.read(server, dataset, '', run, opts) 132 | 133 | # Cache = True (will write files then read) 134 | opts['cache'] = True 135 | 136 | # Read one parameter 137 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 138 | assert compare.read(server, dataset, 'scalar', run, opts) 139 | 140 | # Read two parameters 141 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 142 | assert compare.read(server, dataset, 'scalar,vector', run, opts) 143 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 144 | 145 | # Read all parameters 146 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 147 | assert compare.read(server, dataset, '', run, opts) 148 | 149 | 150 | @pytest.mark.short 151 | def test_cache_short(): 152 | 153 | # Compare read with empty cache with read with hot cache and usecache=True 154 | dataset = 'dataset1' 155 | start = '1970-01-01' 156 | stop = '1970-01-01T00:00:03' 157 | 158 | opts = {'logging': logging, 'cachedir': '/tmp/hapi-data', 'cache': True} 159 | 160 | opts['usecache'] = False 161 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 162 | data, meta = hapi(server, dataset, 'scalarint,vectorint', start, stop, **opts) 163 | 164 | opts['usecache'] = True 165 | data2, meta2 = hapi(server, dataset, 'scalarint,vectorint', start, stop, **opts) 166 | 167 | assert np.array_equal(data, data2) 168 | 169 | 170 | @pytest.mark.short 171 | def test_subset_short(): 172 | 173 | dataset = 'dataset1' 174 | start = '1970-01-01' 175 | stop = '1970-01-01T00:00:03' 176 | opts = {'logging': logging, 'cachedir': '/tmp/hapi-data', 'cache': True} 177 | 178 | opts['usecache'] = False 179 | 180 | # Request two subsets with empty cache. Common parts should be same. 181 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 182 | data, meta = hapi(server, dataset, 'scalarint', start, stop, **opts) 183 | 184 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 185 | data2, meta2 = hapi(server, dataset, 'scalarint,vectorint', start, stop, **opts) 186 | 187 | ok = np.array_equal(data['Time'], data2['Time']) 188 | ok = ok and np.array_equal(data['scalarint'], data2['scalarint']) 189 | assert ok 190 | 191 | # Request all parameters and single parameter. Common parameter should be same. 192 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 193 | data, meta = hapi(server, dataset, '', start, stop, **opts) 194 | 195 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 196 | data2, meta2 = hapi(server, dataset, 'vectorint', start, stop, **opts) 197 | 198 | ok = np.array_equal(data['Time'], data2['Time']) 199 | ok = ok and np.array_equal(data['vectorint'], data2['vectorint']) 200 | assert ok 201 | 202 | 203 | opts['usecache'] = True 204 | 205 | # Request two subsets, with the second request using the cache. Common 206 | # parts should be same. 207 | data, meta = hapi(server, dataset, 'scalarint', start, stop, **opts) 208 | data2, meta2 = hapi(server, dataset, 'scalarint,vectorint', start, stop, **opts) 209 | 210 | ok = np.array_equal(data['Time'], data2['Time']) 211 | ok = ok and np.array_equal(data['scalarint'], data2['scalarint']) 212 | assert ok 213 | 214 | # Request all parameters and single parameter, with the single parameter 215 | # request using the cache with hot cache. Common parameter should be same. 216 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 217 | data, meta = hapi(server, dataset, '', start, stop, **opts) 218 | data2, meta2 = hapi(server, dataset, 'vectorint', start, stop, **opts) 219 | 220 | ok = np.array_equal(data['Time'], data2['Time']) 221 | ok = ok and np.array_equal(data['vectorint'], data2['vectorint']) 222 | assert ok 223 | 224 | @pytest.mark.short 225 | def test_request2path(): 226 | 227 | from hapiclient.hapi import request2path; 228 | 229 | import platform 230 | if platform.system() == 'Windows': 231 | p = request2path('http://server/dir1/dir2','xx','abc<>:"/|?*.','2000-01-01T00:00:00.Z','2000-01-01T00:00:00.Z','') 232 | assert p == 'server_dir1_dir2\\xx_abc@lt@@gt@@colon@@doublequote@@forwardslash@@pipe@@questionmark@@asterisk@._20000101T000000_20000101T000000' 233 | else: 234 | p = request2path('http://server/dir1/dir2','xx/yy','abc/123','2000-01-01T00:00:00.Z','2000-01-01T00:00:00.Z','') 235 | assert p == 'server_dir1_dir2/xx@forwardslash@yy_abc@forwardslash@123_20000101T000000_20000101T000000' 236 | 237 | 238 | @pytest.mark.short 239 | def test_unicode(): 240 | 241 | from hapiclient.util import warning, unicode_error_message 242 | 243 | server = 'http://hapi-server.org/servers/TestData3.1/hapi'; 244 | #datasets = ["dataset1", "dataset1-Zα☃"] 245 | datasets = ["dataset1-Aα☃"] 246 | #datasets = ["dataset1"] 247 | 248 | run = 'short' 249 | 250 | opts = { 251 | 'logging': True, 252 | 'cachedir': '/tmp/hapi-data', 253 | 'usecache': False, 254 | 'cache': False 255 | } 256 | 257 | 258 | shutil.rmtree(opts['cachedir'], ignore_errors=True) 259 | for dataset in datasets: 260 | if unicode_error_message(dataset) != "": 261 | warning("Skipping "+ str(dataset.encode('utf-8')) + " due to " + unicode_error_message(dataset)) 262 | continue 263 | meta = hapi(server, dataset) 264 | for p in meta['parameters']: 265 | 266 | # Read one parameter 267 | parameter = p['name'] 268 | if unicode_error_message(parameter) != "": 269 | warning("Skipping "+ str(parameter.encode('utf-8')) + " due to " + unicode_error_message(parameter)) 270 | continue 271 | 272 | assert compare.read(server, dataset, parameter, run, opts.copy()) 273 | assert compare.cache(server, dataset, parameter, opts.copy()) 274 | 275 | 276 | @pytest.mark.long 277 | def test_reader_long(): 278 | 279 | dataset = 'dataset1' 280 | run = 'long' 281 | 282 | # Read three parameters 283 | opts = {'logging': logging, 'cachedir': '/tmp/hapi-data', 'cache': False, 'usecache': False} 284 | assert compare.read(server, dataset, 'scalar,vector,spectra', run, opts) 285 | 286 | opts = {'logging': logging, 'cachedir': '/tmp/hapi-data', 'cache': True, 'usecache': False} 287 | assert compare.read(server, dataset, 'scalar,vector,spectra', run, opts) 288 | 289 | opts = {'logging': logging, 'cachedir': '/tmp/hapi-data', 'cache': False, 'usecache': True} 290 | assert compare.read(server, dataset, 'scalar,vector,spectra', run, opts) 291 | 292 | 293 | @pytest.mark.short 294 | def test_none_stop(): 295 | import numpy as np 296 | 297 | from hapiclient import hapi 298 | from hapiclient import hapitime2datetime 299 | from hapiclient import datetime2hapitime 300 | from datetime import timedelta 301 | 302 | server = 'http://hapi-server.org/servers/TestData2.0/hapi' 303 | dataset = 'dataset1' 304 | parameters = 'scalar' 305 | 306 | meta = hapi(server, dataset) 307 | stop = meta['stopDate'] 308 | stop_dt = hapitime2datetime(stop)[0] 309 | 310 | start_dt = stop_dt - timedelta(minutes=1) 311 | start = datetime2hapitime(start_dt) 312 | 313 | data1, meta1 = hapi(server, dataset, parameters, start, None) 314 | 315 | data2, meta2 = hapi(server, dataset, parameters, start, stop) 316 | 317 | allequal = True 318 | for name in data1.dtype.names: 319 | assert np.array_equal(data1[name], data2[name]) 320 | 321 | 322 | def runall(): 323 | from hapiclient.test import test_hapi 324 | for i in dir(test_hapi): 325 | item = getattr(test_hapi,i) 326 | if callable(item) and item.__name__.startswith("test_"): 327 | if item.__name__ == 'test_reader_long': 328 | continue 329 | print("Running " + item.__name__) 330 | item() 331 | 332 | 333 | if __name__ == '__main__': 334 | #runall() 335 | #test_dataset() 336 | #test_reader_short() 337 | #test_unicode() 338 | #test_request2path() 339 | test_reader_short() 340 | #test_none_stop() 341 | -------------------------------------------------------------------------------- /hapiclient/hapitime.py: -------------------------------------------------------------------------------- 1 | """Functions for manipulating HAPI times (restricted ISO 8601 strings).""" 2 | import re 3 | import time 4 | 5 | import pandas 6 | import isodate 7 | import numpy as np 8 | 9 | from hapiclient.util import error, log 10 | 11 | def hapitime_reformat(form_to_match, given_form, logging=False): 12 | """Reformat a given HAPI time to match format of another HAPI time. 13 | 14 | ``hapitime_reformat(match, given)`` truncates or pads ``given`` so that it has 15 | the same format as ``match``. 16 | 17 | This function allows for efficient subsetting of arrays of HAPI time 18 | strings. For example, to select all time elements after a time of ``start``, 19 | first convert ``start`` so that it has the same format as the elements of 20 | ``data['Time']`` 21 | 22 | :: 23 | 24 | start = hapitime_reformat(data['Time'][0], start) 25 | 26 | Then subset using 27 | 28 | :: 29 | 30 | data = data[data['Time'] >= start] 31 | 32 | This is much more efficient than converting ``data['Time']`` to ``datetime`` 33 | objects and using ``datetime`` comparsion methods. 34 | 35 | Examples 36 | -------- 37 | :: 38 | from hapiclient.hapitime import hapitime_reformat 39 | hapitime_reformat('1989Z', '1989-01Z') # 1989Z 40 | hapitime_reformat('1989-001T00:00Z', '1999-01-21Z') # 1999-021T00:00Z 41 | 42 | """ 43 | 44 | log('ref: {}'.format(form_to_match), {'logging': logging}) 45 | log('given: {}'.format(given_form), {'logging': logging}) 46 | 47 | if 'T' in given_form: 48 | dt_given = isodate.parse_datetime(given_form) 49 | else: 50 | # Remove trailing Z b/c parse_date does not implement of date with 51 | # trailing Z, which is valid IS8601. 52 | dt_given = isodate.parse_date(given_form[0:-1]) 53 | 54 | # Get format string, e.g., %Y-%m-%dT%H 55 | format_ref = hapitime_format_str([form_to_match]) 56 | 57 | if '%f' in format_ref: 58 | form_to_match = form_to_match.strip('Z') 59 | form_to_match_fractional = form_to_match.split('.')[-1] 60 | form_to_match = ''.join(form_to_match.split('.')[:-1]) 61 | 62 | given_form_fractional = '000000000' 63 | given_form_fmt = hapitime_format_str([given_form]) 64 | given_form = given_form.strip('Z') 65 | 66 | if '%f' in given_form_fmt: 67 | given_form_fractional = given_form.split('.')[-1] 68 | given_form = ''.join(given_form.split('.')[:-1]) 69 | 70 | converted = hapitime_reformat(form_to_match+'Z', given_form+'Z') 71 | converted = converted.strip('Z') 72 | 73 | converted_fractional = '{:0<{}.{}}'.format(given_form_fractional, 74 | len(form_to_match_fractional), 75 | len(form_to_match_fractional)) 76 | converted = converted + '.' + converted_fractional 77 | 78 | if 'Z' in format_ref: 79 | return converted + 'Z' 80 | 81 | return converted 82 | 83 | converted = dt_given.strftime(format_ref) 84 | 85 | if len(converted) > len(form_to_match): 86 | converted = converted[0:len(form_to_match)-1] + "Z" 87 | 88 | log('converted: {}'.format(converted), {'logging': logging}) 89 | log('ref fmt: {}'.format(format_ref), {'logging': logging}) 90 | log('----', {'logging': logging}) 91 | 92 | return converted 93 | 94 | 95 | def hapitime_format_str(Time): 96 | """Determine the time format string for a HAPI time. 97 | Examples 98 | -------- 99 | :: 100 | from hapiclient.hapitime import hapitime_reformat 101 | hapitime_format_str(['1989Z']) # %YZ 102 | hapitime_format_str(['1989-347Z']) # %Y-%jZ 103 | hapitime_format_str(['2002-03-04T05:06Z']) # '%Y-%m-%dT%H:%MZ' 104 | """ 105 | 106 | d = 0 107 | # Catch case where no trailing Z 108 | # Technically HAPI ISO 8601 must have trailing Z; See 109 | # https://github.com/hapi-server/data-specification/blob/master/ 110 | # hapi-dev/HAPI-data-access-spec-dev.md#representation-of-time 111 | if not re.match(r".*Z$", Time[0]): 112 | d = 1 113 | 114 | # Parse date part 115 | # If h=True then hour given. 116 | # If hm=True, then hour and minute given. 117 | # If hms=True, them hour, minute, and second given. 118 | (h, hm, hms) = (False, False, False) 119 | 120 | if len(Time[0]) == 4 or (len(Time[0]) == 5 and Time[0][-1] == "Z"): 121 | fmt = '%Y' 122 | elif re.match(r"[0-9]{4}-[0-9]{3}", Time[0]): 123 | # YYYY-DOY format 124 | fmt = "%Y-%j" 125 | if len(Time[0]) >= 12 - d: 126 | h = True 127 | if len(Time[0]) >= 15 - d: 128 | hm = True 129 | if len(Time[0]) >= 18 - d: 130 | hms = True 131 | elif re.match(r"[0-9]{4}-[0-9]{2}", Time[0]): 132 | # YYYY-MM-DD format 133 | fmt = "%Y-%m" 134 | if len(Time[0]) > 8: 135 | fmt = fmt + "-%d" 136 | if len(Time[0]) >= 14 - d: 137 | h = True 138 | if len(Time[0]) >= 17 - d: 139 | hm = True 140 | if len(Time[0]) >= 20 - d: 141 | hms = True 142 | else: 143 | # TODO: Also check for invalid time string lengths. Use JSON schema 144 | # regular expressions for allowed versions of ISO 8601. 145 | # https://github.com/hapi-server/verifier-nodejs/tree/master/schemas 146 | error('First time value %s is not a valid HAPI Time' % Time[0]) 147 | 148 | if h: 149 | fmt = fmt + "T%H" 150 | if hm: 151 | fmt = fmt + ":%M" 152 | if hms: 153 | fmt = fmt + ":%S" 154 | 155 | if re.match(r".*\.[0-9].*$", Time[0]): 156 | fmt = fmt + ".%f" 157 | if re.match(r".*\.$", Time[0]) or re.match(r".*\.Z$", Time[0]): 158 | fmt = fmt + "." 159 | 160 | if re.match(r".*Z$", Time[0]): 161 | fmt = fmt + "Z" 162 | 163 | return fmt 164 | 165 | 166 | def hapitime2datetime(Time, **kwargs): 167 | """Convert HAPI timestamps to Python datetimes. 168 | 169 | A HAPI-compliant server represents time as an ISO 8601 string 170 | (with several constraints - see the `HAPI specification 171 | `_) 172 | 173 | `hapi()` reads these time strings into a NumPy array of Python byte literals. 174 | This function converts these byte literals to Python datetime objects. 175 | 176 | Typical usage: 177 | 178 | :: 179 | 180 | data = hapi(...) # Get data 181 | DateTimes = hapitime2datetime(data['Time']) # Convert 182 | 183 | 184 | All HAPI time strings must have a trailing Z. This function only checks the 185 | first element in Time array for compliance. 186 | 187 | Parameter 188 | --------- 189 | Time: 190 | - A numpy array of HAPI timestamp byte literals 191 | - A numpy array of HAPI timestamp strings 192 | - A list of HAPI timestamp byte literals 193 | - A list of HAPI timestamp strings 194 | - A HAPI timestamp byte literal 195 | - A HAPI timestamp strings 196 | 197 | Returns 198 | ------- 199 | A NumPy array Python of datetime objects with length = len(Time) 200 | 201 | Examples 202 | -------- 203 | All of the following return 204 | 205 | :: 206 | 207 | array([datetime.datetime(1970, 1, 1, 0, 0, tzinfo=)], dtype=object) 208 | 209 | :: 210 | 211 | from hapiclient.hapitime import hapitime2datetime 212 | import numpy as np 213 | 214 | hapitime2datetime(np.array([b'1970-01-01T00:00:00.000Z'])) # array([datetime.datetime(1970, 1, 1, 0, 0, tzinfo=)], dtype=object) 215 | hapitime2datetime(np.array(['1970-01-01T00:00:00.000Z'])) 216 | 217 | hapitime2datetime([b'1970-01-01T00:00:00.000Z']) 218 | hapitime2datetime(['1970-01-01T00:00:00.000Z']) 219 | 220 | hapitime2datetime([b'1970-01-01T00:00:00.000Z']) 221 | hapitime2datetime('1970-01-01T00:00:00.000Z') 222 | """ 223 | from datetime import datetime 224 | 225 | try: 226 | # Python 2 227 | import pytz 228 | tzinfo = pytz.UTC 229 | except: 230 | tzinfo = datetime.timezone.utc 231 | 232 | if type(Time) == list: 233 | Time = np.asarray(Time) 234 | if not all(list( map(lambda x: type(x) in [np.str_, np.bytes_, str, bytes], Time) )): 235 | raise ValueError 236 | 237 | allow_missing_Z = False 238 | if 'allow_missing_Z' in kwargs and kwargs['allow_missing_Z'] == True: 239 | allow_missing_Z = True 240 | 241 | opts = kwargs.copy() 242 | 243 | if type(Time) == list: 244 | Time = np.asarray(Time) 245 | 246 | if type(Time) != np.ndarray: 247 | Time = np.asarray([Time]) 248 | 249 | if Time.size == 0: 250 | error('Time array is empty.' + '\n') 251 | return 252 | 253 | reshape = False 254 | if Time.shape[0] != Time.size: 255 | reshape = True 256 | shape = Time.shape 257 | Time = Time.flatten() 258 | 259 | if type(Time[0]) == np.bytes_: 260 | try: 261 | Time = Time.astype('U') 262 | except: 263 | error('Problem with time data. First value: ' + str(Time[0]) + '\n') 264 | return 265 | 266 | tic = time.time() 267 | 268 | 269 | if Time[0][-1] != "Z" and allow_missing_Z == False: 270 | error("HAPI Times must have trailing Z. First element of input " + \ 271 | "Time array does not have trailing Z.") 272 | 273 | try: 274 | # This is the fastest conversion option. But it will fail on YYYY-DOY 275 | # format and other valid^* ISO 8601 dates such as 2001-01-01T00:00:03.Z 276 | # ^*Maybe not: https://github.com/hapi-server/client-python/issues/76 277 | # When infer_datetime_format is used, a TimeStamp object returned, 278 | # which is the reason for the to_pydatetime() call. (When format=... is 279 | # used, a datetime object is returned.) 280 | # Although all HAPI timestamps will have trailing Z, in some cases, 281 | # infer_datetime_format will not return a timezone-aware Timestamp. This 282 | # is the reason for the call to tz_convert(tzinfo). 283 | # TODO: Use hapitime_format_str() and pass this as format=... 284 | Timeo = Time[0] 285 | pandas_major_version = int(pandas.__version__.split('.')[0]) 286 | if pandas_major_version < 2: 287 | Time = pandas.to_datetime(Time, infer_datetime_format=True).tz_convert(tzinfo).to_pydatetime() 288 | else: 289 | Time = pandas.to_datetime(Time).tz_convert(tzinfo).to_pydatetime() 290 | if reshape: 291 | Time = np.reshape(Time, shape) 292 | toc = time.time() - tic 293 | log("Pandas processing time = %.4fs, first time = %s" % (toc, Timeo), opts) 294 | return Time 295 | except: 296 | log("Pandas processing failed, first time = %s" % Time[0], opts) 297 | 298 | 299 | # Convert from Python byte literals to unicode strings 300 | # https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.astype.html 301 | # https://www.b-list.org/weblog/2017/sep/05/how-python-does-unicode/ 302 | Time = Time.astype('U') 303 | # The new Time variable requires 4x more memory. 304 | # Could save memory at cost of speed by decoding at each iteration below, e.g. 305 | # Time[i] -> Time[i].decode('utf-8') 306 | 307 | pythonDateTime = np.empty(len(Time), dtype=object) 308 | 309 | fmt = hapitime_format_str(Time) 310 | 311 | # TODO: Will using pandas.to_datetime here with fmt work? 312 | try: 313 | parse_error = True 314 | for i in range(0, len(Time)): 315 | if Time[i][-1] != "Z" and allow_missing_Z == False: 316 | parse_error = False 317 | raise 318 | pythonDateTime[i] = datetime.strptime(Time[i], fmt).replace(tzinfo=tzinfo) 319 | except: 320 | if parse_error: 321 | error('Could not parse time value ' + Time[i] + ' using ' + fmt) 322 | else: 323 | error("HAPI Times must have trailing Z. Time[" + str(i) + "] = " \ 324 | + Time[i] + " does not have trailing Z.") 325 | 326 | toc = time.time() - tic 327 | log("Manual processing time = %.4fs, Input = %s, fmt = %s" % \ 328 | (toc, Time[0], fmt), opts) 329 | 330 | if reshape: 331 | pythonDateTime = np.reshape(pythonDateTime, shape) 332 | 333 | return pythonDateTime 334 | 335 | 336 | def datetime2hapitime(dts): 337 | """Convert Python datetime object(s) to ISO 8601 string(s) 338 | 339 | Typical usage: 340 | :: 341 | 342 | from hapiclient import datetime2hapitime 343 | import datetime 344 | dts = [datetime.datetime(2000, 1, 1), datetime.datetime(2000, 1, 2)] 345 | hapi_times = datetime2hapitime(dts) 346 | print(hapi_times) 347 | # ['2000-01-01T00:00:00.000000Z', '2000-01-02T00:00:00.000000Z'] 348 | 349 | Parameter 350 | --------- 351 | dts: 352 | - A Python datetime object 353 | - A list of Python datetime object(s) 354 | 355 | Returns 356 | ------- 357 | - A ISO 8601 string (if input is Python datetime object) 358 | - A list of ISO 8601 strings (if input is list of Python datetime object) 359 | """ 360 | 361 | single = False 362 | if isinstance(dts, list) == False: 363 | single = True 364 | dts = [dts] 365 | 366 | hapi_times = [dt.strftime('%Y-%m-%dT%H:%M:%S.%fZ') for dt in dts] 367 | 368 | if single == True: 369 | return hapi_times[0] 370 | else: 371 | return hapi_times 372 | -------------------------------------------------------------------------------- /hapiclient/util.py: -------------------------------------------------------------------------------- 1 | def setopts(defaults, given): 2 | """Override default keyword dictionary options. 3 | 4 | kwargs = setopts(defaults, kwargs) 5 | 6 | A warning is shown if kwargs contains a key not found in default. 7 | """ 8 | 9 | # Override defaults 10 | for key, value in given.items(): 11 | if type(given[key]) == dict: 12 | setopts(defaults[key], given[key]) 13 | continue 14 | if key in defaults: 15 | defaults[key] = value 16 | else: 17 | warning('Ignoring invalid keyword option "%s".' % key) 18 | 19 | return defaults 20 | 21 | 22 | def log_test(): 23 | 24 | log("Test 1", {"logging": True}) 25 | log("Test 2", {"logging": False}) 26 | 27 | 28 | def log(msg, opts): 29 | """Print message to console or file.""" 30 | 31 | import os 32 | import sys 33 | 34 | if not 'logging' in opts: 35 | opts = opts.copy() 36 | opts['logging'] = False 37 | 38 | pre = sys._getframe(1).f_code.co_name + '(): ' 39 | if isinstance(opts['logging'], bool) and opts['logging']: 40 | if pythonshell() == 'jupyter-notebook': 41 | # Don't show full path information. 42 | msg = msg.replace(opts['cachedir'] + os.path.sep, '') 43 | msg = msg.replace(opts['cachedir'], '') 44 | print(pre + msg) 45 | elif hasattr(opts['logging'], 'write'): 46 | opts['logging'].write(pre + msg + "\n") 47 | opts['logging'].flush() 48 | else: 49 | pass # TODO: error 50 | 51 | 52 | def jsonparse(res, url): 53 | """Try/catch of json.loads() function with short error message.""" 54 | 55 | from json import loads 56 | try: 57 | return loads(res.read().decode('utf-8')) 58 | except: 59 | error('Could not parse JSON from %s' % url) 60 | 61 | 62 | def pythonshell(): 63 | """Determine python shell 64 | 65 | pythonshell() returns 66 | 67 | 'shell' if started python on command line using "python" 68 | 'ipython' if started ipython on command line using "ipython" 69 | 'ipython-notebook' if running in Spyder or started with "ipython qtconsole" 70 | 'jupyter-notebook' if running in a Jupyter notebook started using executable 71 | named jupyter-notebook 72 | 73 | On Windows, jupyter-notebook cannot be detected and ipython-notebook 74 | will be returned. 75 | 76 | See also https://stackoverflow.com/a/37661854 77 | """ 78 | 79 | import os 80 | 81 | env = os.environ 82 | 83 | program = '' 84 | if '_' in env: 85 | program = os.path.basename(env['_']) 86 | 87 | shell = 'shell' 88 | try: 89 | shell_name = get_ipython().__class__.__name__ 90 | if shell_name == 'TerminalInteractiveShell': 91 | shell = 'ipython' 92 | elif shell_name == 'ZMQInteractiveShell': 93 | if 'jupyter-notebook' in program: 94 | shell = 'jupyter-notebook' 95 | else: 96 | shell = 'ipython-notebook' 97 | # Not needed, but could be used 98 | #if 'spyder' in sys.modules: 99 | # shell = 'spyder-notebook' 100 | except: 101 | pass 102 | 103 | return shell 104 | 105 | 106 | def unicode_error_message(name): 107 | import sys 108 | msg = "" 109 | if sys.version_info[0:2] <= (3, 5): 110 | if not all(ord(char) < 128 for char in name): 111 | msg = "hapiclient cannot handle Unicode dataset or parameter names (" + str(name.encode('utf-8')) + ") for Python < 3.6 on Windows." 112 | return msg 113 | 114 | 115 | def warning_test(): 116 | """For testing warning function.""" 117 | 118 | # Should show warnings in order and only HAPIWarning {1,2} should 119 | # have a different format 120 | from warnings import warn 121 | 122 | warn('Normal warning 1') 123 | warn('Normal warning 2') 124 | 125 | warning('HAPI Warning 1') 126 | warning('HAPI Warning 2') 127 | 128 | warn('Normal warning 3') 129 | warn('Normal warning 4') 130 | 131 | 132 | def warning(*args): 133 | """Display a short warning message. 134 | 135 | warning(message) raises a warning of type HAPIWarning and displays 136 | "Warning: " + message. Use for warnings when a full stack trace is not 137 | needed. 138 | """ 139 | 140 | import warnings 141 | from os import path 142 | from sys import stderr 143 | from inspect import stack 144 | 145 | message = args[0] 146 | if len(args) > 1: 147 | fname = args[1] 148 | else: 149 | fname = stack()[1][1] 150 | 151 | #line = stack()[1][2] 152 | 153 | fname = path.basename(fname) 154 | 155 | def prefix(): 156 | import platform 157 | prefix = "\x1b[31mHAPIWarning:\x1b[0m " 158 | if platform.system() == 'Windows' and pythonshell() == 'shell': 159 | prefix = "HAPIWarning: " 160 | 161 | return prefix 162 | 163 | # Custom warning format function 164 | def _warning(message, category=UserWarning, filename='', lineno=-1, file=None, line=''): 165 | if category.__name__ == "HAPIWarning": 166 | stderr.write(prefix() + str(message) + "\n") 167 | else: 168 | # Use default showwarning function. 169 | showwarning_default(message, category=UserWarning, 170 | filename='', lineno=-1, 171 | file=None, line='') 172 | 173 | stderr.flush() 174 | 175 | # Reset showwarning function to default 176 | warnings.showwarning = showwarning_default 177 | 178 | class HAPIWarning(Warning): 179 | pass 180 | 181 | # Copy default showwarning function 182 | showwarning_default = warnings.showwarning 183 | 184 | # Use custom warning function instead of default 185 | warnings.showwarning = _warning 186 | 187 | # Raise warning 188 | warnings.warn(message, HAPIWarning) 189 | 190 | 191 | class HAPIError(Exception): 192 | pass 193 | 194 | 195 | def error(msg, debug=False): 196 | """Display a short error message. 197 | 198 | error(message) raises an error of type HAPIError and displays 199 | "Error: " + message. Use for errors when a full stack trace is not needed. 200 | 201 | If debug=True, full stack trace is shown. 202 | """ 203 | 204 | import sys 205 | from inspect import stack 206 | from os import path 207 | 208 | debug = False 209 | if pythonshell() != 'shell': 210 | try: 211 | from IPython.core.interactiveshell import InteractiveShell 212 | except: 213 | pass 214 | 215 | sys.stdout.flush() 216 | 217 | fname = stack()[1][1] 218 | fname = path.basename(fname) 219 | #line = stack()[1][2] 220 | 221 | def prefix(): 222 | import platform 223 | prefix = "\033[0;31mHAPIError:\033[0m " 224 | if platform.system() == 'Windows' and pythonshell() == 'shell': 225 | prefix = "HAPIError: " 226 | 227 | return prefix 228 | 229 | def exception_handler_ipython(self, exc_tuple=None, 230 | filename=None, tb_offset=None, 231 | exception_only=False, 232 | running_compiled_code=False): 233 | 234 | exception = sys.exc_info() 235 | if not debug and exception[0].__name__ == "HAPIError": 236 | sys.stderr.write(prefix() + str(exception[1])) 237 | else: 238 | # Use default 239 | showtraceback_default(self, exc_tuple=None, 240 | filename=None, tb_offset=None, 241 | exception_only=False, 242 | running_compiled_code=False) 243 | 244 | sys.stderr.flush() 245 | 246 | # Reset back to default 247 | InteractiveShell.showtraceback = showtraceback_default 248 | 249 | def exception_handler(exception_type, exception, traceback): 250 | if not debug and exception_type.__name__ == "HAPIError": 251 | print("%s%s" % (prefix(), exception)) 252 | else: 253 | # Use default. 254 | sys.__excepthook__(exception_type, exception, traceback) 255 | 256 | sys.stderr.flush() 257 | 258 | # Reset back to default 259 | sys.excepthook = sys.__excepthook__ 260 | 261 | 262 | if pythonshell() == 'shell': 263 | sys.excepthook = exception_handler 264 | else: 265 | try: 266 | # Copy default function 267 | showtraceback_default = InteractiveShell.showtraceback 268 | # TODO: Use set_custom_exc 269 | # https://ipython.readthedocs.io/en/stable/api/generated/IPython.core.interactiveshell.html 270 | InteractiveShell.showtraceback = exception_handler_ipython 271 | except: 272 | # IPython over-rides this, so this does nothing in IPython shell. 273 | # https://stackoverflow.com/questions/1261668/cannot-override-sys-excepthook 274 | # Don't need to copy default function as it is provided as sys.__excepthook__. 275 | sys.excepthook = exception_handler 276 | 277 | raise HAPIError(msg) 278 | 279 | 280 | def head(url): 281 | """HTTP HEAD request on URL.""" 282 | 283 | import urllib3 284 | http = urllib3.PoolManager() 285 | try: 286 | res = http.request('HEAD', url, retries=2) 287 | if res.status != 200: 288 | raise Exception('Head request failed on ' + url) 289 | return res.headers 290 | except Exception as e: 291 | raise e 292 | 293 | return res.headers 294 | 295 | 296 | def urlopen(url): 297 | """Wrapper to request.get() in urllib3""" 298 | 299 | import sys 300 | import urllib3 301 | from json import load 302 | 303 | # https://stackoverflow.com/a/2020083 304 | def get_full_class_name(obj): 305 | module = obj.__class__.__module__ 306 | if module is None or module == str.__class__.__module__: 307 | return obj.__class__.__name__ 308 | return module + '.' + obj.__class__.__name__ 309 | 310 | c = " If problem persists, a contact email for the server may be listed " 311 | c = c + "at http://hapi-server.org/servers/" 312 | msg = ''; 313 | try: 314 | http = urllib3.PoolManager() 315 | res = http.request('GET', url, preload_content=False, retries=2) 316 | if res.status != 200: 317 | try: 318 | jres = load(res) 319 | except Exception as e: 320 | msg = "Problem with " + url + \ 321 | ". Server responded with non-200 HTTP status (" \ 322 | + str(res.status) + \ 323 | ") and an invalid JSON in response body." + c 324 | 325 | if msg == '' and 'status' in jres: 326 | if 'message' in jres['status']: 327 | msg = '%s\n' % (jres['status']['message']) 328 | 329 | if msg == '': 330 | msg = "Problem with " + url + \ 331 | ". Server responded with non-200 HTTP status (" + \ 332 | str(res.status) + \ 333 | ") but no JSON without HAPI error message in response body." + c 334 | 335 | raise HAPIError 336 | 337 | except HAPIError: 338 | error(msg) 339 | except urllib3.exceptions.NewConnectionError: 340 | error('Connection error for : ' + url + c) 341 | except urllib3.exceptions.ConnectTimeoutError: 342 | error('Connection timeout for: ' + url + c) 343 | except urllib3.exceptions.MaxRetryError: 344 | error('Failed to connect to: ' + url + c) 345 | except urllib3.exceptions.ReadTimeoutError: 346 | error('Read timeout for: ' + url + c) 347 | except urllib3.exceptions.LocationParseError: 348 | error('Could not parse URL: ' + url) 349 | except urllib3.exceptions.LocationValueError: 350 | error('Invalid URL: ' + url) 351 | except urllib3.exceptions.HTTPError as e: 352 | error('Exception ' + get_full_class_name(e) + " for: " + url) 353 | except Exception as e: 354 | print(type(sys.exc_info()[1]).__name__ + ': ' \ 355 | + str(e) + ' for URL: ' + url) 356 | 357 | return res 358 | 359 | 360 | def urlretrieve(url, fname, check_last_modified=False, **kwargs): 361 | """Download URL to file 362 | 363 | urlretrieve(url, fname, check_last_modified=False, **kwargs) 364 | 365 | If check_last_modified=True, `fname` is found, URL returns Last-Modfied 366 | header, and `fname` timestamp is after Last-Modfied timestamp, the URL 367 | is not downloaded. 368 | """ 369 | 370 | import shutil 371 | from os import path, utime, makedirs 372 | from time import mktime, strptime 373 | 374 | if check_last_modified: 375 | if modified(url, fname, **kwargs): 376 | log('Downloading ' + url + ' to ' + fname, kwargs) 377 | res = urlretrieve(url, fname, check_last_modified=False) 378 | if "Last-Modified" in res.headers: 379 | # Change access and modfied time to match that on server. 380 | # TODO: Won't need if using file.head in modified(). 381 | urlLastModified = mktime(strptime(res.headers["Last-Modified"], 382 | "%a, %d %b %Y %H:%M:%S GMT")) 383 | utime(fname, (urlLastModified, urlLastModified)) 384 | else: 385 | log('Local version of ' + fname + ' is up-to-date; using it.', kwargs) 386 | 387 | dirname = path.dirname(fname) 388 | if not path.exists(dirname): 389 | makedirs(dirname) 390 | 391 | with open(fname, 'wb') as out: 392 | res = urlopen(url) 393 | shutil.copyfileobj(res, out) 394 | return res 395 | 396 | 397 | def modified(url, fname, **kwargs): 398 | """Check if timestamp on file is later than Last-Modifed in HEAD request""" 399 | 400 | from os import stat, path 401 | from time import mktime, strptime 402 | 403 | debug = False 404 | 405 | if not path.exists(fname): 406 | return True 407 | 408 | # HEAD request on url 409 | log('Making head request on ' + url, kwargs) 410 | headers = head(url) 411 | 412 | # TODO: Write headers to file.head 413 | if debug: 414 | print("Header:\n--\n") 415 | print(headers) 416 | print("--") 417 | 418 | # TODO: Get this from file.head if found 419 | fileLastModified = stat(fname).st_mtime 420 | if "Last-Modified" in headers: 421 | urlLastModified = mktime(strptime(headers["Last-Modified"], 422 | "%a, %d %b %Y %H:%M:%S GMT")) 423 | 424 | if debug: 425 | print("File Last Modified = %s" % fileLastModified) 426 | print("URL Last Modified = %s" % urlLastModified) 427 | 428 | if urlLastModified > fileLastModified: 429 | return True 430 | return False 431 | else: 432 | if debug: 433 | print("No Last-Modified header. Will re-download") 434 | # TODO: Read file.head and compare etag 435 | return True 436 | 437 | 438 | def urlquote(url): 439 | """Python 2/3 urlquote compatability function. 440 | 441 | If Python 3, returns 442 | urllib.parse.quote(url) 443 | 444 | If Python 2, returns 445 | urllib.quote(url) 446 | """ 447 | 448 | import sys 449 | if sys.version_info[0] == 2: 450 | from urllib import quote 451 | return quote(url) 452 | import urllib.parse 453 | return urllib.parse.quote(url) 454 | -------------------------------------------------------------------------------- /hapiclient/hapi.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import json 5 | import time 6 | import pickle 7 | import warnings 8 | from datetime import datetime, timedelta 9 | 10 | import pandas 11 | import isodate 12 | import numpy as np 13 | from joblib import Parallel, delayed 14 | 15 | from hapiclient.hapitime import hapitime2datetime, hapitime_reformat 16 | from hapiclient.util import setopts, log, warning, error 17 | from hapiclient.util import urlopen, urlretrieve, jsonparse, unicode_error_message 18 | 19 | 20 | def subset(meta, params): 21 | """Extract subset of parameters from meta object returned by hapi(). 22 | 23 | ``metar = subset(meta, parameters)`` modifies ``meta["parameters"]`` array 24 | so that it only contains elements for the time variable and the parameters 25 | in the comma-separated string ``parameters``. 26 | """ 27 | 28 | if params == '': 29 | return meta 30 | 31 | p = params.split(',') 32 | pm = [] # Parameter names in metadata 33 | for i in range(0, len(meta['parameters'])): 34 | pm.append(meta['parameters'][i]['name']) 35 | 36 | # Check for parameters requested that are not in metadata 37 | for i in range(0, len(p)): 38 | if p[i] not in pm: 39 | error('Parameter %s is not in meta' % p[i] + '\n') 40 | return 41 | 42 | pa = [meta['parameters'][0]] # First parameter is always the time parameter 43 | 44 | params_reordered = [] # Re-ordered params 45 | # If time parameter explicity requested, put it first in params_reordered. 46 | if meta['parameters'][0]['name'] in p: 47 | params_reordered = [meta['parameters'][0]['name']] 48 | 49 | # Create subset of parameter metadata 50 | for i in range(1, len(pm)): 51 | if pm[i] in p: 52 | pa.append(meta['parameters'][i]) 53 | params_reordered.append(pm[i]) 54 | meta['parameters'] = pa 55 | 56 | params_reordered_str = ','.join(params_reordered) 57 | 58 | if not params == params_reordered_str: 59 | msg = "\n " + "Order requested: " + params 60 | msg = msg + "\n " + "Order required: " + params_reordered_str 61 | error('Order of requested parameters does not match order of ' \ 62 | 'parameters in server info metadata.' + msg + '\n') 63 | 64 | return meta 65 | 66 | 67 | def cachedir(*args): 68 | """HAPI cache directory. 69 | 70 | cachedir() returns tempfile.gettempdir() + os.path.sep + 'hapi-data' 71 | 72 | cachdir(basedir, server) returns basedir + os.path.sep + server2dirname(server) 73 | """ 74 | import tempfile 75 | 76 | if len(args) == 2: 77 | # cachedir(base_dir, server) 78 | return args[0] + os.path.sep + server2dirname(args[1]) 79 | else: 80 | # cachedir() 81 | return tempfile.gettempdir() + os.path.sep + 'hapi-data' 82 | 83 | 84 | def server2dirname(server): 85 | """Convert a server URL to a directory name.""" 86 | 87 | urld = re.sub(r"https*://", "", server) 88 | urld = re.sub(r'/', '_', urld) 89 | return urld 90 | 91 | 92 | def request2path(*args): 93 | # request2path(server, dataset, parameters, start, stop) 94 | # request2path(server, dataset, parameters, start, stop, cachedir) 95 | 96 | if len(args) == 5: 97 | # Use default if cachedir not given. 98 | cachedirectory = cachedir() 99 | else: 100 | cachedirectory = args[5] 101 | 102 | args = list(args) 103 | 104 | # Replace forbidden characters in directory and filename 105 | # Replacements assume that there will be no name collisions, 106 | # e.g., one parameter named abc-< and another abc-@lt@. 107 | # This also introduces an incompatability between caches on Windows 108 | # Unix. 109 | import platform 110 | if platform.system() == 'Windows': 111 | # List and code from responses in 112 | # https://stackoverflow.com/q/1976007 113 | reps = ( 114 | ('<', '@lt@'), 115 | ('>', '@gt@'), 116 | (':', '@colon@'), 117 | ('"', '@doublequote@'), 118 | ('/', '@forwardslash@'), 119 | ('/', '@backslash@'), 120 | ('\\|', '@pipe@'), 121 | ('\\?', '@questionmark@'), 122 | ('\\*', '@asterisk@') 123 | ) 124 | 125 | for element in reps: 126 | args[1] = re.sub(element[0], element[1], args[1]) 127 | args[2] = re.sub(element[0], element[1], args[2]) 128 | 129 | else: 130 | args[1] = re.sub('/','@forwardslash@',args[1]) 131 | args[2] = re.sub('/','@forwardslash@',args[2]) 132 | 133 | # To shorten filenames. 134 | args[3] = re.sub(r'-|:|\.|Z', '', args[3]) 135 | args[4] = re.sub(r'-|:|\.|Z', '', args[4]) 136 | 137 | # URL subdirectory 138 | urldirectory = server2dirname(args[0]) 139 | fname = '%s_%s_%s_%s' % (args[1], args[2], args[3], args[4]) 140 | 141 | return os.path.join(cachedirectory, urldirectory, fname) 142 | 143 | 144 | def hapiopts(): 145 | """Return dict of default options for hapi() 146 | 147 | Used by hapiplot() and hapi(). 148 | 149 | format = 'binary' is used by default and CSV used if binary is not 150 | available from server. This should option should be excluded from the help 151 | string. 152 | 153 | method = 'pandas' is used by default. Other methods 154 | (numpy, pandasnolength, numpynolength) can be used for testing 155 | CSV read methods. See test/test_hapi.py for comparison. 156 | """ 157 | 158 | # Default options 159 | opts = { 160 | 'logging': False, 161 | 'cache': True, 162 | 'cachedir': cachedir(), 163 | 'usecache': False, 164 | 'server_list': 'https://github.com/hapi-server/servers/raw/master/all.txt', 165 | 'format': 'binary', 166 | 'method': '', 167 | 'parallel': False, 168 | 'n_parallel': 5, 169 | 'n_chunks': None, 170 | 'dt_chunk': None, 171 | } 172 | 173 | return opts 174 | 175 | 176 | def hapi(*args, **kwargs): 177 | """Request data from a HAPI server. 178 | 179 | Version: 0.2.7b1 180 | 181 | 182 | Examples 183 | ---------- 184 | `Jupyter Notebook `_ 185 | 186 | Parameters 187 | ---------- 188 | server: str 189 | A string with the URL to a HAPI compliant server. (A HAPI URL \ 190 | always ends with ``/hapi``). 191 | dataset: str 192 | A string specifying a dataset from a `server` 193 | parameters: str 194 | A comma-separated list of parameters in `dataset` 195 | start: str 196 | The start time of the requested data 197 | stop: str or None 198 | The end time of the requested data; end times are exclusive - the 199 | last data record returned by a HAPI server should have a timestamp 200 | before `start`. If `None`, `stopDate` is used. 201 | options: dict 202 | `logging` (``False``) - Log to console 203 | 204 | `cache` (``True``) - Save responses and processed responses in cachedir 205 | 206 | `cachedir` (``'./hapi-data'``) 207 | 208 | `usecache` (``True``) - Use files in `cachedir` if found 209 | 210 | `serverlist` (``'https://github.com/hapi-server/servers/raw/master/all.txt'``) 211 | 212 | `format` (``'binary'``) ``'binary'`` or ``'csv'``; ``'csv``' will force the use of ``format=csv`` in request to server. 213 | 214 | `parallel` (``False``) If ``True``, make up to `n_parallel` requests to server \ 215 | in parallel (uses threads) 216 | 217 | `n_parallel` (``5``) Maximum number of parallel requests to server.\ 218 | Max allowed is 5. 219 | 220 | `n_chunks` (``None``) Get data by making `n_chunks` requests by splitting \ 221 | requested time range. `dt_chunk` is ignored if `n_chunks` is \ 222 | not `None`. Allowed values are integers > 1. 223 | 224 | `dt_chunk` (``'infer'``) For requests that span a time range larger \ 225 | than the default chunk size for a given dataset cadence, the \ 226 | client will split request into chunks if `dt_chunk` is not \ 227 | `None`. 228 | 229 | Allowed values of `dt_chunk` are 'infer', `None`, and an ISO 8601 \ 230 | duration that is unambiguous (durations that include Y and M are not \ 231 | allowed). The default chunk size is determined based on the cadence \ 232 | of the dataset requested according to 233 | 234 | * cadence < PT1S dt_chunk='PT1H' 235 | * PT1S <= cadence <= PT1H dt_chunk='P1D' 236 | * cadence > PT1H dt_chunk='P30D' 237 | * cadence >= P1D dt_chunk='P365D' 238 | 239 | If the dataset does not have a cadence listed in its metadata, an 240 | attempt is made to infer the cadence by requesting a small time range 241 | of data and doubling the time range until 10 records are in the response. 242 | The cadence used to determine the chunk size is then the average time 243 | difference between records. 244 | 245 | If requested time range is < 1/2 of chunk size, only one request is 246 | made. Otherwise, start and/or stop are modified to be at hour, day, 247 | month or year boundaries and requests are made for a time span of a 248 | full chunk, and trimming is performed. For example, 249 | 250 | Cadence = PT1M and request for 251 | 252 | start/stop=1999-11-12T00:10:00/stop=1999-11-12T12:09:00 253 | 254 | Chunk size is P1D and requested time range < 1/2 of this 255 | => Default behavior 256 | 257 | Cadence = PT1M and request for 258 | 259 | start/stop=1999-11-12T00:10:00/1999-11-12T12:10:00 260 | 261 | Chunk size is P1D and requested time range >= 1/2 of this 262 | => One request with start/stop=1999-11-12/1999-11-13 263 | and trim performed 264 | 265 | Cadence = PT1M and request for 266 | 267 | start/stop=1999-11-12T00:10:00/1999-11-13T12:09:00 268 | 269 | Chunk size is P1D and requested time range > than this 270 | => Two requests: 271 | 272 | 1. start/stop=1999-11-12/start=1999-11-13 273 | 2. start/stop=1999-11-13/start=1999-11-14 274 | 275 | and trim performed 276 | 277 | 278 | Returns 279 | ------- 280 | result: varies 281 | `result` depends on the input parameters. 282 | 283 | ``servers = hapi()`` returns a list of available HAPI server URLs from 284 | https://github.com/hapi-server/data-specification/blob/master/all.txt 285 | 286 | ``dataset = hapi(server)`` returns a dict of datasets available from a 287 | URL given by the string `server`. The dictionary structure follows the 288 | `HAPI catalog response JSON structure `_. 289 | 290 | ``parameters = hapi(server, dataset)`` returns a dict containing the HAPI info metadata for all parameters 291 | in the string `dataset`. The dictionary structure follows the 292 | `HAPI info response JSON structure `_. 293 | 294 | ``meta = hapi(server, dataset, parameters)`` returns a dict containing the HAPI info metadata 295 | for each parameter in the comma-separated string ``parameters``. The 296 | dictionary structure follows 297 | `HAPI info response JSON structure `_. 298 | 299 | ``data, = hapi(server, dataset, parameters, start, stop)`` returns a 300 | NumPy array with named fields with field names corresponding to ``parameters``, e.g., if 301 | ``parameters = 'scalar,vector'`` and the number of records in the time 302 | range ``start`` <= t < ``stop`` returned is N, then 303 | 304 | ``data['scalar']`` is a NumPy array of shape (N) 305 | 306 | ``data['vector']`` is a NumPy array of shape (N,3) 307 | 308 | ``data['Time']`` is a NumPy array of byte literals with shape (N). 309 | 310 | Byte literal times can be converted to Python ``datetimes`` using 311 | 312 | ``dtarray = hapitime2datetime(data['Time'])`` 313 | 314 | ``data, meta = hapi(server, dataset, parameters, start, stop)`` returns 315 | the HAPI info metadata for parameters in `meta` (and should contain the 316 | same content as ``meta = hapi(server, dataset, parameters)``). 317 | 318 | 319 | References 320 | ---------- 321 | * `HAPI Server Definition `_ 322 | 323 | """ 324 | 325 | nin = len(args) 326 | 327 | if nin > 0: 328 | SERVER = args[0] 329 | if nin > 1: 330 | DATASET = args[1] 331 | if nin > 2: 332 | PARAMETERS = args[2] 333 | if nin > 3: 334 | START = args[3] 335 | if START[-1] != 'Z': 336 | # TODO: Consider warning. 337 | START = START + 'Z' 338 | if nin > 4: 339 | STOP = args[4] 340 | if STOP is not None and STOP[-1] != 'Z': 341 | # TODO: Consider warning. 342 | STOP = STOP + 'Z' 343 | 344 | # Override defaults 345 | opts = setopts(hapiopts(), kwargs) 346 | 347 | assert (opts['logging'] in [True, False]), "logging keyword must be True of False" 348 | assert (opts['cache'] in [True, False]), "cache keyword must be True of False" 349 | assert (opts['usecache'] in [True, False]), "usecache keyword must be True of False" 350 | assert (opts['format'] in ['binary', 'csv']), "format keyword must be 'csv' or 'binary'" 351 | assert (opts['method'] in ['', 'pandas', 'numpy', 'pandasnolength', 'numpynolength']) 352 | assert (opts['parallel'] in [True, False]), 'parallel keyword must be True or False' 353 | assert (isinstance(opts['n_parallel'], int) and opts['n_parallel'] > 1) 354 | assert (opts['n_chunks'] is None or isinstance(opts['n_chunks'], int) and opts['n_chunks'] > 0) 355 | assert (opts['dt_chunk'] in [None, 'infer', 'PT1H', 'P1D', 'P1M', 'P1Y']) 356 | 357 | from hapiclient import __version__ 358 | log('Running hapi.py version %s' % __version__, opts) 359 | 360 | if nin > 1: 361 | if nin > 1: 362 | if unicode_error_message(DATASET) != "": 363 | error(unicode_error_message(DATASET)) 364 | if nin > 2: 365 | if unicode_error_message(PARAMETERS) != "": 366 | error(unicode_error_message(PARAMETERS)) 367 | 368 | if nin == 0: # hapi() 369 | log('Reading %s' % opts['server_list'], opts) 370 | # decode('utf8') in following needed to make Python 2 and 3 types match. 371 | data = urlopen(opts['server_list']).read().decode('utf8').split('\n') 372 | data = [x for x in data if x] # Remove empty items (if blank lines) 373 | # Display server URLs to console. 374 | log('List of HAPI servers in %s:\n' % opts['server_list'], opts) 375 | for url in data: 376 | log(" %s" % url, opts) 377 | return data 378 | 379 | if nin == 1: # hapi(SERVER) 380 | # TODO: Cache 381 | url = SERVER + '/catalog' 382 | log('Reading %s' % url, opts) 383 | res = urlopen(url) 384 | meta = jsonparse(res, url) 385 | return meta 386 | 387 | if nin == 2: # hapi(SERVER, DATASET) 388 | # TODO: Cache 389 | url = SERVER + '/info?id=' + DATASET 390 | log('Reading %s' % url, opts) 391 | res = urlopen(url) 392 | meta = jsonparse(res, url) 393 | return meta 394 | 395 | if nin == 4: 396 | error('A stop time is required if a start time is given.') 397 | 398 | if nin == 3 or nin == 5: 399 | # hapi(SERVER, DATASET, PARAMETERS) or 400 | # hapi(SERVER, DATASET, PARAMETERS, START, STOP) 401 | 402 | # Extract all parameters. 403 | if re.search(r', ', PARAMETERS): 404 | warning("Removing spaces after commas in given parameter list of '" \ 405 | + PARAMETERS + "'") 406 | PARAMETERS = re.sub(r',\s+', ',', PARAMETERS) 407 | 408 | # urld = url subdirectory of cachedir to store files from SERVER 409 | urld = cachedir(opts["cachedir"], SERVER) 410 | 411 | if opts["cachedir"]: log('file directory = %s' % urld, opts) 412 | 413 | urljson = SERVER + '/info?id=' + DATASET 414 | 415 | # Output from urljson will be saved in a .json file. Parsed json 416 | # will be stored in a .pkl file. Metadata for all parameters is 417 | # requested and response is subsetted so only metadata for PARAMETERS 418 | # is returned. 419 | fname_root = request2path(SERVER, DATASET, '', '', '', opts['cachedir']) 420 | fnamejson = fname_root + '.json' 421 | fnamepkl = fname_root + '.pkl' 422 | 423 | if nin == 5: # Data requested 424 | 425 | if STOP is None: 426 | log('STOP was given as None. Getting stopDate for dataset.', opts) 427 | meta = hapi(SERVER, DATASET) 428 | STOP = meta['stopDate'] 429 | log('Using STOP = {STOP}', opts) 430 | 431 | tic_totalTime = time.time() 432 | 433 | # URL to get CSV (will be used if binary response is not available) 434 | urlcsv = SERVER + '/data?id=' + DATASET + '¶meters=' + \ 435 | PARAMETERS + '&time.min=' + START + '&time.max=' + STOP 436 | # URL for binary request 437 | urlbin = urlcsv + '&format=binary' 438 | 439 | # Raw CSV and HAPI Binary (no header) will be stored in .csv and 440 | # .bin files. Parsed response of either CSV or HAPI Binary will 441 | # be stored in a .npy file. 442 | # fnamepklx will contain additional metadata about the request 443 | # including d/l time, parsing time, and the location of files. 444 | fname_root = request2path(SERVER, DATASET, PARAMETERS, START, STOP, opts['cachedir']) 445 | fnamecsv = fname_root + '.csv' 446 | fnamebin = fname_root + '.bin' 447 | fnamenpy = fname_root + '.npy' 448 | fnamepklx = fname_root + ".pkl" 449 | 450 | metaFromCache = False 451 | if opts["usecache"]: 452 | if nin == 3 and os.path.isfile(fnamepkl): 453 | # Read cached metadata from .pkl file. 454 | # This returns subsetted metadata with no additional "x_" 455 | # information (which is stored in fnamepklx). 456 | log('Reading %s' % fnamepkl.replace(urld + '/', ''), opts) 457 | f = open(fnamepkl, 'rb') 458 | meta = pickle.load(f) 459 | f.close() 460 | metaFromCache = True 461 | # Remove parameters not requested. 462 | meta = subset(meta, PARAMETERS) 463 | return meta 464 | if os.path.isfile(fnamepklx): 465 | # Read subsetted meta file with x_ information 466 | log('Reading %s' % fnamepklx.replace(urld + '/', ''), opts) 467 | f = open(fnamepklx, 'rb') 468 | meta = pickle.load(f) 469 | metaFromCache = True 470 | f.close() 471 | 472 | if not metaFromCache: 473 | # No cached metadata loaded so request it from server. 474 | log('Reading %s' % urld, opts) 475 | res = urlopen(urljson) 476 | meta = jsonparse(res, urljson) 477 | 478 | # Add information to metadata so we can figure out request needed 479 | # to generated it. Will also be used for labeling plots by hapiplot(). 480 | meta.update({"x_server": SERVER}) 481 | meta.update({"x_dataset": DATASET}) 482 | 483 | if opts["cache"]: 484 | if not os.path.exists(urld): os.makedirs(urld) 485 | 486 | if opts['dt_chunk'] == 'infer': 487 | cadence = meta.get('cadence', None) 488 | 489 | # If cadence not given, use 1-day chunks. 490 | if cadence is None: 491 | cadence = 'PT1M' 492 | else: 493 | cadence = isodate.parse_duration(cadence) 494 | if isinstance(cadence, isodate.Duration): 495 | # When a duration does not correspond to an unambiguous 496 | # time duration (e.g., P1M), parse_duration returns an 497 | # isodate.duration.Duration object. Otherwise, it returns 498 | # a datetime.timedelta object. 499 | cadence = cadence.totimedelta(start=datetime.now()) 500 | 501 | pt1s = isodate.parse_duration('PT1S') 502 | pt1h = isodate.parse_duration('PT1H') 503 | p1d = isodate.parse_duration('P1D') 504 | 505 | if cadence < pt1s: 506 | opts['dt_chunk'] = 'PT1H' 507 | elif pt1s <= cadence <= pt1h: 508 | opts['dt_chunk'] = 'P1D' 509 | elif cadence > pt1h: 510 | opts['dt_chunk'] = 'P1M' 511 | elif cadence >= p1d: 512 | opts['dt_chunk'] = 'P1Y' 513 | 514 | if opts['n_chunks'] is not None or opts['dt_chunk'] is not None: 515 | 516 | padz = lambda x: x if 'Z' in x else x + 'Z' 517 | pSTART = hapitime2datetime(padz(START))[0] 518 | pSTOP = hapitime2datetime(padz(STOP))[0] 519 | 520 | if opts['dt_chunk']: 521 | pDELTA = isodate.parse_duration(opts['dt_chunk']) 522 | 523 | if opts['dt_chunk'] == 'P1Y': 524 | half = isodate.parse_duration('P365D') / 2 525 | elif opts['dt_chunk'] == 'P1M': 526 | half = isodate.parse_duration('P30D') / 2 527 | else: 528 | half = pDELTA / 2 529 | 530 | if (pSTOP - pSTART) < half: 531 | opts['n_chunks'] = None 532 | opts['dt_chunk'] = None 533 | return hapi(SERVER, DATASET, PARAMETERS, START, STOP, **opts) 534 | 535 | if opts['dt_chunk'] == 'P1Y': 536 | pSTART = datetime(pSTART.year, 1, 1) 537 | pSTOP = datetime(pSTOP.year + 1, 1, 1) 538 | opts['n_chunks'] = pSTOP.year - pSTART.year 539 | elif opts['dt_chunk'] == 'P1M': 540 | pSTART = datetime(pSTART.year, pSTART.month, 1) 541 | pSTOP = datetime(pSTOP.year, pSTOP.month + 1, 1) 542 | opts['n_chunks'] = (pSTOP.year - pSTART.year) * 12 + (pSTOP.month - pSTART.month) 543 | elif opts['dt_chunk'] == 'P1D': 544 | pSTART = datetime.combine(pSTART.date(), datetime.min.time()) 545 | pSTOP = datetime.combine(pSTOP.date(), datetime.min.time()) + timedelta(days=1) 546 | opts['n_chunks'] = (pSTOP - pSTART).days 547 | elif opts['dt_chunk'] == 'PT1H': 548 | pSTART = datetime.combine(pSTART.date(), datetime.min.time()) + timedelta(hours=pSTART.hour) 549 | pSTOP = datetime.combine(pSTOP.date(), datetime.min.time()) + timedelta(hours=pSTOP.hour + 1) 550 | opts['n_chunks'] = int(((pSTOP - pSTART).total_seconds() / 60) / 60) 551 | else: 552 | pDIFF = pSTOP - pSTART 553 | pDELTA = pDIFF / opts['n_chunks'] 554 | 555 | n_chunks = opts['n_chunks'] 556 | opts['n_chunks'] = None 557 | opts['dt_chunk'] = None 558 | 559 | backend = 'sequential' 560 | if opts['parallel']: 561 | backend = 'threading' 562 | # multiprocessing was not tested. It may work, but will 563 | # need a speed comparison with threading. 564 | # backend = 'multiprocessing' 565 | 566 | log('backend = {}'.format(backend), opts) 567 | 568 | verbose = 0 569 | if opts.get('logging'): 570 | verbose = 100 571 | 572 | def nhapi(SERVER, DATASET, PARAMETERS, pSTART, pDELTA, i, **opts): 573 | START = pSTART + (i * pDELTA) 574 | START = str(START.date())+'T'+str(START.time()) 575 | 576 | STOP = pSTART + ((i + 1) * pDELTA) 577 | STOP = str(STOP.date()) + 'T' + str(STOP.time()) 578 | 579 | data, meta = hapi( 580 | SERVER, 581 | DATASET, 582 | PARAMETERS, 583 | START, 584 | STOP, 585 | **opts 586 | ) 587 | return data, meta 588 | 589 | resD, resM = zip( 590 | *Parallel(n_jobs=opts['n_parallel'], verbose=verbose, backend=backend)( 591 | delayed(nhapi)( 592 | SERVER, 593 | DATASET, 594 | PARAMETERS, 595 | pSTART, 596 | pDELTA, 597 | i, 598 | **opts 599 | ) for i in range(n_chunks) 600 | ) 601 | ) 602 | 603 | resD = list(resD) 604 | 605 | tic_trimTime = time.time() 606 | if sys.version_info < (3, ): 607 | START = hapitime_reformat(str(resD[0]['Time'][0]), START) 608 | resD[0] = resD[0][resD[0]['Time'] >= START] 609 | 610 | STOP = hapitime_reformat(str(resD[-1]['Time'][0]), STOP) 611 | resD[-1] = resD[-1][resD[-1]['Time'] < STOP] 612 | else: 613 | START = hapitime_reformat(resD[0]['Time'][0].decode('UTF-8'), START) 614 | resD[0] = resD[0][resD[0]['Time'] >= bytes(START, 'UTF-8')] 615 | 616 | STOP = hapitime_reformat(resD[-1]['Time'][0].decode('UTF-8'), STOP) 617 | resD[-1] = resD[-1][resD[-1]['Time'] < bytes(STOP, 'UTF-8')] 618 | trimTime = time.time() - tic_trimTime 619 | 620 | tic_catTime = time.time() 621 | data = np.concatenate(resD) 622 | catTime = time.time() - tic_catTime 623 | 624 | meta = resM[0].copy() 625 | meta['x_time.max'] = resM[-1]['x_time.max'] 626 | meta['x_dataFile'] = None 627 | meta['x_dataFiles'] = [resM[i]['x_dataFile'] for i in range(len(resM))] 628 | meta['x_downloadTime'] = sum([resM[i]['x_downloadTime'] for i in range(len(resM))]) 629 | meta['x_downloadTimes'] = [resM[i]['x_downloadTime'] for i in range(len(resM))] 630 | meta['x_readTime'] = sum([resM[i]['x_readTime'] for i in range(len(resM))]) 631 | meta['x_readTimes'] = [resM[i]['x_readTime'] for i in range(len(resM))] 632 | meta['x_trimTime'] = trimTime 633 | meta['x_catTime'] = catTime 634 | meta['x_totalTime'] = time.time() - tic_totalTime 635 | meta['x_dataFileParsed'] = None 636 | meta['x_dataFilesParsed'] = [resM[i]['x_dataFileParsed'] for i in range(len(resM))] 637 | 638 | return data, meta 639 | 640 | if opts["cache"] and not metaFromCache: 641 | # Cache metadata for all parameters if it was not already loaded 642 | # from cache. Note that fnamepklx is written after data downloaded 643 | # and parsed. 644 | log('Writing %s ' % fnamejson.replace(urld + '/', ''), opts) 645 | f = open(fnamejson, 'w') 646 | json.dump(meta, f, indent=4) 647 | f.close() 648 | 649 | log('Writing %s ' % fnamepkl.replace(urld + '/', ''), opts) 650 | f = open(fnamepkl, 'wb') 651 | # protocol=2 used for Python 2.7 compatability. 652 | pickle.dump(meta, f, protocol=2) 653 | f.close() 654 | 655 | # Remove unrequested parameters if they have not have already been 656 | # removed (b/c loaded from cache). 657 | if not metaFromCache: 658 | meta = subset(meta, PARAMETERS) 659 | 660 | if nin == 3: 661 | return meta 662 | 663 | if opts["usecache"] and os.path.isfile(fnamenpy): 664 | # Read cached data file. 665 | log('Reading %s ' % fnamenpy.replace(urld + '/', ''), opts) 666 | f = open(fnamenpy, 'rb') 667 | data = np.load(f) 668 | f.close() 669 | # There is a possibility that the fnamenpy file existed but 670 | # fnamepklx was not found (b/c removed). In this case, the meta 671 | # returned will not have all of the "x_" information inserted below. 672 | # Code that uses this information needs to account for this. 673 | meta['x_totalTime'] = time.time() - tic_totalTime 674 | return data, meta 675 | 676 | cformats = ['csv', 'binary'] # client formats 677 | if not opts['format'] in cformats: 678 | # Check if requested format is implemented by this client. 679 | error('This client does not handle transport ' 680 | 'format "%s". Available options: %s' 681 | % (opts['format'], ', '.join(cformats))) 682 | 683 | # See if server supports binary 684 | if opts['format'] != 'csv': 685 | log('Reading %s' % (SERVER + '/capabilities'), opts) 686 | res = urlopen(SERVER + '/capabilities') 687 | caps = jsonparse(res, SERVER + '/capabilities') 688 | sformats = caps["outputFormats"] # Server formats 689 | if 'format' in kwargs and not kwargs['format'] in sformats: 690 | warning("hapi", 'Requested transport format "%s" not avaiable ' 691 | 'from %s. Will use "csv". Available options: %s' 692 | % (opts['format'], SERVER, ', '.join(sformats))) 693 | opts['format'] = 'csv' 694 | if not 'binary' in sformats: 695 | opts['format'] = 'csv' 696 | 697 | dt, cols, psizes, pnames, ptypes, missing_length = compute_dt(meta, opts) 698 | 699 | # length attribute required for all parameters when serving binary but 700 | # is only required for time parameter when serving CSV. This catches 701 | # case where server provides binary but is missing a length attribute 702 | # in one or more string parameters that were requested. In this case, 703 | # there is not enough information to parse binary. 704 | if opts['format'] == 'binary' and missing_length: 705 | warnings.warn('Requesting CSV instead of binary because of problem with server metadata.') 706 | opts['format'] == 'csv' 707 | 708 | # Read the data. toc0 is time to download to file or into buffer; 709 | # toc is time to parse. 710 | if opts['format'] == 'binary': 711 | 712 | if opts['method'] != '': 713 | warnings.warn("Method argument is ignored when format='binary.") 714 | 715 | # Handle Unicode strings (since HAPI 3.1) 716 | dto = [] 717 | for i in range(len(dt)): 718 | dto.append(dt[i]) 719 | if isinstance(dt[i][1], str) and dt[i][1][0] == 'U' and meta['parameters'][i]['type'] == 'string': 720 | # numpy.frombuffer() requires S instead of U 721 | # because Unicode is variable length. 722 | dt[i] = list(dt[i]) 723 | dt[i][1] = dt[i][1].replace('U', 'S') 724 | dt[i] = tuple(dt[i]) 725 | 726 | # HAPI Binary 727 | if opts["cache"]: 728 | log('Writing %s to %s' % (urlbin, fnamebin.replace(urld + '/', '')), opts) 729 | tic0 = time.time() 730 | urlretrieve(urlbin, fnamebin) 731 | toc0 = time.time() - tic0 732 | log('Reading and parsing %s' % fnamebin.replace(urld + '/', ''), opts) 733 | tic = time.time() 734 | try: 735 | data = np.fromfile(fnamebin, dtype=dt) 736 | except: 737 | error('Malformed response? Could not read: {}'.format(urlbin)) 738 | else: 739 | from io import BytesIO 740 | log('Writing %s to buffer' % urlbin.replace(urld + '/', ''), opts) 741 | tic0 = time.time() 742 | buff = BytesIO(urlopen(urlbin).read()) 743 | toc0 = time.time() - tic0 744 | log('Parsing BytesIO buffer.', opts) 745 | tic = time.time() 746 | try: 747 | data = np.frombuffer(buff.read(), dtype=dt) 748 | except: 749 | error('Malformed response? Could not read: {}'.format(urlbin)) 750 | 751 | 752 | # Handle Unicode 753 | time_name = meta['parameters'][0]['name'] 754 | datanew = np.ndarray(shape=data[time_name].shape, dtype=dto) 755 | for i in range(0, len(dto)): 756 | name = meta['parameters'][i]['name'] 757 | if sys.version_info[0] < 3: 758 | # str() here is needed for Python 2.7. Numpy does not allow 759 | # Unicode names in this version and if a dt is created 760 | # with Unicode, it automatically converts Unicode chars to 761 | # slash encoded ASCII. 762 | name = str(name) 763 | if data[name].size > 0 and isinstance(dt[i][1], str) and 'U' in dto[i][1]: 764 | # Decode data. 765 | datanew[name] = np.char.decode(data[name]) 766 | else: 767 | datanew[name] = data[name] 768 | data = datanew 769 | 770 | else: 771 | # HAPI CSV 772 | 773 | file_empty = False 774 | 775 | if opts["cache"]: 776 | log('Writing %s to %s' % (urlcsv, fnamecsv.replace(urld + '/', '')), opts) 777 | tic0 = time.time() 778 | urlretrieve(urlcsv, fnamecsv) 779 | toc0 = time.time() - tic0 780 | log('Reading and parsing %s' % fnamecsv.replace(urld + '/', ''), opts) 781 | tic = time.time() 782 | if os.path.getsize(fnamecsv) == 0: 783 | file_empty = True 784 | data = np.array([], dtype=dt) 785 | else: 786 | from io import StringIO 787 | log('Writing %s to buffer' % urlcsv.replace(urld + '/', ''), opts) 788 | tic0 = time.time() 789 | fnamecsv = StringIO(urlopen(urlcsv).read().decode()) 790 | fnamecsv.seek(0, os.SEEK_END) 791 | if fnamecsv.tell() == 0: 792 | file_empty = True 793 | data = np.array([], dtype=dt) 794 | else: 795 | fnamecsv.seek(0) 796 | toc0 = time.time() - tic0 797 | log('Parsing StringIO buffer.', opts) 798 | tic = time.time() 799 | 800 | if file_empty == False: 801 | if not missing_length: 802 | # All string and isotime parameters have a length in metadata. 803 | if opts['method'] == 'numpy': 804 | try: 805 | data = np.genfromtxt(fnamecsv, 806 | dtype=dt, 807 | delimiter=',', 808 | replace_space=' ', 809 | deletechars='', 810 | encoding='utf-8') 811 | except: 812 | error('Malformed response? Could not read response: {}'.format(urlcsv)) 813 | if opts['method'] == '' or opts['method'] == 'pandas': 814 | # Read file into Pandas DataFrame 815 | try: 816 | df = pandas.read_csv(fnamecsv, 817 | sep=',', 818 | header=None, 819 | encoding='utf-8') 820 | except: 821 | error('Malformed response? Could not read response: {}'.format(urlcsv)) 822 | # Allocate output N-D array (It is not possible to pass dtype=dt 823 | # as computed to pandas.read_csv; pandas dtype is different 824 | # from numpy's dtype.) 825 | data = np.ndarray(shape=(len(df)), dtype=dt) 826 | # Insert data from dataframe 'df' columns into N-D array 'data' 827 | for i in range(0, len(pnames)): 828 | shape = np.append(len(data), psizes[i]) 829 | # In numpy 1.8.2 and Python 2.7, this throws an error 830 | # for no apparent reason. Works as expected in numpy 1.10.4 831 | data[pnames[i]] = np.squeeze( 832 | np.reshape(df.values[:, np.arange(cols[i][0], cols[i][1] + 1)], shape)) 833 | else: 834 | data = parse_missing_length(fnamecsv, dt, cols, psizes, pnames, ptypes, opts) 835 | 836 | toc = time.time() - tic 837 | 838 | # Extra metadata associated with request will be saved in 839 | # a pkl file with same base name as npy data file. 840 | meta.update({"x_server": SERVER}) 841 | meta.update({"x_dataset": DATASET}) 842 | meta.update({"x_parameters": PARAMETERS}) 843 | meta.update({"x_time.min": START}) 844 | meta.update({"x_time.max": STOP}) 845 | meta.update({"x_requestDate": datetime.now().isoformat()[0:19]}) 846 | meta.update({"x_cacheDir": urld}) 847 | meta.update({"x_downloadTime": toc0}) 848 | meta.update({"x_readTime": toc}) 849 | meta.update({"x_metaFileParsed": fnamepkl}) 850 | meta.update({"x_dataFileParsed": fnamenpy}) 851 | meta.update({"x_metaFile": fnamejson}) 852 | if opts['format'] == 'binary': 853 | meta.update({"x_dataFile": fnamebin}) 854 | else: 855 | meta.update({"x_dataFile": fnamecsv}) 856 | 857 | if opts["cache"]: 858 | if not os.path.exists(opts["cachedir"]): 859 | os.makedirs(opts["cachedir"]) 860 | if not os.path.exists(urld): 861 | os.makedirs(urld) 862 | 863 | log('Writing %s' % fnamepklx, opts) 864 | with open(fnamepklx, 'wb') as f: 865 | pickle.dump(meta, f, protocol=2) 866 | 867 | log('Writing %s' % fnamenpy, opts) 868 | np.save(fnamenpy, data) 869 | 870 | meta['x_totalTime'] = time.time() - tic_totalTime 871 | 872 | return data, meta 873 | 874 | 875 | def compute_dt(meta, opts): 876 | 877 | # Compute data type variable dt used to read HAPI response into 878 | # a data structure. 879 | pnames, psizes, ptypes, dt = [], [], [], [] 880 | # Each element of cols is an array with start/end column number of 881 | # parameter. 882 | 883 | cols = np.zeros([len(meta["parameters"]), 2], dtype=np.int32) 884 | ss = 0 # running sum of prod(size) 885 | 886 | # missing_length = True will be set if HAPI String or ISOTime 887 | # parameter has no length attribute in metadata (length attribute is 888 | # required for both in binary but only for primary time column in CSV). 889 | # When missing_length=True the CSV read gets more complicated. 890 | missing_length = False 891 | 892 | # Extract sizes and types of parameters. 893 | for i in range(0, len(meta["parameters"])): 894 | ptype = meta["parameters"][i]["type"] 895 | 896 | ptypes.append(ptype) 897 | 898 | pnames.append(str(meta["parameters"][i]["name"])) 899 | if 'size' in meta["parameters"][i]: 900 | psizes.append(meta["parameters"][i]['size']) 901 | else: 902 | psizes.append(1) 903 | 904 | # For size = [N] case, readers want 905 | # dtype = ('name', type, N) 906 | # not 907 | # dtype = ('name', type, [N]) 908 | if type(psizes[i]) is list and len(psizes[i]) == 1: 909 | psizes[i] = psizes[i][0] 910 | 911 | if type(psizes[i]) is list and len(psizes[i]) > 1: 912 | # psizes[i] = list(reversed(psizes[i])) 913 | psizes[i] = list(psizes[i]) 914 | 915 | # First column of ith parameter. 916 | cols[i][0] = ss 917 | # Last column of ith parameter. 918 | cols[i][1] = ss + np.prod(psizes[i]) - 1 919 | # Running sum of columns. 920 | ss = cols[i][1] + 1 921 | 922 | # HAPI numerical formats are 64-bit LE floating point and 32-bit LE 923 | # signed integers. 924 | if ptype == 'double': 925 | dtype = (pnames[i], ' 2 and dtype[2] == 1: 1045 | dtype = dtype[0:2] 1046 | dt2.append(dtype) 1047 | 1048 | # Create new N-D array that won't have any parameters with 1049 | # type = 'O'. 1050 | data2 = np.ndarray(data.shape, dt2) 1051 | 1052 | for i in range(0, len(pnames)): 1053 | if data[pnames[i]].dtype == 'O': 1054 | data2[pnames[i]] = data[pnames[i]].astype(dt2[i][1]) 1055 | else: 1056 | data2[pnames[i]] = data[pnames[i]] 1057 | # Save memory by not copying (does this help?) 1058 | # data2[pnames[i]] = np.array(data[pnames[i]],copy=False) 1059 | 1060 | return data2 1061 | --------------------------------------------------------------------------------