├── .flake8 ├── .github └── workflows │ ├── pypi.yml │ └── unit_test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── LICENSE ├── README.md ├── demos ├── ctd_demo.ipynb └── hydrophone_demo.ipynb ├── docs ├── api_docs.md ├── conf.py ├── figures │ ├── OOIPY_Logo.png │ ├── ctd_profile.png │ ├── psd_bb.png │ └── psd_mean_median.png ├── getting_started.md ├── index.md └── refs.bib ├── imgs ├── OOIPY_Logo.png ├── OOIPY_favicon.ico ├── ooipy_banner.png └── ooipy_banner2.png ├── pyproject.toml ├── src └── ooipy │ ├── __init__.py │ ├── ctd │ ├── __init__.py │ └── basic.py │ ├── hydrophone │ ├── __init__.py │ ├── basic.py │ └── calibration_by_assetID.csv │ ├── request │ ├── __init__.py │ ├── authentification.py │ ├── ctd_request.py │ └── hydrophone_request.py │ ├── scripts │ ├── __init__.py │ └── download_hydrophone_data.py │ ├── surface_buoy │ └── __init__.py │ └── tools │ ├── __init__.py │ └── ooiplotlib.py └── tests ├── __init__.py └── test_request.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=100 3 | extend-ignore = E203 4 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | # Publish archives to PyPI and TestPyPI using GitHub Actions 2 | name: Publish to PyPI 3 | 4 | # Only run for tagged releases and pushes to the master branch 5 | on: 6 | release: 7 | types: 8 | - published 9 | push: 10 | branches: 11 | - main 12 | workflow_dispatch: 13 | 14 | jobs: 15 | publish-pypi: 16 | name: Publish to PyPI 17 | runs-on: ubuntu-latest 18 | if: github.repository == 'Ocean-Data-Lab/ooipy' 19 | 20 | steps: 21 | - name: Checkout 22 | uses: actions/checkout@v3 23 | with: 24 | # fetch all history so that setuptools-scm works 25 | fetch-depth: 0 26 | 27 | - name: Set up Python 28 | uses: actions/setup-python@v4.5.0 29 | with: 30 | python-version: 3.x 31 | 32 | - name: Install dependencies 33 | run: python -m pip install --upgrade hatch hatch-vcs 34 | 35 | - name: Build source and wheel distributions 36 | run: | 37 | hatch build 38 | echo "" 39 | echo "Generated files:" 40 | ls -lh dist/ 41 | - name: Publish to Test PyPI 42 | run: hatch publish -r test -u __token__ -a ${{ secrets.TEST_PYPI_API_TOKEN }} 43 | 44 | - name: Publish to PyPI 45 | if: startsWith(github.ref, 'refs/tags') 46 | run: hatch publish -u __token__ -a ${{ secrets.PYPI_API_TOKEN }} 47 | -------------------------------------------------------------------------------- /.github/workflows/unit_test.yml: -------------------------------------------------------------------------------- 1 | name: Unit Tests 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | tests: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | python-version: 14 | - "3.10" 15 | - "3.11" 16 | - "3.12" 17 | - "3.13" 18 | name: Check Python ${{ matrix.python-version }} 19 | steps: 20 | - uses: actions/checkout@v4 21 | with: 22 | fetch-depth: 0 # Only needed if using setuptools-scm 23 | 24 | - name: Setup Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v5 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | allow-prereleases: true 29 | 30 | - name: Install package 31 | run: python -m pip install -e .[test] 32 | 33 | - name: Test package 34 | run: python -m pytest -p no:warnings 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .ipynb_checkpoints/ 3 | .vscode/ 4 | .DS_Store 5 | .cache 6 | .pytest_cache 7 | *.pyc 8 | *.pkl 9 | !calibration_by_assetID.pkl 10 | build/ 11 | _build/ 12 | flake8_log.txt 13 | ooipy/tests/figures 14 | .idea 15 | debug.ipynb 16 | dist/ 17 | ooipy.egg-info/ 18 | *.wav 19 | ooi_auth.txt 20 | .eggs 21 | _ooipy_version.py 22 | *.nc 23 | src/ooipy/version.py 24 | dev/ 25 | 26 | # Environments 27 | .env 28 | .venv 29 | env/ 30 | venv/ 31 | ENV/ 32 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: | 2 | (?x)^( 3 | docs/source/_static/test_request.html| 4 | docs/source/conf.py 5 | ) 6 | repos: 7 | - repo: https://github.com/pre-commit/pre-commit-hooks 8 | rev: v5.0.0 9 | hooks: 10 | - id: trailing-whitespace 11 | - id: end-of-file-fixer 12 | - id: check-docstring-first 13 | - id: check-json 14 | - id: check-yaml 15 | - id: pretty-format-json 16 | args: ["--autofix", "--indent=2", "--no-sort-keys"] 17 | 18 | - repo: https://github.com/PyCQA/isort 19 | rev: 6.0.1 20 | hooks: 21 | - id: isort 22 | args: ["--profile", "black", "--filter-files"] 23 | 24 | - repo: https://github.com/psf/black 25 | rev: 25.1.0 26 | hooks: 27 | - id: black 28 | args: ["--line-length", "100"] 29 | 30 | - repo: https://github.com/PyCQA/flake8 31 | rev: 7.1.2 32 | hooks: 33 | - id: flake8 34 | 35 | - repo: https://github.com/codespell-project/codespell 36 | rev: v2.4.1 37 | hooks: 38 | - id: codespell 39 | # Ignores `.ipynb` files, `_build` and `_static` folders 40 | args: ["--skip=*.ipynb,docs", "--ignore-words-list=authentification", "-w", "docs", "ooipy"] 41 | 42 | ci: 43 | autofix_commit_msg: | 44 | [pre-commit.ci] auto fixes from pre-commit.com hooks 45 | 46 | for more information, see https://pre-commit.ci 47 | autofix_prs: true 48 | autoupdate_branch: '' 49 | autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' 50 | autoupdate_schedule: weekly 51 | skip: [] 52 | submodules: false 53 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version and other tools you might need 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.12" 12 | 13 | # Build documentation in the "docs/" directory with Sphinx 14 | sphinx: 15 | configuration: docs/conf.py 16 | # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs 17 | # builder: "dirhtml" 18 | # Fail on all warnings to avoid broken references 19 | # fail_on_warning: true 20 | 21 | 22 | python: 23 | install: 24 | - method: pip 25 | path: . 26 | extra_requirements: 27 | - docs 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 202x ooipy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OOIPY 2 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) ![unit test](https://github.com/Ocean-Data-Lab/ooipy/actions/workflows/unit_test.yml/badge.svg) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4276862.svg)](https://doi.org/10.5281/zenodo.4276862) [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/Ocean-Data-Lab/ooipy/master.svg)](https://results.pre-commit.ci/latest/github/Ocean-Data-Lab/ooipy/master) [![Documentation](https://readthedocs.org/projects/ooipy/badge/?version=latest)](https://ooipy.readthedocs.io/en/latest/?badge=latest) 3 | 4 | 5 | 6 | 7 | 8 | OOIPY is a python toolbox designed to aid in scientific analysis of Ocean Observatories Initiative (OOI) data. Some data (such as broadband hydrophone data) is not available through the OOI API (M2M). This package is designed to help with the acquiring of datasets from the OOI Raw Data Server. Additionally, tools to analyze the data, such as spectrogram and power spectral density plotting are also provided. 9 | -------------------------------------------------------------------------------- /docs/api_docs.md: -------------------------------------------------------------------------------- 1 | # API Documentation 2 | This module provides the tools to download OOI data from- the OOI Raw Data 3 | Server. 4 | 5 | ## Request Module 6 | Tools for downloading OOI data 7 | 8 | ### Hydrophone Request 9 | - [Oregon Shelf Base Seafloor (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=CE02SHBP-LJ01D-11-HYDBBA106) 10 | - 'LJ01D' 11 | - [Oregon Slope Base Seafloor (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=RS01SLBS-LJ01A-09-HYDBBA102) 12 | - 'LJ01A' 13 | - [Slope Base Shallow (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=RS01SBPS-PC01A-08-HYDBBA103) 14 | - 'PC01A' 15 | - [Axial Base Shallow Profiler (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=RS03AXPS-PC03A-08-HYDBBA303) 16 | - 'PC03A' 17 | - [Offshore Base Seafloor (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=CE04OSBP-LJ01C-11-HYDBBA105) 18 | - 'LJ01C' 19 | - [Axial Base Seafloor (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=RS03AXBS-LJ03A-09-HYDBBA302) 20 | - 'LJ03A' 21 | - [Axial Base Seafloor (Fs = 200 Hz)](https://ooinet.oceanobservatories.org/data_access/?search=RS03AXBS-MJ03A-05-HYDLFA301) 22 | - 'Axial_Base' 23 | - 'AXABA1' 24 | - [Central Caldera (Fs = 200 Hz)](https://ooinet.oceanobservatories.org/data_access/?search=RS03CCAL-MJ03F-06-HYDLFA305) 25 | - 'Central_Caldera' 26 | - 'AXCC1' 27 | - [Eastern Caldera (Fs = 200 Hz)](https://ooinet.oceanobservatories.org/data_access/?search=RS03ECAL-MJ03E-09-HYDLFA304) 28 | - 'Eastern_Caldera' 29 | - 'AXEC2' 30 | - [Southern Hydrate (Fs = 200 Hz)](https://ooinet.oceanobservatories.org/data_access/?search=RS01SUM1-LJ01B-05-HYDLFA104) 31 | - 'Southern_Hydrate' 32 | - 'HYS14' 33 | - [Oregon Slope Base Seafloor (Fs = 200 Hz)](https://ooinet.oceanobservatories.org/data_access/?search=RS01SLBS-MJ01A-05-HYDLFA101) 34 | - 'Slope_Base' 35 | - 'HYSB1' 36 | 37 | ```{eval-rst} 38 | .. automodule:: ooipy.request.hydrophone_request 39 | :members: 40 | ``` 41 | 42 | ### CTD Request 43 | ```{eval-rst} 44 | .. automodule:: ooipy.request.ctd_request 45 | :members: 46 | ``` 47 | 48 | ## Hydrophone data object 49 | ```{eval-rst} 50 | .. automodule:: ooipy.hydrophone.basic 51 | :members: 52 | ``` 53 | 54 | ## CTD data object 55 | ```{eval-rst} 56 | .. automodule:: ooipy.ctd.basic 57 | :members: 58 | ``` 59 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import importlib.metadata 4 | from typing import Any 5 | 6 | project = "ooipy" 7 | copyright = "2024" 8 | maintainer = "John Ragland" 9 | version = release = importlib.metadata.version("ooipy") 10 | 11 | extensions = [ 12 | "myst_parser", 13 | "sphinx.ext.autodoc", 14 | "sphinx.ext.intersphinx", 15 | "sphinx.ext.mathjax", 16 | "sphinx.ext.napoleon", 17 | "sphinx_autodoc_typehints", 18 | "sphinx_copybutton", 19 | "sphinxcontrib.bibtex", 20 | ] 21 | 22 | source_suffix = [".rst", ".md"] 23 | exclude_patterns = [ 24 | "_build", 25 | "**.ipynb_checkpoints", 26 | "Thumbs.db", 27 | ".DS_Store", 28 | ".env", 29 | ".venv", 30 | ] 31 | 32 | html_theme = "furo" 33 | html_theme_options: dict[str, Any] = { 34 | "footer_icons": [ 35 | { 36 | "name": "GitHub", 37 | "url": "https://github.com/Ocean-Data-Lab/ooipy", 38 | "class": "", 39 | }, 40 | ], 41 | "source_repository": "https://github.com/Ocean-Data-Lab/ooipy", 42 | "source_branch": "main", 43 | "source_directory": "docs/", 44 | } 45 | 46 | html_favicon = "../imgs/OOIPY_favicon.ico" 47 | 48 | myst_enable_extensions = [ 49 | "colon_fence", 50 | "html_admonition", 51 | "html_image", 52 | ] 53 | 54 | intersphinx_mapping = { 55 | "python": ("https://docs.python.org/3", None), 56 | "scipy": ("https://docs.scipy.org/doc/scipy/", None), 57 | "obspy": ("https://docs.obspy.org/", "https://docs.obspy.org/objects.inv"), 58 | "xarray": ("https://xarray.pydata.org/en/stable/", None), 59 | } 60 | 61 | nitpick_ignore = [ 62 | ("py:class", "_io.StringIO"), 63 | ("py:class", "_io.BytesIO"), 64 | ] 65 | 66 | always_document_param_types = True 67 | 68 | bibtex_bibfiles = ["refs.bib"] 69 | bibtex_default_style = "unsrt" 70 | bibtex_reference_style = "author_year" 71 | -------------------------------------------------------------------------------- /docs/figures/OOIPY_Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/docs/figures/OOIPY_Logo.png -------------------------------------------------------------------------------- /docs/figures/ctd_profile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/docs/figures/ctd_profile.png -------------------------------------------------------------------------------- /docs/figures/psd_bb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/docs/figures/psd_bb.png -------------------------------------------------------------------------------- /docs/figures/psd_mean_median.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/docs/figures/psd_mean_median.png -------------------------------------------------------------------------------- /docs/getting_started.md: -------------------------------------------------------------------------------- 1 | # User Guide 2 | 3 | ## Installation 4 | 5 | you can install ooipy with pip 6 | 7 | ```console 8 | pip install ooipy 9 | ``` 10 | 11 | If you want to get latest (unreleast) versions, or contribute to development you can clone the repository and install it from the source code. 12 | 13 | ```console 14 | git clone https://github.com/Ocean-Data-Lab/ooipy.git 15 | cd ooipy 16 | pip install -e . 17 | ``` 18 | The -e flag allows you to edit the source code and not have to reinstall the package. 19 | 20 | The installation has several extras for different development cases ['dev','docs']. These optional dependencies can be installed with `pip install -e .[dev]` or `pip install -e .[docs]`. 21 | 22 | ## Download Hydrophone Data 23 | 24 | How to download data from broadband hydrophones 25 | ```python 26 | import ooipy 27 | import datetime 28 | from matplotlib import pyplot as plt 29 | 30 | # Specify start time, end time, and node for data download (1 minutes of data) 31 | start_time = datetime.datetime(2017,7,1,0,0,0) 32 | end_time = datetime.datetime(2017,7,1,0,1,0) 33 | node1 = 'LJ01D' 34 | 35 | # Download Broadband data 36 | print('Downloading Broadband Data:') 37 | hdata_broadband = ooipy.get_acoustic_data(start_time, end_time, node1, verbose=True) 38 | ``` 39 | 40 | How to download data from low frequency hydrophones 41 | ```python 42 | start_time = datetime.datetime(2017,7,1,0,0,0) 43 | end_time = datetime.datetime(2017,7,1,0,1,0) 44 | node='Eastern_Caldera' 45 | 46 | # Download low frequency data 47 | print('Downloading Low Frequency Data:') 48 | hdata_lowfreq = ooipy.get_acoustic_data_LF(start_time, end_time, node2, verbose=True, zero_mean=True) 49 | ``` 50 | 51 | The {py:func}`ooipy.hydrophone.basic.HydrophoneData` object has all of the functionality of the {py:class}`obspy.core.trace.Trace`, which includes plotting, resampling, filtering and more. See obspy documentation for more information. 52 | 53 | ### Hydrophone nodes 54 | 55 | **Broadband Hydrophones** 56 | * [Oregon Shelf Base Seafloor (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=CE02SHBP-LJ01D-11-HYDBBA106) 57 | * 'LJ01D' 58 | * [Oregon Slope Base Seafloor (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=RS01SLBS-LJ01A-09-HYDBBA102) 59 | * 'LJ01A' 60 | * [Slope Base Shallow (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=RS01SBPS-PC01A-08-HYDBBA103) 61 | * 'PC01A' 62 | * [Axial Base Shallow Profiler (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=RS03AXPS-PC03A-08-HYDBBA303) 63 | * 'PC03A' 64 | * [Offshore Base Seafloor (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=CE04OSBP-LJ01C-11-HYDBBA105) 65 | * 'LJ01C' 66 | * [Axial Base Seafloor (Fs = 64 kHz)](https://ooinet.oceanobservatories.org/data_access/?search=RS03AXBS-LJ03A-09-HYDBBA302) 67 | * 'LJ03A' 68 | 69 | **Low Frequency Hydrophones** 70 | * [Axial Base Seaflor (Fs = 200 Hz)](https://ooinet.oceanobservatories.org/data_access/?search=RS03AXBS-MJ03A-05-HYDLFA301) 71 | * 'Axial_Base' 72 | * 'AXABA1' 73 | * [Central Caldera (Fs = 200 Hz)](https://ooinet.oceanobservatories.org/data_access/?search=RS03CCAL-MJ03F-06-HYDLFA305) 74 | * 'Central_Caldera' 75 | * 'AXCC1' 76 | * [Eastern Caldera (Fs = 200 Hz)](https://ooinet.oceanobservatories.org/data_access/?search=RS03ECAL-MJ03E-09-HYDLFA304) 77 | * 'Eastern_Caldera' 78 | * 'AXEC2' 79 | * [Southern Hydrate (Fs = 200 Hz)](https://ooinet.oceanobservatories.org/data_access/?search=RS01SUM1-LJ01B-05-HYDLFA104) 80 | * 'Southern_Hydrate' 81 | * 'HYS14' 82 | * ['Oregon Slope Base Seafloor (Fs = 200 Hz)](https://ooinet.oceanobservatories.org/data_access/?search=RS01SLBS-MJ01A-05-HYDLFA101) 83 | * 'Slope_Base' 84 | * 'HYSB1' 85 | 86 | For more detailed information about hydrophones nodes see https://ooinet.oceanobservatories.org/ 87 | 88 | Here is a [map of hydrophones supported by OOIpy](https://www.google.com/maps/d/u/1/viewer?mid=1_QKOPTxX2m5CTwgKR5fAGLO0lmbBgT7w&ll=45.16765319565428%2C-127.15744999999998&z=7) 89 | 90 | ## Compute PSDs and Spectrograms 91 | The {py:class}`ooipy.hydrophone.basic.HydrophoneData` data object, which is a wrapper for the {py:class}`obspy.core.trace.Trace` provides methods to compute Power Spectral Densities, and Spectrograms. The spectrograms are actually multiple power-spectral density estimates as a function of time, instead of a typical short-time fourier transform, such as {py:func}`scipy.signal.stft`. 92 | 93 | The OOI hydrophones often have colocated instruments that can corrupt than ambient sound measurements. We've found that the welch method with median averaging {footcite}`schwock2021e` gives the best result for power spectral density estimates, and is what has been used for much research on spectral levels with OOI hydrophones {footcite}`ragland2022, schwock2021d, schwock2021b` 94 | 95 | ### Calibration 96 | ooipy also handles calibration of the hydrophones. The low-frequency hydrophones were calibrated before their deployment in 2014, and have not been calibrated since. The calibration information for the low frequency hydrophones can be found on the [IRIS website](http://ds.iris.edu/mda/OO/). 97 | 98 | The broadband hydrophones are recovered every year and calibrated by Ocean Sonics, the information about hydrophone deployments can be found [here](https://github.com/OOI-CabledArray/deployments/blob/main/HYDBBA_deployments.csv), and the calibration sheets can be found [here](https://github.com/OOI-CabledArray/calibrationFiles/tree/master/HYDBBA). 99 | 100 | :::{warning} 101 | Calibration needs to be updated. The deployments and calibration files are not updated past 2021. 102 | ::: 103 | 104 | ### compute Power spectral density 105 | 106 | Use the {py:meth}`ooipy.hydrophone.basic.HydrophoneData.compute_psd_welch` method to estimate the power spectral density. The power spectral density estimate is returned as an {py:class}`xarray.DataArray`. 107 | 108 | ```python 109 | psd1 = hdata_broadband.compute_psd_welch() 110 | psd1.plot() 111 | plt.xlim([10,32000]) 112 | plt.grid() 113 | plt.xscale('log') 114 | ``` 115 | 116 | psd 117 | 118 | ### difference between different averaging methods 119 | 120 | ```python 121 | # power spectral density estimate of noise data using Welch's method 122 | fig, ax = plt.subplots(figsize=(6,4)) 123 | 124 | # 1. using median averaging (default) 125 | psd_med = hdata_broadband.compute_psd_welch() 126 | 127 | # 2. using mean averaging 128 | psd_mean = hdata_broadband.compute_psd_welch(avg_method='mean') 129 | 130 | psd_med.plot(c='k', alpha=0.8, label='median') 131 | psd_mean.plot(c='r', alpha=0.8, label='mean') 132 | 133 | 134 | plt.xlabel('frequency [kHz]') 135 | plt.ylabel('SDF [dB re µPa**2/Hz]') 136 | plt.xlim(1000,25000) 137 | plt.ylim(25,70) 138 | plt.legend() 139 | plt.grid() 140 | ``` 141 | Power Spectral Density - Mean vs Median 142 | 143 | ### Compute Spectrogram 144 | Use the {py:meth}`ooipy.hydrophone.basic.HydropheData.compute_spectrogram` method to compute the spectrogram. 145 | 146 | ```python 147 | spec1 = hdata_broadband.compute_spectrogram() 148 | ``` 149 | 150 | ## Batch hydrophone downloads 151 | 152 | There is a program for batch downloading ooi hydrophone data provided with the installation of ooipy. You can access it from your terminal: 153 | ```console 154 | download_hydrophone_data --csv --output_path 155 | ``` 156 | 157 | Here is an example csv file: 158 | 159 | ```csv 160 | node,start_time,end_time,file_format,downsample_factor 161 | LJ03A,2019-08-03T08:00:00,2019-08-03T08:01:00,wav,1 162 | AXBA1,2019-08-03T12:01:00,2019-08-03T12:02:00,wav,1 163 | ``` 164 | 165 | ## Download CTD Data 166 | :::{warning} 167 | CTD downloads currently works, but are not actively supported. You should first look at the [OOI Data Explorer](https://dataexplorer.oceanobservatories.org/). 168 | ::: 169 | 170 | import packages and initialize your api token 171 | ```python 172 | import ooipy 173 | import datetime 174 | from matplotlib import pyplot as plt 175 | 176 | USERNAME = <'YOUR_USERNAME'> 177 | TOKEN = <'YOUR_TOKEN'> 178 | ooipy.request.authentification.set_authentification(USERNAME, TOKEN) 179 | ``` 180 | request 1-hour of CTD data from the oregonoffshore location 181 | ```python 182 | start = datetime.datetime(2016, 12, 12, 2, 0, 0) 183 | end = datetime.datetime(2016, 12, 12, 3, 0, 0) 184 | ctd_data = ooipy.request.ctd_request.get_ctd_data(start, end, 'oregon_offshore', limit=10000) 185 | print('CTD data object: ', ctd_data) 186 | print('number of data points: ', len(ctd_data.raw_data)) 187 | print('first data point: ', ctd_data.raw_data[0]) 188 | ``` 189 | 190 | request 1-day of CTD data from the oregonoffshore location 191 | ```python 192 | import time 193 | day = datetime.datetime(2016, 12, 12) 194 | t = time.time() 195 | ctd_data = ooipy.request.ctd_request.get_ctd_data_daily(day, 'oregon_offshore') 196 | print(time.time() - t) 197 | print('CTD data object: ', ctd_data) 198 | print('number of data points: ', len(ctd_data.raw_data)) 199 | print('first data point: ', ctd_data.raw_data[0]) 200 | ``` 201 | 202 | compute sound speed profile for daily data 203 | ```python 204 | c_profile = ctd_data.get_profile(600, 'sound_speed') 205 | ``` 206 | 207 | plot ctd profile mean and standard deviation 208 | ```python 209 | c_profile.plot(xlabel='sound speed') 210 | ``` 211 | Sound speed profile 212 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # ooipy 2 | 3 | 4 | 5 | ```{include} ../README.md 6 | :start-after: 7 | ``` 8 | **Sections** 9 | ```{toctree} 10 | :maxdepth: 2 11 | getting_started 12 | api_docs 13 | ``` 14 | -------------------------------------------------------------------------------- /docs/refs.bib: -------------------------------------------------------------------------------- 1 | @article{ragland2022, 2 | title = {An Overview of Ambient Sound Using {{Ocean Observatories Initiative}} Hydrophones}, 3 | author = {Ragland, John and Schwock, Felix and Munson, Matthew and Abadi, Shima}, 4 | year = {2022}, 5 | month = mar, 6 | journal = {The Journal of the Acoustical Society of America}, 7 | volume = {151}, 8 | number = {3}, 9 | pages = {2085--2100}, 10 | publisher = {Acoustical Society of America}, 11 | issn = {0001-4966}, 12 | doi = {10.1121/10.0009836}, 13 | urldate = {2022-04-04}, 14 | abstract = {The Ocean Observatories Initiative (OOI) sensor network provides a unique opportunity to study ambient sound in the north-east Pacific Ocean. The OOI sensor network has five low frequency (Fs = 200\,Hz) and six broadband (Fs = 64 kHz) hydrophones that have been recording ambient sound since 2015. In this paper, we analyze acoustic data from 2015 to 2020 to identify prominent features that are present in the OOI acoustic dataset. Notable features in the acoustic dataset that are highlighted in this paper include volcanic and seismic activity, rain and wind noise, marine mammal vocalizations, and anthropogenic sound, such as shipping noise. For all low frequency hydrophones and four of the six broadband hydrophones, we will present long-term spectrograms, median time-series trends for different spectral bands, and different statistical metrics about the acoustic environment. We find that 6-yr acoustic trends vary, depending on the location of the hydrophone and the spectral band that is observed. Over the course of six years, increases in spectral levels are seen in some locations and spectral bands, while decreases are seen in other locations and spectral bands. Last, we discuss future areas of research to which the OOI dataset lends itself.}, 15 | file = {/Users/john/Library/CloudStorage/OneDrive-UW/zotero_library/Ragland et al_2022_An overview of ambient sound using Ocean Observatories Initiative hydrophones.pdf} 16 | } 17 | 18 | @article{schwock2021d, 19 | title = {Characterizing Underwater Noise during Rain at the Northeast {{Pacific}} Continental Margin}, 20 | author = {Schwock, Felix and Abadi, Shima}, 21 | year = {2021}, 22 | month = jun, 23 | journal = {The Journal of the Acoustical Society of America}, 24 | volume = {149}, 25 | number = {6}, 26 | pages = {4579--4595}, 27 | publisher = {Acoustical Society of America}, 28 | issn = {0001-4966}, 29 | doi = {10.1121/10.0005440}, 30 | urldate = {2022-04-04}, 31 | abstract = {Large scale studies of underwater noise during rain are important for assessing the ocean environment and enabling remote sensing of rain rates over the open ocean. In this study, approximately 3.5\,yrs of acoustical and meteorological data recorded at the northeast Pacific continental margin are evaluated. The acoustic data are recorded at a sampling rate of 64\,kHz and depths of 81 and 581\,m at the continental shelf and slope, respectively. Rain rates and wind speeds are provided by surface buoys located in the vicinity of each hydrophone. Average power spectra have been computed for different rain rates and wind speeds, and linear and nonlinear regression have been performed. The main findings are (1) the linear regression slopes highly depends on the frequency range, rain rate, wind speed, and measurement depth; (2) noise levels during rain between 200\,Hz and 10\,kHz significantly increase with increasing wind speed; and (3) the highest correlation between the spectral level and rain rate occurs at 13\,kHz, thus, coinciding with the spectral peak due to small raindrops. The results of this study indicate that previously proposed algorithms for estimating rain rates from acoustic data are not universally applicable but rather have to be adapted for different locations.}, 32 | file = {/Users/john/Library/CloudStorage/OneDrive-UW/zotero_library/Schwock_Abadi_2021_Characterizing underwater noise during rain at the northeast Pacific.pdf} 33 | } 34 | 35 | @inproceedings{schwock2021e, 36 | title = {Statistical {{Properties}} of a {{Modified Welch Method That Uses Sample Percentiles}}}, 37 | booktitle = {{{ICASSP}} 2021 - 2021 {{IEEE International Conference}} on {{Acoustics}}, {{Speech}} and {{Signal Processing}} ({{ICASSP}})}, 38 | author = {Schwock, Felix and Abadi, Shima}, 39 | year = {2021}, 40 | month = jun, 41 | pages = {5165--5169}, 42 | issn = {2379-190X}, 43 | doi = {10.1109/ICASSP39728.2021.9415074}, 44 | abstract = {We present and analyze an alternative, more robust approach to the Welch's overlapped segment averaging (WOSA) spectral estimator. Our method computes sample percentiles instead of averaging over multiple periodograms to estimate power spectral densities (PSDs). Bias and variance of the proposed estimator are derived for varying sample sizes and arbitrary percentiles. We have found excellent agreement between our expressions and data sampled from a white Gaussian noise process.}, 45 | keywords = {Acoustics,Conferences,Estimation variance,Gaussian noise,Mathematical model,Signal processing,Spectral estimation,Speech processing,Welch method}, 46 | file = {/Users/john/Zotero/storage/B88P3ML9/Schwock and Abadi - 2021 - Statistical Properties of a Modified Welch Method .pdf;/Users/john/Zotero/storage/UNZ2MF4L/9415074.html} 47 | } 48 | 49 | @article{schwock2021b, 50 | title = {Statistical Analysis and Modeling of Underwater Wind Noise at the Northeast Pacific Continental Margin}, 51 | author = {Schwock, Felix and Abadi, Shima}, 52 | year = {2021}, 53 | month = dec, 54 | journal = {The Journal of the Acoustical Society of America}, 55 | volume = {150}, 56 | number = {6}, 57 | pages = {4166--4177}, 58 | publisher = {Acoustical Society of America}, 59 | issn = {0001-4966}, 60 | doi = {10.1121/10.0007463}, 61 | urldate = {2022-04-04}, 62 | abstract = {Approximately 11\,400\,h of acoustic recordings from two sites off the Oregon coast have been evaluated to characterize and model the frequency and wind dependence of wind noise in the northeast Pacific continental margin. Acoustic data are provided by two bottom-mounted broadband hydrophones (64\,kHz sampling frequency) deployed at depths of 81 and 581\,m at the continental shelf and slope, respectively. To describe the spectral level versus frequency relation, separate linear models for the 0.2--3\,kHz and 3--25\,kHz frequency range are fitted to the data. While spectral slopes for the 0.2--3\,kHz range generally decrease with increasing wind speed, slopes remain constant (shallow location) or increase with increasing wind speed (deep location) above 3\,kHz. The latter is in strong contrast to results from previous studies. The relation between spectral level and wind speed is described by a piecewise linear model where spectral levels are approximately constant below a critical wind speed {$v$} c vc and increase linearly with logarithmic wind speed above {$v$} c vc . It is shown that the critical wind speed and the slopes of the piecewise linear model strongly depend on the acoustic frequency.}, 63 | file = {/Users/john/Library/CloudStorage/OneDrive-UW/zotero_library/Schwock_Abadi_2021_Statistical analysis and modeling of underwater wind noise at the northeast.pdf} 64 | } 65 | -------------------------------------------------------------------------------- /imgs/OOIPY_Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/imgs/OOIPY_Logo.png -------------------------------------------------------------------------------- /imgs/OOIPY_favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/imgs/OOIPY_favicon.ico -------------------------------------------------------------------------------- /imgs/ooipy_banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/imgs/ooipy_banner.png -------------------------------------------------------------------------------- /imgs/ooipy_banner2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/imgs/ooipy_banner2.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling", "hatch-vcs"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "ooipy" 7 | dynamic = ["version"] 8 | description = 'toolbox for downloading and analyzing OOI hydrophone data' 9 | readme = "README.md" 10 | requires-python = ">=3.8" 11 | license = "MIT" 12 | keywords = [] 13 | maintainers = [ 14 | { name = "John Ragland", email = "jhrag@uw.edu" }, 15 | ] 16 | classifiers = [ 17 | "Development Status :: 4 - Beta", 18 | "Programming Language :: Python", 19 | "Programming Language :: Python :: 3.8", 20 | "Programming Language :: Python :: 3.9", 21 | "Programming Language :: Python :: 3.10", 22 | "Programming Language :: Python :: 3.11", 23 | "Programming Language :: Python :: 3.12", 24 | "Programming Language :: Python :: Implementation :: CPython", 25 | "Programming Language :: Python :: Implementation :: PyPy", 26 | ] 27 | dependencies = [ 28 | "fsspec", 29 | "xarray", 30 | "obspy", 31 | "pandas", 32 | "numpy", 33 | "tqdm", 34 | "aiohttp", 35 | "netcdf4" 36 | ] 37 | 38 | [project.optional-dependencies] 39 | test = [ 40 | "pytest >= 6.0", 41 | ] 42 | dev = [ 43 | "pre-commit", 44 | "black", 45 | ] 46 | docs = [ 47 | "furo", 48 | "myst_parser >=0.13", 49 | "sphinx >=4.0", 50 | "sphinx-copybutton", 51 | "sphinx-autodoc-typehints", 52 | "sphinxcontrib-bibtex" 53 | ] 54 | 55 | [project.urls] 56 | Documentation = "https://ooipy.readthedocs.io" 57 | Issues = "https://github.com/Ocean-Data-Lab/ooipy/issues" 58 | Source = "https://github.com/Ocean-Data-Lab/ooipy" 59 | 60 | [project.scripts] 61 | download_hydrophone_data = "ooipy.scripts.download_hydrophone_data:main" 62 | 63 | [tool.hatch.version] 64 | source = "vcs" 65 | 66 | [tool.hatch.version.raw-options] 67 | local_scheme = "no-local-version" 68 | 69 | [tool.hatch.build.hooks.vcs] 70 | version-file = "src/ooipy/version.py" 71 | 72 | [tool.hatch.envs.types] 73 | extra-dependencies = [ 74 | "mypy>=1.0.0", 75 | ] 76 | [tool.hatch.envs.types.scripts] 77 | check = "mypy --install-types --non-interactive {args:src/ooipy tests}" 78 | 79 | [tool.coverage.run] 80 | source_pkgs = ["ooipy", "tests"] 81 | branch = true 82 | parallel = true 83 | omit = [ 84 | "src/ooipy/__about__.py", 85 | ] 86 | 87 | [tool.coverage.paths] 88 | ooipy = ["src/ooipy", "*/ooipy/src/ooipy"] 89 | tests = ["tests", "*/ooipy/tests"] 90 | 91 | [tool.coverage.report] 92 | exclude_lines = [ 93 | "no cov", 94 | "if __name__ == .__main__.:", 95 | "if TYPE_CHECKING:", 96 | ] 97 | 98 | [tool.pytest.ini_options] 99 | filterwarnings = [ 100 | "error", 101 | "ignore::DeprecationWarning", # temporarily adding for python 3.13 support 102 | ] 103 | # adding catch for pkg_resources API deprecation warning until obspy updates 104 | minversion = "6.0" 105 | addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] 106 | xfail_strict = true 107 | log_cli_level = "info" 108 | testpaths = [ 109 | "tests", 110 | ] 111 | 112 | [tool.black] 113 | line-length = 100 114 | -------------------------------------------------------------------------------- /src/ooipy/__init__.py: -------------------------------------------------------------------------------- 1 | from ooipy.request.authentification import set_authentification 2 | from ooipy.request.ctd_request import get_ctd_data, get_ctd_data_daily 3 | from ooipy.request.hydrophone_request import get_acoustic_data, get_acoustic_data_LF 4 | 5 | __all__ = [ 6 | set_authentification, 7 | get_ctd_data, 8 | get_ctd_data_daily, 9 | get_acoustic_data, 10 | get_acoustic_data_LF, 11 | ] 12 | -------------------------------------------------------------------------------- /src/ooipy/ctd/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/src/ooipy/ctd/__init__.py -------------------------------------------------------------------------------- /src/ooipy/ctd/basic.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for CTD data objects 3 | """ 4 | 5 | import datetime 6 | 7 | import numpy as np 8 | 9 | import ooipy 10 | 11 | 12 | class CtdData: 13 | """ 14 | Object that stores conductivity, temperature, depth (CTD) data, and 15 | provides functions for calculating sound speed, temperature, 16 | pressure, and salinity profiles. When a CtdData object is created 17 | and extract_parameters = True (default), then temperature, pressure, 18 | salinity, and time are automatically extracted from the raw data. 19 | 20 | Attributes 21 | ---------- 22 | raw_data : list of dict 23 | list containing sample from CTD. Each sample is a dictionary 24 | containing all parameters measured by the CTD. 25 | temperature : numpy.ndarray 26 | array containing temperature samples in degree celsius. 27 | pressure : numpy.ndarray 28 | array containing pressure samples in dbar. 29 | salinity : numpy.ndarray 30 | array containing salinity samples in parts per thousand. 31 | depth : numpy.ndarray 32 | array containing depth samples in meter. 33 | density : numpy.ndarray 34 | array containing density samples in kg/cubic meter. 35 | conductivity : numpy.ndarray 36 | array containing conductivity samples in siemens/meter. 37 | sound_speed : numpy.ndarray 38 | array containing sound speed samples in meter/second. 39 | time : numpy.ndarray 40 | array containing time samples as datetime.datetime objects. 41 | sound_speed_profile : :class:`ooipy.ctd.basic.CtdProfile` 42 | object for sound speed profile. 43 | temperature_profile : :class:`ooipy.ctd.basic.CtdProfile` 44 | object for temperature profile. 45 | salinity_profile : :class:`ooipy.ctd.basic.CtdProfile` 46 | object for salinity profile. 47 | pressure_profile : :class:`ooipy.ctd.basic.CtdProfile` 48 | object for pressure profile. 49 | density_profile : :class:`ooipy.ctd.basic.CtdProfile` 50 | object for density profile. 51 | conductivity_profile : :class:`ooipy.ctd.basic.CtdProfile` 52 | object for conductivity profile. 53 | 54 | """ 55 | 56 | def __init__(self, raw_data=None, extract_parameters=True): 57 | self.raw_data = raw_data 58 | 59 | if self.raw_data is not None and extract_parameters: 60 | self.temperature = self.get_parameter_from_rawdata("temperature") 61 | self.pressure = self.get_parameter_from_rawdata("pressure") 62 | self.salinity = self.get_parameter_from_rawdata("salinity") 63 | self.time = self.get_parameter_from_rawdata("time") 64 | else: 65 | self.temperature = None 66 | self.pressure = None 67 | self.salinity = None 68 | self.time = None 69 | 70 | self.sound_speed = None 71 | self.depth = None 72 | self.density = None 73 | self.conductivity = None 74 | self.sound_speed_profile = None 75 | self.temperature_profile = None 76 | self.salinity_profile = None 77 | self.pressure_profile = None 78 | self.density_profile = None 79 | self.conductivity_profile = None 80 | 81 | def ntp_seconds_to_datetime(self, ntp_seconds): 82 | """ 83 | Converts timestamp into dattime object. 84 | """ 85 | ntp_epoch = datetime.datetime(1900, 1, 1) 86 | unix_epoch = datetime.datetime(1970, 1, 1) 87 | ntp_delta = (unix_epoch - ntp_epoch).total_seconds() 88 | return datetime.datetime.utcfromtimestamp(ntp_seconds - ntp_delta).replace(microsecond=0) 89 | 90 | def get_parameter_from_rawdata(self, parameter): 91 | """ 92 | Extracts parameters from raw data dictionary. 93 | """ 94 | param_arr = [] 95 | for item in self.raw_data: 96 | if parameter == "temperature": 97 | if "seawater_temperature" in item: 98 | param_arr.append(item["seawater_temperature"]) 99 | elif "sea_water_temperature" in item: 100 | param_arr.append(item["sea_water_temperature"]) 101 | elif "temperature" in item: 102 | param_arr.append(item["temperature"]) 103 | else: 104 | param_arr.append(item["temp"]) 105 | if parameter == "pressure": 106 | if "ctdbp_no_seawater_pressure" in item: 107 | param_arr.append(item["ctdbp_no_seawater_pressure"]) 108 | elif "seawater_pressure" in item: 109 | param_arr.append(item["seawater_pressure"]) 110 | elif "sea_water_pressure" in item: 111 | param_arr.append(item["sea_water_pressure"]) 112 | else: 113 | param_arr.append(item["pressure"]) 114 | 115 | if parameter == "salinity": 116 | if "practical_salinity" in item: 117 | param_arr.append(item["practical_salinity"]) 118 | elif "sea_water_practical_salinity" in item: 119 | param_arr.append(item["sea_water_practical_salinity"]) 120 | else: 121 | param_arr.append(item["salinity"]) 122 | 123 | if parameter == "density": 124 | if "seawater_density" in item: 125 | param_arr.append(item["seawater_density"]) 126 | elif "sea_water_density" in item: 127 | param_arr.append(item["sea_water_density"]) 128 | else: 129 | param_arr.append(item["density"]) 130 | 131 | if parameter == "conductivity": 132 | if "ctdbp_no_seawater_conductivity" in item: 133 | param_arr.append(item["ctdbp_no_seawater_conductivity"]) 134 | elif "seawater_conductivity" in item: 135 | param_arr.append(item["seawater_conductivity"]) 136 | elif "dpc_ctd_seawater_conductivity" in item: 137 | param_arr.append(item["dpc_ctd_seawater_conductivity"]) 138 | else: 139 | param_arr.append(item["conductivity"]) 140 | 141 | if parameter == "time": 142 | param_arr.append(self.ntp_seconds_to_datetime(item["pk"]["time"])) 143 | 144 | return np.array(param_arr) 145 | 146 | def get_parameter(self, parameter): 147 | """ 148 | Extension of get_parameters_from_rawdata. Also sound speed and 149 | depth can be requested. 150 | """ 151 | if parameter in [ 152 | "temperature", 153 | "pressure", 154 | "salinity", 155 | "time", 156 | "density", 157 | "conductivity", 158 | ]: 159 | param = self.get_parameter_from_rawdata(parameter) 160 | elif parameter == "sound_speed": 161 | param = self.calc_sound_speed() 162 | elif parameter == "depth": 163 | param = self.calc_depth_from_pressure() 164 | else: 165 | param = None 166 | 167 | return param 168 | 169 | def calc_depth_from_pressure(self): 170 | """ 171 | Calculates depth from pressure array 172 | """ 173 | 174 | if self.pressure is None: 175 | self.pressure = self.get_parameter_from_rawdata("pressure") 176 | 177 | press_MPa = 0.01 * self.pressure 178 | 179 | # TODO: adapt for each hydrophone 180 | lat = 44.52757 # deg 181 | 182 | # Calculate gravity constant for given latitude 183 | g_phi = 9.780319 * ( 184 | 1 185 | + 5.2788e-3 * (np.sin(np.deg2rad(lat)) ** 2) 186 | + 2.36e-5 * (np.sin(np.deg2rad(lat)) ** 4) 187 | ) 188 | 189 | # Calculate Depth for Pressure array 190 | self.depth = ( 191 | 9.72659e2 * press_MPa 192 | - 2.512e-1 * press_MPa**2 193 | + 2.279e-4 * press_MPa**3 194 | - 1.82e-7 * press_MPa**4 195 | ) / (g_phi + 1.092e-4 * press_MPa) 196 | 197 | return self.depth 198 | 199 | def calc_sound_speed(self): 200 | """ 201 | Calculates sound speed from temperature, salinity and pressure 202 | array. The equation for calculating the sound speed is from: 203 | Chen, C. T., & Millero, F. J. (1977). Speed of sound in seawater 204 | at high pressures. Journal of the Acoustical Society of America, 205 | 62(5), 1129–1135. https://doi.org/10.1121/1.381646 206 | """ 207 | if self.pressure is None: 208 | self.pressure = self.get_parameter_from_rawdata("pressure") 209 | if self.temperature is None: 210 | self.temperature = self.get_parameter_from_rawdata("temperature") 211 | if self.salinity is None: 212 | self.salinity = self.get_parameter_from_rawdata("salinity") 213 | 214 | press_MPa = 0.01 * self.pressure 215 | 216 | C00 = 1402.388 217 | A02 = 7.166e-5 218 | C01 = 5.03830 219 | A03 = 2.008e-6 220 | C02 = -5.81090e-2 221 | A04 = -3.21e-8 222 | C03 = 3.3432e-4 223 | A10 = 9.4742e-5 224 | C04 = -1.47797e-6 225 | A11 = -1.2583e-5 226 | C05 = 3.1419e-9 227 | A12 = -6.4928e-8 228 | C10 = 0.153563 229 | A13 = 1.0515e-8 230 | C11 = 6.8999e-4 231 | A14 = -2.0142e-10 232 | C12 = -8.1829e-6 233 | A20 = -3.9064e-7 234 | C13 = 1.3632e-7 235 | A21 = 9.1061e-9 236 | C14 = -6.1260e-10 237 | A22 = -1.6009e-10 238 | C20 = 3.1260e-5 239 | A23 = 7.994e-12 240 | 241 | C21 = -1.7111e-6 242 | A30 = 1.100e-10 243 | C22 = 2.5986e-8 244 | A31 = 6.651e-12 245 | C23 = -2.5353e-10 246 | A32 = -3.391e-13 247 | C24 = 1.0415e-12 248 | B00 = -1.922e-2 249 | C30 = -9.7729e-9 250 | B01 = -4.42e-5 251 | C31 = 3.8513e-10 252 | B10 = 7.3637e-5 253 | C32 = -2.3654e-12 254 | B11 = 1.7950e-7 255 | A00 = 1.389 256 | D00 = 1.727e-3 257 | A01 = -1.262e-2 258 | D10 = -7.9836e-6 259 | 260 | T = 3 261 | S = 1 262 | P = 700 263 | T = self.temperature 264 | S = self.salinity 265 | P = press_MPa * 10 266 | 267 | D = D00 + D10 * P 268 | B = B00 + B01 * T + (B10 + B11 * T) * P 269 | A = ( 270 | (A00 + A01 * T + A02 * T**2 + A03 * T**3 + A04 * T**4) 271 | + (A10 + A11 * T + A12 * T**2 + A13 * T**3 + A14 * T**4) * P 272 | + (A20 + A21 * T + A22 * T**2 + A23 * T**3) * P**2 273 | + (A30 + A31 * T + A32 * T**2) * P**3 274 | ) 275 | Cw = ( 276 | (C00 + C01 * T + C02 * T**2 + C03 * T**3 + C04 * T**4 + C05 * T**5) 277 | + (C10 + C11 * T + C12 * T**2 + C13 * T**3 + C14 * T**4) * P 278 | + (C20 + C21 * T + C22 * T**2 + C23 * T**3 + C24 * T**4) * P**2 279 | + (C30 + C31 * T + C32 * T**2) * P**3 280 | ) 281 | 282 | # Calculate Speed of Sound 283 | self.sound_speed = Cw + A * S + B * S ** (3 / 2) + D * S**2 284 | 285 | return self.sound_speed 286 | 287 | def get_profile(self, max_depth, parameter): 288 | """ 289 | Compute the profile for sound speed, temperature, pressure, or 290 | salinity over the vater column. 291 | 292 | Parameters 293 | ---------- 294 | max_depth : int 295 | The profile will be computed from 0 meters to max_depth 296 | meters in 1-meter increments 297 | parameter : str 298 | * 'sound_speed' 299 | * 'temperature' 300 | * 'salinity' 301 | * 'pressure' 302 | * 'density' 303 | * 'conductivity' 304 | """ 305 | 306 | param_dct = {} 307 | for k in range(max_depth): 308 | param_dct[str(k)] = {"param": [], "d": []} 309 | 310 | param_arr = self.get_parameter(parameter) 311 | 312 | if self.depth is None: 313 | self.depth = self.get_parameter("depth") 314 | 315 | for d, p in zip(self.depth, param_arr): 316 | if str(int(d)) in param_dct: 317 | param_dct[str(int(d))]["d"].append(d) 318 | param_dct[str(int(d))]["param"].append(p) 319 | 320 | param_mean = [] 321 | depth_mean = [] 322 | param_var = [] 323 | depth_var = [] 324 | 325 | n_samp = [] 326 | 327 | for key in param_dct: 328 | param_mean.append(np.mean(param_dct[key]["param"])) 329 | depth_mean.append(np.mean(param_dct[key]["d"])) 330 | param_var.append(np.var(param_dct[key]["param"])) 331 | depth_var.append(np.var(param_dct[key]["d"])) 332 | n_samp.append(len(param_dct[key]["d"])) 333 | 334 | idx = np.argsort(depth_mean) 335 | 336 | depth_mean = np.array(depth_mean)[idx] 337 | param_mean = np.array(param_mean)[idx] 338 | depth_var = np.array(depth_var)[idx] 339 | param_var = np.array(param_var)[idx] 340 | n_samp = np.array(n_samp)[idx] 341 | 342 | param_profile = CtdProfile(param_mean, param_var, depth_mean, depth_var, n_samp) 343 | 344 | if parameter == "temperature": 345 | self.temperature_profile = param_profile 346 | elif parameter == "salinity": 347 | self.salinity_profile = param_profile 348 | elif parameter == "pressure": 349 | self.pressure_profile = param_profile 350 | elif parameter == "sound_speed": 351 | self.sound_speed_profile = param_profile 352 | elif parameter == "density": 353 | self.density_profile = param_profile 354 | elif parameter == "conductivity": 355 | self.conductivity_profile = param_profile 356 | 357 | return param_profile 358 | 359 | 360 | class CtdProfile: 361 | """ 362 | Simple object that stores a parameter profile over the water column. 363 | For each 1-meter interval, there is one data point in the profile. 364 | 365 | Attributes 366 | ---------- 367 | parameter_mean : array of float 368 | mean of parameter within each 1-meter depth interval 369 | parameter_var : array of float 370 | variance of parameter within each 1-meter depth interval 371 | depth_mean : array of float 372 | mean of depth within each 1-meter depth interval 373 | depth_var : array of float 374 | variance of depth within each 1-meter depth interval 375 | n_samp : array of int 376 | number of samples within each 1-meter depth interval 377 | """ 378 | 379 | def __init__(self, parameter_mean, parameter_var, depth_mean, depth_var, n_samp): 380 | self.parameter_mean = parameter_mean 381 | self.parameter_var = parameter_var 382 | self.depth_mean = depth_mean 383 | self.depth_var = depth_var 384 | self.n_samp = n_samp 385 | 386 | def plot(self, **kwargs): 387 | """ 388 | redirects to ooipy.ooiplotlib.plot_ctd_profile() 389 | please see :meth:`ooipy.hydrophone.basic.plot_psd` 390 | """ 391 | ooipy.tools.ooiplotlib.plot_ctd_profile(self, **kwargs) 392 | 393 | def convert_to_ssp(self): 394 | """ 395 | converts to numpy array with correct format for arlpy simulation 396 | 397 | Returns 398 | ------- 399 | ssp : numpy array 400 | 2D numpy array containing sound speed profile column 0 is depth, 401 | column 1 is sound speed (in m/s) 402 | """ 403 | ssp = np.vstack((self.depth_mean, self.parameter_mean)).T 404 | # insert 0 depth term 405 | ssp = np.insert(ssp, 0, np.array((0, ssp[0, 1])), 0) 406 | 407 | # remove NaN Terms 408 | first_nan = np.where(np.isnan(ssp))[0][0] 409 | ssp = ssp[: first_nan - 1, :] 410 | 411 | return ssp 412 | -------------------------------------------------------------------------------- /src/ooipy/hydrophone/__init__.py: -------------------------------------------------------------------------------- 1 | # TODO: this file ensures that the module 'basic' can be imported by calling ooipy.acoustic.basic 2 | -------------------------------------------------------------------------------- /src/ooipy/hydrophone/basic.py: -------------------------------------------------------------------------------- 1 | """ 2 | The {py:meth}`ooipy.get_acoustic_data` and {py:meth}`ooipy.get_acoustic_data_LF` 3 | functions return the {py:class}`ooipy.HydrophoneData` object. 4 | 5 | The :class:`ooipy.HydrophoneData` objects inherits from obspy.Trace, and methods for 6 | computing calibrated spectrograms and power spectral densities are added. 7 | """ 8 | 9 | import datetime 10 | import os 11 | import pickle 12 | import warnings 13 | 14 | import numpy as np 15 | import pandas as pd 16 | import xarray as xr 17 | from obspy import Trace 18 | from scipy import signal 19 | from scipy.interpolate import interp1d 20 | from scipy.io import savemat, wavfile 21 | 22 | import ooipy 23 | 24 | 25 | class HydrophoneData(Trace): 26 | """ 27 | Object that stores hydrophone data 28 | 29 | Attributes 30 | ---------- 31 | type : str 32 | Either 'broadband' or 'low_frequency' specifies the type of hydrophone 33 | that the date is from. 34 | 35 | """ 36 | 37 | def __init__(self, data=np.array([]), header=None, node=""): 38 | super().__init__(data, header) 39 | self.stats.location = node_id(node) 40 | 41 | self.spectrogram = None 42 | self.psd = None 43 | self.psd_list = None 44 | self.type = None 45 | 46 | # TODO: use correct frequency response for all hydrophones 47 | def frequency_calibration(self, N): 48 | """ 49 | Apply a frequency dependent sensitivity correction to the 50 | acoustic data based on the information from the calibration 51 | sheets. 52 | Hydrophone deployments are found at 53 | https://github.com/OOI-CabledArray/deployments 54 | Hydrophone calibration sheets are found at 55 | https://github.com/OOI-CabledArray/calibrationFiles 56 | Parameters 57 | ---------- 58 | N : int 59 | length of the data segment 60 | 61 | Returns 62 | ------- 63 | output_array : np.array 64 | array with correction coefficient for every frequency 65 | """ 66 | # Load calibation file and get appropriate calibration info 67 | filename = os.path.dirname(ooipy.__file__) + "/hydrophone/calibration_by_assetID.csv" 68 | # Use deployment CSV to determine asset_ID 69 | assetID = self.get_asset_ID() 70 | # load calibration data as pandas dataframe 71 | cal_by_assetID = pd.read_csv(filename, header=[0, 1]) 72 | 73 | f_calib = cal_by_assetID[assetID]["Freq (kHz)"].to_numpy() * 1000 74 | sens_calib_0 = cal_by_assetID[assetID]["0 phase"].to_numpy() 75 | sens_calib_90 = cal_by_assetID[assetID]["90 phase"].to_numpy() 76 | sens_calib = 0.5 * (sens_calib_0 + sens_calib_90) 77 | f = np.linspace(0, round(self.stats.sampling_rate / 2), N) 78 | 79 | # Convert calibration to correct units 80 | if round(self.stats.sampling_rate) == 200: 81 | if self.stats.channel == "HDH": 82 | sens_calib = 20 * np.log10(sens_calib * 1e-6) 83 | elif ( 84 | (self.stats.channel == "HNZ") 85 | | (self.stats.channel == "HNE") 86 | | (self.stats.channel == "HNN") 87 | ): 88 | sens_calib = 20 * np.log10(sens_calib) 89 | # units for seismograms are in dB rel to m/s^2 90 | elif round(self.stats.sampling_rate) == 64000: 91 | sens_calib = sens_calib + 128.9 92 | else: 93 | raise Exception("Invalid sampling rate") 94 | 95 | sens_interpolated = interp1d(f_calib, sens_calib) 96 | 97 | f_calib = sens_interpolated(f) 98 | return f_calib 99 | 100 | def compute_spectrogram( 101 | self, 102 | win="hann", 103 | L=4096, 104 | avg_time=None, 105 | overlap=0.5, 106 | verbose=True, 107 | average_type="median", 108 | ): 109 | """ 110 | Compute spectrogram of acoustic signal. For each time step of the 111 | spectrogram either a modified periodogram (avg_time=None) 112 | or a power spectral density estimate using Welch's method with median 113 | or mean averaging is computed. 114 | 115 | Parameters 116 | ---------- 117 | win : str, optional 118 | Window function used to taper the data. See scipy.signal.get_window 119 | for a list of possible window functions (Default is Hann-window.) 120 | L : int, optional 121 | Length of each data block for computing the FFT (Default is 4096). 122 | avg_time : float, optional 123 | Time in seconds that is covered in one time step of the 124 | spectrogram. Default value is None and one time step covers L 125 | samples. If the signal covers a long time period it is recommended 126 | to use a higher value for avg_time to avoid memory overflows and 127 | to facilitate visualization. 128 | overlap : float, optional 129 | Percentage of overlap between adjacent blocks if Welch's method is 130 | used. Parameter is ignored if avg_time is None. (Default is 50%) 131 | verbose : bool, optional 132 | If true (default), exception messages and some comments are printed. 133 | average_type : str 134 | type of averaging if Welch PSD estimate is used. options are 135 | 'median' (default) and 'mean'. 136 | 137 | Returns 138 | ------- 139 | spectrogram : xr.DataArray 140 | An ``xarray.DataArray`` object that contains time and frequency bins as 141 | well as corresponding values. If no noise date is available, 142 | None is returned. 143 | """ 144 | specgram = [] 145 | time = [] 146 | 147 | if any(self.data) is None: 148 | if verbose: 149 | print("Data object is empty. Spectrogram cannot be computed") 150 | self.spectrogram = None 151 | return None 152 | 153 | # sampling frequency 154 | fs = self.stats.sampling_rate 155 | 156 | # number of time steps 157 | if avg_time is None: 158 | nbins = int((len(self.data) - L) / ((1 - overlap) * L)) + 1 159 | else: 160 | nbins = int(np.ceil(len(self.data) / (avg_time * fs))) 161 | 162 | # sensitivity correction 163 | sense_corr = -self.frequency_calibration(int(L / 2 + 1)) 164 | 165 | # compute spectrogram. For avg_time=None 166 | # (periodogram for each time step), the last data samples are ignored 167 | # if len(noise[0].data) != k * L 168 | if avg_time is None: 169 | n_hop = int(L * (1 - overlap)) 170 | for n in range(nbins): 171 | f, Pxx = signal.periodogram( 172 | x=self.data[n * n_hop : n * n_hop + L], 173 | fs=fs, 174 | window=win, # noqa 175 | ) 176 | if len(Pxx) != int(L / 2) + 1: 177 | if verbose: 178 | print("Error while computing periodogram for segment", n) 179 | self.spectrogram = None 180 | return None 181 | else: 182 | Pxx = 10 * np.log10(Pxx * np.power(10, sense_corr / 10)) 183 | 184 | specgram.append(Pxx) 185 | time.append( 186 | self.stats.starttime.datetime + datetime.timedelta(seconds=n * L / fs / 2) 187 | ) 188 | 189 | else: 190 | for n in range(nbins - 1): 191 | f, Pxx = signal.welch( 192 | x=self.data[n * int(fs * avg_time) : (n + 1) * int(fs * avg_time)], # noqa 193 | fs=fs, 194 | window=win, 195 | nperseg=L, 196 | noverlap=int(L * overlap), 197 | nfft=L, 198 | average=average_type, 199 | ) 200 | 201 | if len(Pxx) != int(L / 2) + 1: 202 | if verbose: 203 | print( 204 | "Error while computing " "Welch estimate for segment", 205 | n, 206 | ) 207 | self.spectrogram = None 208 | return None 209 | else: 210 | Pxx = 10 * np.log10(Pxx * np.power(10, sense_corr / 10)) 211 | specgram.append(Pxx) 212 | time.append( 213 | self.stats.starttime.datetime + datetime.timedelta(seconds=n * avg_time) 214 | ) 215 | 216 | # compute PSD for residual segment 217 | # if segment has more than L samples 218 | if len(self.data[int((nbins - 1) * fs * avg_time) :]) >= L: # noqa 219 | f, Pxx = signal.welch( 220 | x=self.data[int((nbins - 1) * fs * avg_time) :], # noqa 221 | fs=fs, 222 | window=win, 223 | nperseg=L, 224 | noverlap=int(L * overlap), 225 | nfft=L, 226 | average=average_type, 227 | ) 228 | if len(Pxx) != int(L / 2) + 1: 229 | if verbose: 230 | print("Error while computing Welch " "estimate residual segment") 231 | self.spectrogram = None 232 | return None 233 | else: 234 | Pxx = 10 * np.log10(Pxx * np.power(10, sense_corr / 10)) 235 | specgram.append(Pxx) 236 | time.append( 237 | self.stats.starttime.datetime 238 | + datetime.timedelta(seconds=(nbins - 1) * avg_time) 239 | ) 240 | 241 | if len(time) == 0: 242 | if verbose: 243 | print("Spectrogram does not contain any data") 244 | self.spectrogram = None 245 | return None 246 | else: 247 | spec_xr = xr.DataArray( 248 | np.array(specgram), 249 | dims=["time", "frequency"], 250 | coords={"time": np.array(time), "frequency": np.array(f)}, 251 | attrs=dict( 252 | start_time=self.stats.starttime.datetime, 253 | end_time=self.stats.endtime.datetime, 254 | nperseg=L, 255 | units="dB rel µ Pa^2 / Hz", 256 | ), 257 | name="spectrogram", 258 | ) 259 | return spec_xr 260 | 261 | def compute_psd_welch( 262 | self, 263 | win="hann", 264 | L=4096, 265 | overlap=0.5, 266 | avg_method="median", 267 | interpolate=None, 268 | scale="log", 269 | verbose=True, 270 | ): 271 | """ 272 | Compute power spectral density estimates of noise data using 273 | Welch's method. 274 | 275 | Parameters 276 | ---------- 277 | win : str, optional 278 | Window function used to taper the data. See scipy.signal.get_window 279 | for a list of possible window functions (Default is Hann-window.) 280 | L : int, optional 281 | Length of each data block for computing the FFT (Default is 4096). 282 | overlap : float, optional 283 | Percentage of overlap between adjacent blocks if Welch's method is 284 | used. Parameter is ignored if avg_time is None. (Default is 50%) 285 | avg_method : str, optional 286 | Method for averaging the periodograms when using Welch's method. 287 | Either 'mean' or 'median' (default) can be used 288 | interpolate : float, optional 289 | Resolution in frequency domain in Hz. If None (default), the 290 | resolution will be sampling frequency fs divided by L. If 291 | interpolate is smaller than fs/L, the PSD will be interpolated 292 | using zero-padding 293 | scale : str, optional 294 | If 'log' (default) PSD in logarithmic scale (dB re 1µPa^2/H) is 295 | returned. If 'lin', PSD in linear scale 296 | (1µPa^2/H) is returned 297 | verbose : bool, optional 298 | If true (default), exception messages and some comments are 299 | printed. 300 | 301 | Returns 302 | ------- 303 | psd : xr.DataArray 304 | An ``xarray.DataArray`` object that contains frequency bins and PSD values. If no 305 | noise date is available, None is returned. 306 | """ 307 | # get noise data segment for each entry in rain_event 308 | # each noise data segment contains usually 1 min of data 309 | if any(self.data) is None: 310 | if verbose: 311 | print("Data object is empty. PSD cannot be computed") 312 | self.psd = None 313 | return None 314 | fs = self.stats.sampling_rate 315 | 316 | # compute nfft if zero padding is desired 317 | if interpolate is not None: 318 | if fs / L > interpolate: 319 | nfft = int(fs / interpolate) 320 | else: 321 | nfft = L 322 | else: 323 | nfft = L 324 | 325 | # compute Welch median for entire data segment 326 | f, Pxx = signal.welch( 327 | x=self.data, 328 | fs=fs, 329 | window=win, 330 | nperseg=L, 331 | noverlap=int(L * overlap), 332 | nfft=nfft, 333 | average=avg_method, 334 | ) 335 | 336 | if len(Pxx) != int(nfft / 2) + 1: 337 | if verbose: 338 | print("PSD cannot be computed.") 339 | self.psd = None 340 | return None 341 | 342 | sense_corr = -self.frequency_calibration(int(nfft / 2 + 1)) 343 | if scale == "log": 344 | Pxx = 10 * np.log10(Pxx * np.power(10, sense_corr / 10)) 345 | elif scale == "lin": 346 | Pxx = Pxx * np.power(10, sense_corr / 10) 347 | else: 348 | raise Exception('scale has to be either "lin" or "log".') 349 | 350 | psd_xr = xr.DataArray( 351 | np.array(Pxx), 352 | dims=["frequency"], 353 | coords={"frequency": np.array(f)}, 354 | attrs=dict( 355 | start_time=self.stats.starttime.datetime, 356 | end_time=self.stats.endtime.datetime, 357 | nperseg=L, 358 | units="dB rel µ Pa^2 / Hz", 359 | ), 360 | name="psd", 361 | ) 362 | return psd_xr 363 | 364 | def wav_write(self, filename, norm=False, new_sample_rate=None): 365 | """ 366 | method that stores HydrophoneData into .wav file 367 | 368 | Parameters 369 | ---------- 370 | filename : str 371 | filename to store .wav file as 372 | norm : bool 373 | specifies whether data should be normalized to 1 374 | new_sample_rate : float 375 | specifies new sample rate of wav file to be saved. (Resampling is 376 | done with scipy.signal.resample()). Default is None which keeps 377 | original sample rate of data. 378 | """ 379 | if norm: 380 | data = self.data / np.abs(np.max(self.data)) 381 | else: 382 | data = self.data 383 | 384 | if new_sample_rate is None: 385 | sampling_rate = self.stats.sampling_rate 386 | else: 387 | if new_sample_rate > self.stats.sampling_rate: 388 | upsamp_fac = new_sample_rate / self.stats.sampling_rate 389 | new_npts = self.stats.npts * upsamp_fac 390 | data = signal.resample(data, int(new_npts)) 391 | sampling_rate = new_sample_rate 392 | elif new_sample_rate == self.stats.sampling_rate: 393 | warnings.warn("New sample rate is same as original data. " "No resampling done.") 394 | sampling_rate = self.stats.sampling_rate 395 | elif new_sample_rate < self.stats.sampling_rate: 396 | warnings.warn( 397 | "New sample rate is lower than original sample" 398 | " rate. Chebychev 1 anti-aliasing filter used" 399 | ) 400 | if self.stats.sampling_rate % new_sample_rate != 0: 401 | raise Exception("New Sample Rate is not factor of original sample rate") 402 | else: 403 | data = signal.decimate(data, int(self.stats.sampling_rate / new_sample_rate)) 404 | sampling_rate = new_sample_rate 405 | 406 | wavfile.write(filename, int(sampling_rate), data) 407 | 408 | def get_asset_ID(self): 409 | """ 410 | get_asset_ID returns the hydrophone asset ID for a given data sample. 411 | This data can be found `here `_ for 413 | broadband hydrophones. Since Low frequency hydrophones remain 414 | constant with location and time, if the hydrophone is low frequency, 415 | {location}-{channel} string combination is returned 416 | """ 417 | # Low frequency hydrophone 418 | if round(self.stats.sampling_rate) == 200: 419 | asset_ID = f"{self.stats.location}-{self.stats.channel}" 420 | 421 | elif round(self.stats.sampling_rate) == 64000: 422 | url = ( 423 | "https://raw.githubusercontent.com/OOI-CabledArray/" 424 | "deployments/main/HYDBBA_deployments.csv" 425 | ) 426 | hyd_df = pd.read_csv(url) 427 | 428 | # LJ01D'Oregon Shelf Base Seafloor 429 | if self.stats.location == "LJ01D": 430 | ref = "CE02SHBP-LJ01D-11-HYDBBA106" 431 | # LJ01AOregon Slope Base Seafloor 432 | if self.stats.location == "LJ01A": 433 | ref = "RS01SLBS-LJ01A-09-HYDBBA102" 434 | # Oregan Slope Base Shallow 435 | if self.stats.location == "PC01A": 436 | ref = "RS01SBPS-PC01A-08-HYDBBA103" 437 | # Axial Base Shallow Profiler 438 | if self.stats.location == "PC03A": 439 | ref = "RS03AXPS-PC03A-08-HYDBBA303" 440 | # Oregon Offshore Base Seafloor 441 | if self.stats.location == "LJ01C": 442 | ref = "CE04OSBP-LJ01C-11-HYDBBA105" 443 | # Axial Base Seafloor 444 | if self.stats.location == "LJ03A": 445 | ref = "RS03AXBS-LJ03A-09-HYDBBA302" 446 | 447 | hyd_df["referenceDesignator"] 448 | 449 | df_ref = hyd_df.loc[hyd_df["referenceDesignator"] == ref] 450 | 451 | df_start = df_ref.loc[ 452 | (df_ref["startTime"] < self.stats.starttime) 453 | & (df_ref["endTime"] > self.stats.starttime) 454 | ] 455 | 456 | df_end = df_ref.loc[ 457 | (df_ref["startTime"] < self.stats.endtime) 458 | & (df_ref["endTime"] > self.stats.endtime) 459 | ] 460 | 461 | if df_start.index.to_numpy() == df_end.index.to_numpy(): 462 | idx = df_start.index.to_numpy() 463 | asset_ID = df_start["assetID"][int(idx)] 464 | elif (len(df_start) == 0) | (len(df_end) == 0): 465 | """^ covers case where currently deployed hydrophone is the 466 | one that is used in data segment. 467 | """ 468 | asset_ID = df_ref["assetID"][df_ref.index.to_numpy()[-1]] 469 | else: 470 | raise Exception( 471 | "Hydrophone Data involves multiple" "deployments. Feature to be added later" 472 | ) 473 | else: 474 | raise Exception("Invalid hydrophone sampling rate") 475 | 476 | return asset_ID 477 | 478 | def save(self, file_format, filename, wav_kwargs={}) -> None: 479 | """ 480 | save hydrophone data in specified method. Supported methods are: 481 | - pickle - saves the HydrophoneData object as a pickle file 482 | - netCDF - saves HydrophoneData object as netCDF. Time coordinates are not included 483 | - mat - saves HydrophoneData object as a .mat file 484 | - wav - calls wav_write method to save HydrophoneData object as a .wav file 485 | 486 | Parameters 487 | ---------- 488 | file_format : str 489 | format to save HydrophoneData object as. Supported formats are 490 | ['pkl', 'nc', 'mat', 'wav'] 491 | filepath : str 492 | filepath to save HydrophoneData object. file extension should not be included 493 | wav_kwargs : dict 494 | dictionary of keyword arguments to pass to wav_write method 495 | 496 | Returns 497 | ------- 498 | None 499 | """ 500 | 501 | try: 502 | self.data 503 | except AttributeError: 504 | raise AttributeError("HydrophoneData object does not contain any data") 505 | 506 | if file_format == "pkl": 507 | # save HydrophoneData object as pickle file 508 | 509 | print(filename + ".pkl") 510 | with open(filename + ".pkl", "wb") as f: 511 | pickle.dump(self, f) 512 | elif file_format == "nc": 513 | # save HydrophoneData object as netCDF file 514 | attrs = dict(self.stats) 515 | attrs["starttime"] = self.stats.starttime.strftime("%Y-%m-%dT%H:%M:%S.%f") 516 | attrs["endtime"] = self.stats.endtime.strftime("%Y-%m-%dT%H:%M:%S.%f") 517 | attrs["mseed"] = str(attrs["mseed"]) 518 | hdata_x = xr.DataArray(self.data, dims=["time"], attrs=attrs) 519 | hdata_x.to_netcdf(filename + ".nc") 520 | elif file_format == "mat": 521 | # save HydrophoneData object as .mat file 522 | data_dict = dict(self.stats) 523 | data_dict["data"] = self.data 524 | data_dict["starttime"] = self.stats.starttime.strftime("%Y-%m-%dT%H:%M:%S.%f") 525 | data_dict["endtime"] = self.stats.endtime.strftime("%Y-%m-%dT%H:%M:%S.%f") 526 | savemat(filename + ".mat", {self.stats.location: data_dict}) 527 | 528 | elif file_format == "wav": 529 | # save HydrophoneData object as .wav file 530 | self.wav_write(filename + ".wav", **wav_kwargs) 531 | else: 532 | raise Exception( 533 | "Invalid file format. Supported formats are: ['pkl', 'nc', 'mat', 'wav']" 534 | ) 535 | 536 | 537 | def node_id(node): 538 | """ 539 | mapping of name of hydrophone node to ID 540 | 541 | Parameter 542 | --------- 543 | node : str 544 | name or ID of the hydrophone node 545 | 546 | Returns 547 | ------- 548 | str 549 | ID of hydrophone node 550 | """ 551 | # broadband hydrophones 552 | if node == "Oregon_Shelf_Base_Seafloor" or node == "LJ01D": 553 | return "LJ01D" 554 | if node == "Oregon_Slope_Base_Seafloor" or node == "LJ01A": 555 | return "LJ01A" 556 | if node == "Oregon_Slope_Base_Shallow" or node == "PC01A": 557 | return "PC01A" 558 | if node == "Axial_Base_Shallow" or node == "PC03A": 559 | return "PC03A" 560 | if node == "Oregon_Offshore_Base_Seafloor" or node == "LJ01C": 561 | return "LJ01C" 562 | if node == "Axial_Base_Seafloor" or node == "LJ03A": 563 | return "LJ03A" 564 | 565 | # low frequency hydrophones 566 | if node == "Slope_Base" or node == "HYSB1": 567 | return "HYSB1" 568 | if node == "Southern_Hydrate" or node == "HYS14": 569 | return "HYS14" 570 | if node == "Axial_Base" or node == "AXBA1": 571 | return "AXBA1" 572 | if node == "Central_Caldera" or node == "AXCC1": 573 | return "AXCC1" 574 | if node == "Eastern_Caldera" or node == "AXEC2": 575 | return "AXEC2" 576 | 577 | # 200 Hz Seismometers 578 | if node == "AXAS1": 579 | return "AXAS1" 580 | if node == "AXAS2": 581 | return "AXAS2" 582 | if node == "AXEC1": 583 | return "AXEC1" 584 | if node == "AXEC3": 585 | return "AXEC3" 586 | if node == "AXID1": 587 | return "AXID1" 588 | if node == "HYS11": 589 | return "HYS11" 590 | if node == "HYS12": 591 | return "HYS12" 592 | if node == "HYS13": 593 | return "HYS13" 594 | 595 | else: 596 | print("No node exists for name or ID " + node) 597 | return "" 598 | 599 | 600 | def node_name(node): 601 | """ 602 | mapping of ID of hydrophone node to name 603 | 604 | Parameter 605 | --------- 606 | node : str 607 | ID or name of the hydrophone node 608 | 609 | Returns 610 | ------- 611 | str 612 | name of hydrophone node 613 | """ 614 | # broadband hydrophones 615 | if node == "Oregon_Shelf_Base_Seafloor" or node == "LJ01D": 616 | return "Oregon_Shelf_Base_Seafloor" 617 | if node == "Oregon_Slope_Base_Seafloor" or node == "LJ01A": 618 | return "Oregon_Slope_Base_Seafloor" 619 | if node == "Oregon_Slope_Base_Shallow" or node == "PC01A": 620 | return "Oregon_Slope_Base_Shallow" 621 | if node == "Axial_Base_Shallow" or node == "PC03A": 622 | return "Axial_Base_Shallow" 623 | if node == "Oregon_Offshore_Base_Seafloor" or node == "LJ01C": 624 | return "Oregon_Offshore_Base_Seafloor" 625 | if node == "Axial_Base_Seafloor" or node == "LJ03A": 626 | return "Axial_Base_Seafloor" 627 | 628 | # low frequency hydrophones 629 | if node == "Slope_Base" or node == "HYSB1": 630 | return "Slope_Base" 631 | if node == "Southern_Hydrate" or node == "HYS14": 632 | return "Southern_Hydrate" 633 | if node == "Axial_Base" or node == "AXBA1": 634 | return "Axial_Base" 635 | if node == "Central_Caldera" or node == "AXCC1": 636 | return "Central_Caldera" 637 | if node == "Eastern_Caldera" or node == "AXEC2": 638 | return "Eastern_Caldera" 639 | 640 | else: 641 | print("No node exists for ID or name " + node) 642 | return "" 643 | 644 | 645 | def _spectrogram_mp_helper(ooi_hyd_data_obj, win, L, avg_time, overlap, verbose, average_type): 646 | """ 647 | Helper function for compute_spectrogram_mp 648 | """ 649 | ooi_hyd_data_obj.compute_spectrogram(win, L, avg_time, overlap, verbose, average_type) 650 | return ooi_hyd_data_obj.spectrogram 651 | 652 | 653 | def _psd_mp_helper(ooi_hyd_data_obj, win, L, overlap, avg_method, interpolate, scale): 654 | """ 655 | Helper function for compute_psd_welch_mp 656 | """ 657 | ooi_hyd_data_obj.compute_psd_welch(win, L, overlap, avg_method, interpolate, scale) 658 | return ooi_hyd_data_obj.psd 659 | -------------------------------------------------------------------------------- /src/ooipy/hydrophone/calibration_by_assetID.csv: -------------------------------------------------------------------------------- 1 | ATOSU-58324-00012,ATOSU-58324-00012,ATOSU-58324-00012,ATOSU-58324-00015,ATOSU-58324-00015,ATOSU-58324-00015,ATOSU-58324-00014,ATOSU-58324-00014,ATOSU-58324-00014,ATAPL-58324-00005,ATAPL-58324-00005,ATAPL-58324-00005,ATAPL-58324-00008,ATAPL-58324-00008,ATAPL-58324-00008,ATAPL-58324-00007,ATAPL-58324-00007,ATAPL-58324-00007,ATAPL-58324-00006,ATAPL-58324-00006,ATAPL-58324-00006,ATAPL-58324-00010,ATAPL-58324-00010,ATAPL-58324-00010,ATAPL-58324-00003,ATAPL-58324-00003,ATAPL-58324-00003,ATAPL-58324-00011,ATAPL-58324-00011,ATAPL-58324-00011,ATOSU-58324-00016,ATOSU-58324-00016,ATOSU-58324-00016,ATAPL-58324-00009,ATAPL-58324-00009,ATAPL-58324-00009,ATAPL-58324-00004,ATAPL-58324-00004,ATAPL-58324-00004,ATOSU-58324-00013,ATOSU-58324-00013,ATOSU-58324-00013,HYSB1-HDH,HYSB1-HDH,HYSB1-HDH,HYS14-HDH,HYS14-HDH,HYS14-HDH,AXBA1-HDH,AXBA1-HDH,AXBA1-HDH,AXEC2-HDH,AXEC2-HDH,AXEC2-HDH,AXCC1-HDH,AXCC1-HDH,AXCC1-HDH,HYSB1-HNN,HYSB1-HNN,HYSB1-HNN,HYS14-HNN,HYS14-HNN,HYS14-HNN,AXBA1-HNN,AXBA1-HNN,AXBA1-HNN,AXEC2-HNN,AXEC2-HNN,AXEC2-HNN,AXCC1-HNN,AXCC1-HNN,AXCC1-HNN,HYSB1-HNE,HYSB1-HNE,HYSB1-HNE,HYS14-HNE,HYS14-HNE,HYS14-HNE,AXBA1-HNE,AXBA1-HNE,AXBA1-HNE,AXEC2-HNE,AXEC2-HNE,AXEC2-HNE,AXCC1-HNE,AXCC1-HNE,AXCC1-HNE,HYSB1-HNZ,HYSB1-HNZ,HYSB1-HNZ,HYS14-HNZ,HYS14-HNZ,HYS14-HNZ,AXBA1-HNZ,AXBA1-HNZ,AXBA1-HNZ,AXEC2-HNZ,AXEC2-HNZ,AXEC2-HNZ,AXCC1-HNZ,AXCC1-HNZ,AXCC1-HNZ 2 | Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase,Freq (kHz),0 phase,90 phase 3 | 0,-168.6,-168.6,0,-168.5,-168.5,0,-170.5,-170.5,0,-169.2,-169.2,0,-168.5,-168.5,0,-170.1,-170.1,0,-170.2,-170.2,0,-168.8,-168.8,0,-169.1,-169.1,0,-168.9,-168.9,0,-169,-169,0,-168.7,-168.7,0,-169.7,-169.7,0,-171.8,-171.8,0,2311.11,2311.11,0,2499.09,2499.09,0,2257.3,2257.3,0,2421,2421,0,2480.98,2480.98,0,315168,315168,0,316501,316501,0,314142,314142,0,315881,315881,0,316390,316390,0,314659,314659,0,315779,315779,0,315066,315066,0,314964,314964,0,316195,316195,0,315779,315779,0,315372,315372,0,315666,315666,0,315787,315787,0,314252,314252 4 | 26,-168.6,-168.6,26,-168.5,-168.5,26,-170.5,-170.5,26,-169.2,-169.2,26,-168.5,-168.5,26,-170.1,-170.1,26,-170.2,-170.2,26,-168.8,-168.8,26,-169.1,-169.1,26,-168.9,-168.9,26,-169,-169,26,-168.7,-168.7,26,-169.7,-169.7,10,-171.8,-171.8,100,2311.11,2311.11,100,2499.09,2499.09,100,2257.3,2257.3,100,2421,2421,100,2480.98,2480.98,100,315168,315168,100,316501,316501,100,314142,314142,100,315881,315881,100,316390,316390,100,314659,314659,100,315779,315779,100,315066,315066,100,314964,314964,100,316195,316195,100,315779,315779,100,315372,315372,100,315666,315666,100,315787,315787,100,314252,314252 5 | 10,-169.1,-169.3,10,-167.5,-167.7,10,-169.6,-169.9,13.5,-168.6,-168.6,10,-169.1,-168.9,10,-169.5,-169.7,10,-168.4,-168.4,10,-169.6,-170.2,10,10,-168.3,10,-169.8,-169.7,10,-169.7,-169.5,10,-169.7,-169.8,10,-168.9,-169.1,27,-172,-172,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 6 | 20.1,-170.9,-170.9,20.1,-169.3,-169.6,20.1,-172.9,-172.8,27.1,-168.4,-168.4,20.1,-170.5,-170.8,20.1,-171.7,-171.7,20.1,-168.1,-168.1,20.1,-171.8,-171.7,20.1,20.1,-166.5,20.1,-170.3,-170.8,20.1,-171.6,-171,20.1,-173.1,-173.4,20.1,-169.4,-169.5,40.5,-171.4,-171.4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 7 | 30.1,-171.1,-171.8,30.1,-171.7,-172,30.1,-174,-172.7,40.6,-169.8,-169.8,30.1,-170.8,-169.8,30.1,-172.2,-171.7,30.1,-171.1,-171.1,30.1,-171.5,-171.7,30.1,30.1,-167.9,30.1,-170.8,-171.1,30.1,-171.1,-171,30.1,-174.5,-172.7,30.1,-169.8,-170.1,52,-174,-174,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 8 | 40.2,-171.8,-171.3,40.2,-172.7,-172.6,40.2,-173.7,-172,54.1,-169.5,-169.5,40.2,-169.7,-169.7,40.2,-171.9,-171.4,40.2,-168.5,-168.5,40.2,-171.2,-171.2,40.2,40.2,-167.1,40.2,-170.3,-170.7,40.2,-170.9,-170.7,40.2,-173.1,-171.1,40.2,-169.7,-169.7,68,-173,-173,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 9 | 50.2,-174.5,-173.9,50.2,-172,-171.4,50.2,-173.1,-171.4,67.7,-170,-170,50.2,-173.5,-173.4,50.2,-171.7,-172.3,50.2,-169.6,-169.6,50.2,-172.8,-172.5,50.2,50.2,-167.4,50.2,-173,-174.2,50.2,-172.6,-174.1,50.2,-174.8,-174.2,50.2,-170.8,-170.3,80.5,-170.6,-170.6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 10 | 60.2,-173.3,-172.9,60.2,-172.2,-170.6,60.2,-173.4,-171.4,81.2,-171.7,-171.7,60.2,-172.1,-172.8,60.2,-171.6,-175.1,60.2,-171,-171,60.2,-171.6,-171.7,60.2,60.2,-167.6,60.2,-172.4,-173.3,60.2,-174,-176.2,60.2,-174.1,-171.9,60.2,-172.5,-171.2,94.5,-172.5,-172.5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 11 | 70.3,-174.4,-173.4,70.3,-171.4,-171,70.3,-173.2,-172.7,94.7,-172.1,-172.1,70.3,-173.4,-172.3,70.3,-171,-173.2,70.3,-169.9,-169.9,70.3,-173,-172.1,70.3,70.3,-167.4,70.3,-174,-175.2,70.3,-175.2,-176.4,70.3,-174.6,-171.7,70.3,-173,-170.9,108.5,-169.1,-169.1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 12 | 80.3,-175.9,-173.9,80.3,-172.4,-170.8,80.3,-175.9,-173.9,108.3,-172.9,-172.9,80.3,-174,-173.5,80.3,-172.4,-174.3,80.3,-171.1,-171.1,80.3,-173.8,-173.3,80.3,80.3,-169.1,80.3,-173.6,-176.3,80.3,-175.6,-177.1,80.3,-174.8,-171.6,80.3,-173.5,-171.6,121,-170.2,-170.2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 13 | 90.4,-175.5,-176.3,90.4,-172.1,-170.3,90.4,-175.9,-173,121.8,-172.3,-172.3,90.4,-174.7,-176.4,90.4,-172.7,-174.2,90.4,-169.9,-169.9,90.4,-175.7,-174.9,90.4,90.4,-169,90.4,-175.2,-177.4,90.4,-176.5,-178.8,90.4,-174.8,-172.5,90.4,-172.7,-171.4,135,-168.3,-168.3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 14 | 100.4,-174,-175.6,100.4,-172.2,-170.5,100.4,-175.7,-172.3,135.3,-169.6,-169.6,100.4,-175.8,-177.9,100.4,-172.1,-175.2,100.4,-172.3,-172.3,100.4,-175.1,-174.9,100.4,100.4,-172.9,100.4,-175.4,-176.6,100.4,-177.5,-178.3,100.4,-174,-172.6,100.4,-171.8,-171.7,149,-167.1,-167.1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 15 | 110.4,-175.4,-177.4,110.4,-172.7,-171.7,110.4,-175.1,-170.7,148.8,-166.2,-166.2,110.4,-176.9,-178.7,110.4,-171.7,-175.9,110.4,-172.1,-172.1,110.4,-175.3,-177.6,110.4,110.4,-172.2,110.4,-177.3,-177.7,110.4,-178.4,-179,110.4,-173.4,-174.8,110.4,-170.5,-170.9,162,-167,-167,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 16 | 120.5,-173.7,-174.2,120.5,-172.2,-173.8,120.5,-172.7,-169.7,162.4,-165.3,-165.3,120.5,-173.4,-178.4,120.5,-171.1,-175.7,120.5,-170.8,-170.8,120.5,-175.2,-176.4,120.5,120.5,-172.6,120.5,-176.7,-176.6,120.5,-179,-178.9,120.5,-170.5,-174.6,120.5,-171.4,-169.8,176,-163.5,-163.5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 17 | 130.5,-171.8,-171.8,130.5,-170.7,-172.2,130.5,-169.6,-167.7,175.9,-167.1,-167.1,130.5,-171.3,-175.7,130.5,-169.9,-173.5,130.5,-171.4,-171.4,130.5,-173.8,-174.3,130.5,130.5,-170.5,130.5,-175.2,-175,130.5,-178.9,-177.4,130.5,-169.3,-173.2,130.5,-170.9,-168.3,190,-170,-170,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 18 | 140.5,-171,-169.9,140.5,-170.2,-170.7,140.5,-168,-166.4,189.4,-170.9,-170.9,140.5,-169.4,-172.6,140.5,-169,-171.9,140.5,-169,-169,140.5,-171.9,-171,140.5,140.5,-168.7,140.5,-173.7,-172.5,140.5,-177.9,-174.1,140.5,-168.1,-171.3,140.5,-169.1,-166.9,200,-172,-172,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 19 | 150.6,-168.5,-168.3,150.6,-169.8,-170.2,150.6,-167.2,-165.6,200,-173.8,-173.8,150.6,-168,-170.7,150.6,-167.9,-170.4,150.6,-167.9,-167.9,150.6,-171.5,-169.8,150.6,150.6,-165,150.6,-171.3,-169.7,150.6,-174.1,-173.1,150.6,-168.6,-168.9,150.6,-167.6,-165.8,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 20 | 160.6,-165.2,-165.2,160.6,-167.8,-167.8,160.6,-165.7,-164.8,,,,160.6,-166.1,-166.5,160.6,-166.7,-167.7,160.6,-166.7,-166.7,160.6,-167,-166.7,160.6,160.6,-162,160.6,-168.5,-167.1,160.6,-170.8,-170.2,160.6,-167.5,-166.6,160.6,-165.2,-163.2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 21 | 170.7,-165.2,-163.6,170.7,-166.4,-167.3,170.7,-165.2,-164.9,,,,170.7,-165.6,-167.2,170.7,-167.2,-165.9,170.7,-166.5,-166.5,170.7,-165.7,-166.2,170.7,170.7,-164.4,170.7,-167.6,-166.3,170.7,-170.1,-168.6,170.7,-164.9,-164.4,170.7,-164.4,-163.6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 22 | 180.7,-167.7,-167.5,180.7,-169.6,-170,180.7,-167.8,-167.6,,,,180.7,-168.3,-170.3,180.7,-170,-169.1,180.7,-170,-170,180.7,-168.5,-169,180.7,180.7,-169.2,180.7,-170,-169.6,180.7,-175.8,-172.4,180.7,-167.6,-167.2,180.7,-166.9,-166.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 23 | 190.7,-170.4,-172.2,190.7,-173.5,-172.4,190.7,-171.6,-170.2,,,,190.7,-171.5,-175.1,190.7,-172.9,-173.7,190.7,-172.7,-172.7,190.7,-172.7,-172.2,190.7,190.7,-172.4,190.7,-173.5,-172.7,190.7,-178.8,-176.4,190.7,-171,-170.6,190.7,-169.7,-169.9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 24 | 200,-173.6,-175.2,200,-174.4,-173.3,200,-174.4,-171.8,,,,200,-174.9,-178.1,200,-175.3,-178.1,,,,200,-174.9,-174.1,,,,200,-176.4,-174.4,200,-180.5,-178.4,200,-174.2,-174.2,200,-171.6,-172.3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 25 | -------------------------------------------------------------------------------- /src/ooipy/request/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/src/ooipy/request/__init__.py -------------------------------------------------------------------------------- /src/ooipy/request/authentification.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides functions for the automatic authentification at the 3 | OOI. The authentification is based on a username and a token, which will 4 | be generated after registering an account at 5 | https://ooinet.oceanobservatories.org/. 6 | """ 7 | 8 | import os.path 9 | 10 | 11 | def set_authentification(username, token): 12 | """ 13 | Writes username and token to a text file. When requesting data from 14 | the OOI webservers, this textfile will automatically be accessed 15 | 16 | Parameters 17 | ---------- 18 | username : str 19 | Username automatically generated by the OOI. It typically starts 20 | with "OOIAPI-...". 21 | token : str 22 | Token automatically generated by the OOI. It typically starts 23 | with "TEMP-TOKEN-...". 24 | """ 25 | 26 | filename = "ooi_auth.txt" 27 | if not os.path.isfile(filename): 28 | file = open(filename, "w+") 29 | file.write("username\n" + username + "\n" + "token\n" + token) 30 | file.close() 31 | 32 | 33 | def get_authentification(): 34 | """ 35 | Open ooi_auth.txt file and return the username and the token 36 | 37 | Returns 38 | ------- 39 | (str, str) 40 | """ 41 | 42 | filename = "ooi_auth.txt" 43 | if os.path.isfile(filename): 44 | file = open(filename) 45 | auth = file.readlines() 46 | username = auth[1].split("\n")[0] 47 | token = auth[3].split("\n")[0] 48 | file.close() 49 | return username, token 50 | -------------------------------------------------------------------------------- /src/ooipy/request/ctd_request.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for downloading CTD Data. 3 | The first place you should look is the ooi data explorer, 4 | but if the data you need is not available, then these tools may be helpful. 5 | """ 6 | 7 | import concurrent.futures 8 | import datetime 9 | import multiprocessing as mp 10 | 11 | import requests 12 | 13 | import ooipy.request.authentification 14 | from ooipy.ctd.basic import CtdData 15 | 16 | 17 | def get_ctd_data( 18 | start_datetime, 19 | end_datetime, 20 | location, 21 | limit=10000, 22 | only_profilers=False, 23 | delivery_method="auto", 24 | ): 25 | """ 26 | Requests CTD data between start_detetime and end_datetime for the 27 | specified location if data is available. For each location, data of 28 | all available CTDs are requested and concatenated in a list. That is 29 | the final data list can consists of multiple segments of data, where 30 | each segment contains the data from one instrument. This means that 31 | the final list might not be ordered in time, but rather should be 32 | treated as an unordered list of CTD data points. 33 | 34 | Parameters 35 | ---------- 36 | start_datetime : datetime.datetime 37 | time of first sample from CTD 38 | end_datetime : datetime.datetime 39 | time of last sample from CTD 40 | location : str 41 | location for which data are requested. Possible choices are: 42 | * 'oregon_inshore' 43 | * 'oregon_shelf' 44 | * 'oregon_offshore' 45 | * 'oregon_slope' 46 | * 'washington_inshore' 47 | * 'washington_shelf' 48 | * 'washington_offshore' 49 | * 'axial_base' 50 | limit : int 51 | maximum number of data points returned in one request. The limit 52 | applies for each instrument separately. That is the final list 53 | of data points can contain more samples then indicated by limit 54 | if data from multiple CTDs is available at the given location 55 | and time. Default is 10,0000. 56 | only_profilers : bool 57 | Specifies whether only data from the water column profilers 58 | should be requested. Default is False 59 | delivery_method : str 60 | Specifies which delivery method is considered. For details 61 | please refer to http://oceanobservatories.org/glossary/. Options 62 | are: 63 | * 'auto' (default): automatically uses method that has data 64 | available 65 | * 'streamed': only considers data that are streamed to shore 66 | via cable 67 | * 'telemetered': only considers data that are streamed to shore 68 | via satellite 69 | * 'recovered': only considers data that were reteived when the 70 | instrument was retrieved 71 | 72 | Returns 73 | ------- 74 | ctd_data : :class:`ooipy.ctd.basic.CtdProfile` 75 | object, where the data array is stored in the raw_data attribute. Each 76 | data sample consists of a dictionary of parameters measured by the CTD. 77 | 78 | """ 79 | 80 | USERNAME, TOKEN = ooipy.request.authentification.get_authentification() 81 | # Sensor Inventory 82 | DATA_API_BASE_URL = "https://ooinet.oceanobservatories.org/api/m2m/12576/sensor/inv/" 83 | 84 | # Oregon Shelf 85 | if location == "oregon_shelf": 86 | url_list = [ 87 | "CE02SHSP/SP001/08-CTDPFJ000/telemetered/ctdpf_j_cspp_instrument?", 88 | "CE02SHSP/SP001/08-CTDPFJ000/recovered_cspp/ctdpf_j_cspp_instrument_recovered?", 89 | ] 90 | if not only_profilers: 91 | url_list.extend( 92 | [ 93 | "CE02SHBP/LJ01D/06-CTDBPN106/streamed/ctdbp_no_sample?", 94 | "CE02SHSM/RID27/03-CTDBPC000/telemetered/ctdbp_cdef_dcl_instrument?", 95 | ] 96 | ) 97 | 98 | elif location == "oregon_offshore": 99 | url_list = [ 100 | "CE04OSPS/SF01B/2A-CTDPFA107/streamed/ctdpf_sbe43_sample?", 101 | "CE04OSPD/DP01B/01-CTDPFL105/recovered_inst/dpc_ctd_instrument_recovered?", 102 | "CE04OSPD/DP01B/01-CTDPFL105/recovered_wfp/dpc_ctd_instrument_recovered?", 103 | ] 104 | if not only_profilers: 105 | url_list.extend( 106 | [ 107 | "CE04OSPS/PC01B/4A-CTDPFA109/streamed/ctdpf_optode_sample?", 108 | "CE04OSSM/RID27/03-CTDBPC000/telemetered/ctdbp_cdef_dcl_instrument?", 109 | "CE04OSBP/LJ01C/06-CTDBPO108/streamed/ctdbp_no_sample?", 110 | ] 111 | ) 112 | 113 | elif location == "oregon_slope": 114 | url_list = [ 115 | "RS01SBPD/DP01A/01-CTDPFL104/recovered_inst/dpc_ctd_instrument_recovered?", 116 | "RS01SBPD/DP01A/01-CTDPFL104/recovered_wfp/dpc_ctd_instrument_recovered?", 117 | "RS01SBPS/SF01A/2A-CTDPFA102/streamed/ctdpf_sbe43_sample?", 118 | ] 119 | if not only_profilers: 120 | url_list.extend( 121 | [ 122 | "RS01SBPS/PC01A/4A-CTDPFA103/streamed/ctdpf_optode_sample?", 123 | "RS01SLBS/LJ01A/12-CTDPFB101/streamed/ctdpf_optode_sample?", 124 | ] 125 | ) 126 | 127 | elif location == "oregon_inshore": 128 | url_list = [ 129 | "CE01ISSP/SP001/09-CTDPFJ000/recovered_cspp/ctdpf_j_cspp_instrument_recovered?", 130 | "CE01ISSP/SP001/09-CTDPFJ000/telemetered/ctdpf_j_cspp_instrument?", 131 | ] 132 | if not only_profilers: 133 | url_list.extend( 134 | [ 135 | "CE01ISSM/SBD17/06-CTDBPC000/telemetered/ctdbp_cdef_dcl_instrument?", 136 | "CE01ISSM/SBD17/06-CTDBPC000/recovered_inst/" 137 | + "ctdbp_cdef_instrument_recovered?", 138 | "CE01ISSM/RID16/03-CTDBPC000/recovered_inst/" 139 | + "ctdbp_cdef_instrument_recovered?", 140 | "CE01ISSM/RID16/03-CTDBPC000/telemetered/ctdbp_cdef_dcl_instrument?", 141 | "CE01ISSM/MFD37/03-CTDBPC000/recovered_inst/" 142 | + "ctdbp_cdef_instrument_recovered?", 143 | "CE01ISSM/MFD37/03-CTDBPC000/telemetered/ctdbp_cdef_dcl_instrument?", 144 | ] 145 | ) 146 | 147 | elif location == "washington_inshore": 148 | url_list = [ 149 | "CE06ISSP/SP001/09-CTDPFJ000/telemetered/ctdpf_j_cspp_instrument?", 150 | "CE06ISSP/SP001/09-CTDPFJ000/recovered_cspp/ctdpf_j_cspp_instrument_recovered?", 151 | ] 152 | if not only_profilers: 153 | url_list.extend( 154 | [ 155 | "CE06ISSM/SBD17/06-CTDBPC000/recovered_host/" 156 | + "ctdbp_cdef_dcl_instrument_recovered?", 157 | "CE06ISSM/SBD17/06-CTDBPC000/recovered_inst/" 158 | + "ctdbp_cdef_instrument_recovered?", 159 | "CE06ISSM/SBD17/06-CTDBPC000/telemetered/ctdbp_cdef_dcl_instrument?", 160 | "CE06ISSM/RID16/03-CTDBPC000/recovered_inst/" 161 | + "ctdbp_cdef_instrument_recovered?", 162 | "CE06ISSM/RID16/03-CTDBPC000/telemetered/ctdbp_cdef_dcl_instrument?", 163 | "CE06ISSM/MFD37/03-CTDBPC000/telemetered/ctdbp_cdef_dcl_instrument?", 164 | "CE06ISSM/MFD37/03-CTDBPC000/recovered_inst/" 165 | + "ctdbp_cdef_instrument_recovered?", 166 | ] 167 | ) 168 | 169 | elif location == "washington_shelf": 170 | url_list = ["CE07SHSP/SP001/08-CTDPFJ000/recovered_cspp/ctdpf_j_cspp_instrument_recovered?"] 171 | if not only_profilers: 172 | url_list.extend( 173 | [ 174 | "CE07SHSM/RID27/03-CTDBPC000/telemetered/ctdbp_cdef_dcl_instrument?", 175 | "CE07SHSM/RID27/03-CTDBPC000/recovered_inst/" 176 | + "ctdbp_cdef_instrument_recovered?", 177 | "CE07SHSM/MFD37/03-CTDBPC000/telemetered/ctdbp_cdef_dcl_instrument?", 178 | "CE07SHSM/MFD37/03-CTDBPC000/recovered_inst/" 179 | + "ctdbp_cdef_instrument_recovered?", 180 | ] 181 | ) 182 | 183 | elif location == "washington_offshore": 184 | url_list = [ 185 | "CE09OSPM/WFP01/03-CTDPFK000/telemetered/ctdpf_ckl_wfp_instrument?", 186 | "CE09OSPM/WFP01/03-CTDPFK000/recovered_wfp/ctdpf_ckl_wfp_instrument_recovered?", 187 | ] 188 | if not only_profilers: 189 | url_list.extend( 190 | [ 191 | "CE09OSSM/RID27/03-CTDBPC000/telemetered/ctdbp_cdef_dcl_instrument?", 192 | "CE09OSSM/RID27/03-CTDBPC000/recovered_inst/" 193 | + "ctdbp_cdef_instrument_recovered?", 194 | "CE09OSSM/MFD37/03-CTDBPE000/telemetered/ctdbp_cdef_dcl_instrument?", 195 | "CE09OSSM/MFD37/03-CTDBPE000/recovered_inst/" 196 | + "ctdbp_cdef_instrument_recovered?", 197 | ] 198 | ) 199 | 200 | elif location == "axial_base": 201 | url_list = [ 202 | "RS03AXPD/DP03A/01-CTDPFL304/recovered_inst/dpc_ctd_instrument_recovered?", 203 | "RS03AXPD/DP03A/01-CTDPFL304/recovered_wfp/dpc_ctd_instrument_recovered?", 204 | "RS03AXPS/SF03A/2A-CTDPFA302/streamed/ctdpf_sbe43_sample?", 205 | ] 206 | if not only_profilers: 207 | url_list.extend( 208 | [ 209 | "RS03AXPS/PC03A/4A-CTDPFA303/streamed/ctdpf_optode_sample?", 210 | "RS03AXBS/LJ03A/12-CTDPFB301/streamed/ctdpf_optode_sample?", 211 | ] 212 | ) 213 | 214 | else: 215 | raise Exception("Do not know given location.") 216 | 217 | # remove URLs if delivery method is specified 218 | url_list_method = [] 219 | if ( 220 | delivery_method == "telemetered" 221 | or delivery_method == "streamed" 222 | or delivery_method == "recovered" 223 | ): 224 | for url in url_list: 225 | if delivery_method in url: 226 | url_list_method.append(url) 227 | elif delivery_method == "auto": 228 | url_list_method = url_list 229 | else: 230 | raise Exception( 231 | "deliveriy method need to be either 'auto', " 232 | + "'telemetered', 'streamed', or 'recovered', but not " 233 | + str(delivery_method) 234 | ) 235 | 236 | beginDT = start_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" 237 | endDT = end_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" 238 | 239 | dataraw = [] 240 | processed_ctds = [] 241 | 242 | for url in url_list_method: 243 | ctd_id = __get_instrument_id(url) 244 | if ctd_id in processed_ctds: 245 | continue 246 | 247 | data_request_url = ( 248 | DATA_API_BASE_URL 249 | + url 250 | + "beginDT=" 251 | + beginDT 252 | + "&endDT=" 253 | + endDT 254 | + "&limit=" 255 | + str(limit) 256 | ) 257 | 258 | r = requests.get(data_request_url, auth=(USERNAME, TOKEN)) 259 | dataraw2 = r.json() 260 | if "message" not in dataraw2: 261 | dataraw.extend(dataraw2) 262 | processed_ctds.append(ctd_id) 263 | 264 | ctd_data = CtdData(raw_data=dataraw) 265 | return ctd_data 266 | 267 | 268 | def __get_instrument_id(url_str): 269 | ctd_id_arr = url_str.split("/")[:3] 270 | ctd_id = ctd_id_arr[0] + "/" + ctd_id_arr[1] + "/" + ctd_id_arr[2] 271 | 272 | return ctd_id 273 | 274 | 275 | def get_ctd_data_daily( 276 | datetime_day, 277 | location, 278 | limit=10000, 279 | only_profilers=False, 280 | delivery_method="auto", 281 | ): 282 | """ 283 | Requests CTD data for specified day and location. The day is split 284 | into 24 1-hour periods and for each 1-hour period 285 | :func:`ooipy.ctd_request.get_ctd_data is called. The data for all 286 | 1-hour periods are then concatednated in stored in a 287 | :class:`ooipy.ctd.basic.CtdProfile` object. 288 | 289 | Parameters 290 | ---------- 291 | datetime_day : datetime.datetime 292 | Day for which CTD data are requested 293 | location : str 294 | See :func:`ooipy.ctd_request.get_ctd_data 295 | limit : int 296 | See :func:`ooipy.ctd_request.get_ctd_data 297 | only_profilers : bool 298 | See :func:`ooipy.ctd_request.get_ctd_data 299 | delivery_method : str 300 | See :func:`ooipy.ctd_request.get_ctd_data 301 | 302 | Returns 303 | ------- 304 | :class:`ooipy.ctd.basic.CtdProfile` object, where the data array is 305 | stored in the raw_data attribute. Each data sample consists of a 306 | dictionary of parameters measured by the CTD. 307 | """ 308 | # get CTD data for one hour 309 | year = datetime_day.year 310 | month = datetime_day.month 311 | day = datetime_day.day 312 | 313 | start_end_list = [] 314 | 315 | for hour in range(24): 316 | start = datetime.datetime(year, month, day, hour, 0, 0) 317 | end = datetime.datetime(year, month, day, hour, 59, 59, 999) 318 | start_end_list.append((start, end)) 319 | 320 | raw_data_arr = __map_concurrency( 321 | __get_ctd_data_concurrent, 322 | start_end_list, 323 | { 324 | "location": location, 325 | "limit": limit, 326 | "only_profilers": only_profilers, 327 | "delivery_method": delivery_method, 328 | }, 329 | ) 330 | 331 | raw_data_falttened = [] 332 | for item in raw_data_arr: 333 | if item is None: 334 | continue 335 | else: 336 | raw_data_falttened.extend(item.raw_data) 337 | 338 | return CtdData(raw_data=raw_data_falttened) 339 | 340 | 341 | def __map_concurrency(func, iterator, args=(), max_workers=-1): 342 | """ 343 | Helper function to support multiprocessing for 344 | :func:`ooipy.ctd_request.get_ctd_data_daily 345 | """ 346 | # automatically set max_workers to 2x(available cores) 347 | if max_workers == -1: 348 | max_workers = 2 * mp.cpu_count() 349 | 350 | results = [] 351 | with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: 352 | # Start the load operations and mark each future with its URL 353 | future_to_url = {executor.submit(func, i, **args): i for i in iterator} 354 | for future in concurrent.futures.as_completed(future_to_url): 355 | data = future.result() 356 | results.append(data) 357 | return results 358 | 359 | 360 | def __get_ctd_data_concurrent(start_end_tuple, location, limit, only_profilers, delivery_method): 361 | """ 362 | Helper function to support multiprocessing for 363 | :func:`ooipy.ctd_request.get_ctd_data_daily 364 | """ 365 | start = start_end_tuple[0] 366 | end = start_end_tuple[1] 367 | 368 | rawdata = get_ctd_data( 369 | start, 370 | end, 371 | location=location, 372 | limit=limit, 373 | only_profilers=only_profilers, 374 | delivery_method=delivery_method, 375 | ) 376 | return rawdata 377 | -------------------------------------------------------------------------------- /src/ooipy/request/hydrophone_request.py: -------------------------------------------------------------------------------- 1 | """ 2 | This modules handles the downloading of OOI Data. As of current, the supported 3 | OOI sensors include all broadband hydrophones (Fs = 64 kHz), all low 4 | frequency hydrophones (Fs = 200 Hz), and bottom mounted OBSs. 5 | All supported hydrophone nodes are listed in the Hydrophone Nodes section below. 6 | """ 7 | 8 | import concurrent.futures 9 | import multiprocessing as mp 10 | import sys 11 | from datetime import datetime, timedelta 12 | from functools import partial 13 | 14 | import fsspec 15 | import numpy as np 16 | import obspy 17 | import requests 18 | from obspy import Stream, Trace, read 19 | from obspy.core import UTCDateTime 20 | from tqdm import tqdm 21 | 22 | # Import all dependencies 23 | from ooipy.hydrophone.basic import HydrophoneData 24 | 25 | 26 | def get_acoustic_data( 27 | starttime: datetime, 28 | endtime: datetime, 29 | node: str, 30 | fmin: float = None, 31 | fmax: float = None, 32 | max_workers: int = -1, 33 | append: bool = True, 34 | verbose: bool = False, 35 | mseed_file_limit: int = None, 36 | large_gap_limit: float = 1800.0, 37 | obspy_merge_method: int = 0, 38 | gapless_merge: bool = True, 39 | single_ms_buffer: bool = False, 40 | ): 41 | """ 42 | Get broadband acoustic data for specific time frame and sensor node. The 43 | data is returned as a :class:`.HydrophoneData` object. This object is 44 | based on the obspy data trace. 45 | 46 | >>> import ooipy 47 | >>> start_time = datetime.datetime(2017,3,10,0,0,0) 48 | >>> end_time = datetime.datetime(2017,3,10,0,5,0) 49 | >>> node = 'PC01A' 50 | >>> data = ooipy.request.get_acoustic_data(start_time, end_time, node) 51 | >>> # To access stats for retrieved data: 52 | >>> print(data.stats) 53 | >>> # To access numpy array of data: 54 | >>> print(data.data) 55 | 56 | Parameters 57 | ---------- 58 | start_time : datetime.datetime 59 | time of the first noise sample 60 | end_time : datetime.datetime 61 | time of the last noise sample 62 | node : str 63 | hydrophone name or identifier 64 | fmin : float, optional 65 | lower cutoff frequency of hydrophone's bandpass filter. Default 66 | is None which results in no filtering. 67 | fmax : float, optional 68 | higher cutoff frequency of hydrophones bandpass filter. Default 69 | is None which results in no filtering. 70 | print_exceptions : bool, optional 71 | whether or not exceptions are printed in the terminal line 72 | max_workers : int, optional 73 | number of maximum workers for concurrent processing. 74 | Default is -1 (uses number of available cores) 75 | append : bool, optional 76 | specifies if extra mseed files should be appended at beginning 77 | and end in case of boundary gaps in data. Default is True 78 | verbose : bool, optional 79 | specifies whether print statements should occur or not 80 | mseed_file_limit: int, optional 81 | If the number of mseed traces to be merged exceed this value, the 82 | function returns None. For some days the mseed files contain 83 | only a few seconds or milli seconds of data and merging a huge 84 | amount of files can dramatically slow down the program. if None 85 | (default), the number of mseed files will not be limited. This also 86 | limits the number of traces in a single file. 87 | large_gap_limit: float, optional 88 | Defines the length in second of large gaps in the data. 89 | Sometimes, large data gaps are present on particular days. This 90 | can cause long interpolation times if data_gap_mode 0 or 2 are 91 | used, possibly resulting in a memory overflow. If a data gap is 92 | longer than large_gap_limit, data are only retrieved before (if 93 | the gap stretches beyond the requested time) or after (if the gap 94 | starts prior to the requested time) the gap, or not at all (if 95 | the gap is within the requested time). 96 | obspy_merge_method : int, optional 97 | either [0,1], see [obspy documentation](https://docs.obspy.org/packages/autogen/ 98 | obspy.core.trace.Trace.html#handling-overlaps) 99 | for description of merge methods 100 | gapless_merge: bool, optional 101 | OOI BB hydrophones have had problems with data fragmentation, where 102 | individual files are only fractions of seconds long. Before June 2023, 103 | these were saved as separate mseed files. after 2023 (and in some cases, 104 | but not all retroactively), 5 minute mseed files contain many fragmented 105 | traces. These traces are essentially not possible to merge with 106 | obspy.merge. If True, then method to merge traces without 107 | consideration of gaps will be attempted. This will only be done if there 108 | is full data coverage over 5 min file length, but could still result in 109 | unalligned data. Default value is True. You should probably not use 110 | this method for data before June 2023 because it will likely cause an error. 111 | single_ms_buffer : bool 112 | If true, than 5 minute samples that have ± 1ms of data will also be allowed 113 | when using gapless merge. There is an issue in the broadband hydrophone 114 | data where there is occasionally ± 1 ms of data for a 5 minute segment 115 | (64 samples). This is likely due to the GPS clock errors that cause the 116 | data fragmentation in the first place. 117 | 118 | Returns 119 | ------- 120 | HydrophoneData 121 | 122 | """ 123 | # set number of workers 124 | if max_workers == -1: 125 | max_workers = mp.cpu_count() 126 | 127 | # data_gap = False 128 | sampling_rate = 64000.0 129 | 130 | if verbose: 131 | print("Fetching URLs...") 132 | 133 | # get URL for first day 134 | day_start = UTCDateTime(starttime.year, starttime.month, starttime.day, 0, 0, 0) 135 | data_url_list = __get_mseed_urls(starttime.strftime("/%Y/%m/%d/"), node, verbose) 136 | 137 | if data_url_list is None: 138 | if verbose: 139 | print( 140 | "No data available for specified day and node. " 141 | "Please change the day or use a different node" 142 | ) 143 | return None 144 | 145 | # increment day start by 1 day 146 | day_start = day_start + 24 * 3600 147 | 148 | # get all urls for each day until endtime is reached 149 | while day_start < endtime: 150 | urls_list_next_day = __get_mseed_urls(day_start.strftime("/%Y/%m/%d/"), node, verbose) 151 | if urls_list_next_day is None: 152 | day_start = day_start + 24 * 3600 153 | else: 154 | data_url_list.extend(urls_list_next_day) 155 | day_start = day_start + 24 * 3600 156 | 157 | if append: 158 | # Save last mseed of previous day to data_url_list if not None 159 | prev_day = starttime - timedelta(days=1) 160 | data_url_list_prev_day = __get_mseed_urls(prev_day.strftime("/%Y/%m/%d/"), node, verbose) 161 | if data_url_list_prev_day is not None: 162 | data_url_list = [data_url_list_prev_day[-1]] + data_url_list 163 | 164 | # get 1 more day of urls 165 | data_url_last_day_list = __get_mseed_urls(day_start.strftime("/%Y/%m/%d/"), node, verbose) 166 | if data_url_last_day_list is not None: 167 | data_url_list = data_url_list + [data_url_last_day_list[0]] 168 | 169 | if verbose: 170 | print("Sorting valid URLs for Time Window...") 171 | # Create list of urls for specified time range 172 | 173 | valid_data_url_list = [] 174 | first_file = True 175 | 176 | # Create List of mseed urls for valid time range 177 | for i in range(len(data_url_list)): 178 | # get UTC time of current and next item in URL list 179 | # extract start time from ith file 180 | utc_time_url_start = UTCDateTime(data_url_list[i].split("YDH")[1][1:].split(".mseed")[0]) 181 | 182 | # this line assumes no gaps between current and next file 183 | if i != len(data_url_list) - 1: 184 | utc_time_url_stop = UTCDateTime( 185 | data_url_list[i + 1].split("YDH")[1][1:].split(".mseed")[0] 186 | ) 187 | else: 188 | base_time = UTCDateTime(data_url_list[i].split("YDH")[1][1:].split(".mseed")[0]) 189 | utc_time_url_stop = UTCDateTime( 190 | year=base_time.year, 191 | month=base_time.month, 192 | day=base_time.day, 193 | hour=23, 194 | minute=59, 195 | second=59, 196 | microsecond=999999, 197 | ) 198 | 199 | # if current segment contains desired data, store data segment 200 | if ( 201 | (utc_time_url_start >= starttime and utc_time_url_start < endtime) 202 | or (utc_time_url_stop >= starttime and utc_time_url_stop < endtime) 203 | or (utc_time_url_start <= starttime and utc_time_url_stop >= endtime) 204 | ): 205 | if append: 206 | if i == 0: 207 | first_file = False 208 | valid_data_url_list.append(data_url_list[i]) 209 | 210 | elif first_file: 211 | first_file = False 212 | valid_data_url_list = [ 213 | data_url_list[i - 1], 214 | data_url_list[i], 215 | ] 216 | else: 217 | valid_data_url_list.append(data_url_list[i]) 218 | else: 219 | if i == 0: 220 | first_file = False 221 | valid_data_url_list.append(data_url_list[i]) 222 | 223 | # adds one more mseed file to st_ll 224 | else: 225 | # Checks if last file has been downloaded within time period 226 | if first_file is False: 227 | first_file = True 228 | if append: 229 | valid_data_url_list.append(data_url_list[i]) 230 | break 231 | 232 | # Check if number of mseed files exceed limit 233 | if isinstance(mseed_file_limit, int): 234 | if len(valid_data_url_list) > mseed_file_limit: 235 | if verbose: 236 | print("Number of mseed files to be merged exceed limit.") 237 | return None 238 | 239 | # handle large data gaps within one day 240 | if len(valid_data_url_list) >= 2: 241 | # find gaps 242 | gaps = [] 243 | for i in range(len(valid_data_url_list) - 1): 244 | utc_time_url_first = UTCDateTime( 245 | valid_data_url_list[i].split("YDH")[1][1:].split(".mseed")[0] 246 | ) 247 | utc_time_url_second = UTCDateTime( 248 | valid_data_url_list[i + 1].split("YDH")[1][1:].split(".mseed")[0] 249 | ) 250 | if utc_time_url_second - utc_time_url_first >= large_gap_limit: 251 | gaps.append(i) 252 | 253 | gap_cnt = 0 254 | # check if gap at beginning 255 | if 0 in gaps: 256 | del valid_data_url_list[0] 257 | gap_cnt += 1 258 | if verbose: 259 | print("Removed large data gap at beginning of requested time") 260 | # check if gap at the end 261 | if len(valid_data_url_list) - 2 in gaps: 262 | del valid_data_url_list[-1] 263 | gap_cnt += 1 264 | if verbose: 265 | print("Removed large data gap at end of requested time") 266 | # check if gap within requested time 267 | if len(gaps) > gap_cnt: 268 | if verbose: 269 | print("Found large data gap within requested time") 270 | return None 271 | 272 | if verbose: 273 | print("Downloading mseed files...") 274 | 275 | # removed max workers argument in following statement 276 | st_list = __map_concurrency( 277 | __read_mseed, valid_data_url_list, verbose=verbose, max_workers=max_workers 278 | ) 279 | 280 | st_list_new = [] 281 | # combine traces from single files into one trace if gapless merge is set to true 282 | # if a single 5 minute file is is not compatible with gapless merge, it is currently removed 283 | if gapless_merge: 284 | for k, st in enumerate(st_list): 285 | 286 | # count total number of points in stream 287 | npts_total = 0 288 | for tr in st: 289 | npts_total += tr.stats.npts 290 | 291 | # if valid npts, merge traces w/o consideration to gaps 292 | if single_ms_buffer: 293 | allowed_lengths = [300, 299.999, 300.001] 294 | else: 295 | allowed_lengths = [300] 296 | if npts_total / sampling_rate in allowed_lengths: 297 | # NOTE npts_total is nondeterminstically off by ± 64 samples. I have 298 | 299 | # if verbose: 300 | # print(f"gapless merge for {valid_data_url_list[k]}") 301 | 302 | data = [] 303 | for tr in st: 304 | data.append(tr.data) 305 | data_cat = np.concatenate(data) 306 | 307 | stats = dict(st[0].stats) 308 | stats["starttime"] = UTCDateTime(valid_data_url_list[k][-33:-6]) 309 | stats["endtime"] = UTCDateTime(stats["starttime"] + timedelta(minutes=5)) 310 | stats["npts"] = len(data_cat) 311 | st_list_new.append(Stream(traces=Trace(data_cat, header=stats))) 312 | else: 313 | # if verbose: 314 | # print( 315 | # f"Data segment {valid_data_url_list[k]}, \ 316 | # with npts {npts_total}, is not compatible with gapless merge" 317 | # ) 318 | 319 | # check if start times contain unique values 320 | start_times = [] 321 | for tr in st_list[k]: 322 | start_times.append(tr.stats.starttime.strftime("%Y-%m-%dT%H:%M:%S")) 323 | un_starttimes = set(start_times) 324 | if len(un_starttimes) == len(st_list[k]): 325 | if verbose: 326 | print("file fragmented but timestamps are unique. Segment kept") 327 | st_list_new.append(st_list[k]) 328 | else: 329 | if verbose: 330 | print("file fragmented and timestamps are corrupt. Segment thrown out") 331 | pass 332 | st_list = st_list_new 333 | 334 | # check if number of traces in st_list exceeds limit 335 | if mseed_file_limit is not None: 336 | for k, st in enumerate(st_list): 337 | if len(st) > mseed_file_limit: 338 | if verbose: 339 | print( 340 | f"Number of traces in mseed file, {valid_data_url_list[k]}\n\ 341 | exceed mseed_file_limit: {mseed_file_limit}." 342 | ) 343 | return None 344 | 345 | # combine list of single traces into stream of straces 346 | st_all = None 347 | for st in st_list: 348 | if st: 349 | if st[0].stats.sampling_rate != sampling_rate: 350 | if verbose: 351 | print("Some data have different sampling rate") 352 | else: 353 | if not isinstance(st_all, Stream): 354 | st_all = st 355 | else: 356 | st_all += st 357 | 358 | if st_all is None: 359 | if verbose: 360 | print("No data available for selected time") 361 | return None 362 | 363 | st_all = st_all.sort() 364 | 365 | # Merging Data - this section distributes obspy.merge to available cores 366 | if verbose: 367 | print(f"Merging {len(st_all)} Traces...") 368 | 369 | if len(st_all) < max_workers * 3: 370 | # don't use multiprocessing if there are less than 3 traces per worker 371 | st_all = st_all.merge(method=obspy_merge_method) 372 | else: 373 | # break data into num_worker segments 374 | num_segments = max_workers 375 | segment_size = len(st_all) // num_segments 376 | segments = [st_all[i : i + segment_size] for i in range(0, len(st_all), segment_size)] 377 | 378 | with mp.Pool(max_workers) as p: 379 | segments_merged = p.map( 380 | partial(__merge_singlecore, merge_method=obspy_merge_method), segments 381 | ) 382 | # final pass with just 4 cores 383 | if len(segments_merged) > 12: 384 | with mp.Pool(4) as p: 385 | segments_merged = p.map( 386 | partial(__merge_singlecore, merge_method=obspy_merge_method), segments_merged 387 | ) 388 | 389 | # merge merged segments 390 | for k, tr in enumerate(segments_merged): 391 | if k == 0: 392 | stream_merge = tr 393 | else: 394 | stream_merge += tr 395 | st_all = stream_merge.merge() 396 | 397 | # Slice data to desired window 398 | st_all = st_all.slice(UTCDateTime(starttime), UTCDateTime(endtime)) 399 | 400 | if len(st_all) == 0: 401 | if verbose: 402 | print("No data available for selected time frame.") 403 | return None 404 | 405 | if isinstance(st_all[0].data, np.ma.core.MaskedArray): 406 | # data_gap = True 407 | if verbose: # Note this will only trip if masked array is returned 408 | print("Data has Gaps") 409 | # interpolated is treated as if there is no gap 410 | 411 | # Filter Data 412 | try: 413 | if fmin is not None and fmax is not None: 414 | st_all = st_all.filter("bandpass", freqmin=fmin, freqmax=fmax) 415 | if verbose: 416 | print("Signal Filtered") 417 | # return st_all[0] 418 | return HydrophoneData(st_all[0].data, st_all[0].stats, node) 419 | except Exception: 420 | if st_all is None: 421 | if verbose: 422 | print("No data available for selected time frame.") 423 | else: 424 | if verbose: 425 | print(Exception) 426 | return None 427 | 428 | 429 | def get_acoustic_data_LF( 430 | starttime, 431 | endtime, 432 | node, 433 | fmin=None, 434 | fmax=None, 435 | verbose=False, 436 | zero_mean=False, 437 | channel="HDH", 438 | correct=False, 439 | ): 440 | """ 441 | Get low frequency acoustic data for specific time frame and sensor 442 | node. The data is returned as a :class:`.HydrophoneData` object. 443 | This object is based on the obspy data trace. Example usage is shown 444 | below. This function does not include the full functionality 445 | provided by the `IRIS data portal 446 | `_. 447 | 448 | If there is no data for the specified time window, then None is returned 449 | 450 | >>> starttime = datetime.datetime(2017,3,10,7,0,0) 451 | >>> endtime = datetime.datetime(2017,3,10,7,1,30) 452 | >>> location = 'Axial_Base' 453 | >>> fmin = None 454 | >>> fmax = None 455 | >>> # Returns ooipy.ooipy.hydrophone.base.HydrophoneData Object 456 | >>> data_trace = hydrophone_request.get_acoustic_data_LF( 457 | starttime, endtime, location, fmin, fmax, zero_mean=True) 458 | >>> # Access data stats 459 | >>> data_trace.stats 460 | >>> # Access numpy array containing data 461 | >>> data_trace.data 462 | 463 | Parameters 464 | ---------- 465 | start_time : datetime.datetime 466 | time of the first noise sample 467 | end_time : datetime.datetime 468 | time of the last noise sample 469 | node : str 470 | hydrophone 471 | fmin : float, optional 472 | lower cutoff frequency of hydrophone's bandpass filter. Default 473 | is None which results in no filtering. 474 | fmax : float, optional 475 | higher cutoff frequency of hydrophones bandpass filter. Default 476 | is None which results in no filtering. 477 | verbose : bool, optional 478 | specifies whether print statements should occur or not 479 | zero_mean : bool, optional 480 | specifies whether the mean should be removed. Default to False 481 | channel : str 482 | Channel of hydrophone to get data from. Currently supported options 483 | are 'HDH' - hydrophone, 'HNE' - east seismometer, 'HNN' - north 484 | seismometer, 'HNZ' - z seismometer. NOTE calibration is only valid for 485 | 'HDH' channel. All other channels are for raw data only at this time. 486 | correct : bool 487 | whether or not to use IRIS calibration code. NOTE: when this is true, 488 | computing PSDs is currently broken as calibration is computed twice 489 | 490 | Returns 491 | ------- 492 | hydrophone_data : :class:`.HydrophoneData` 493 | Hyrophone data object. If there is no data in the time window, None 494 | is returned 495 | """ 496 | 497 | if fmin is None and fmax is None: 498 | bandpass_range = None 499 | else: 500 | bandpass_range = [fmin, fmax] 501 | 502 | url = __build_LF_URL( 503 | node, 504 | starttime, 505 | endtime, 506 | bandpass_range=bandpass_range, 507 | zero_mean=zero_mean, 508 | channel=channel, 509 | correct=correct, 510 | ) 511 | if verbose: 512 | print("Downloading mseed file...") 513 | 514 | try: 515 | data_stream = read(url) 516 | except requests.HTTPError: 517 | if verbose: 518 | print(" error loading data from OOI server.") 519 | print(" likely that time window doesn't have data") 520 | return None 521 | 522 | # removing this (John 9/29/22) not sure if this will caused unknown errors... 523 | # Try downloading data 5 times. If fails every time raise exception 524 | # for k in range(5): 525 | # try: 526 | # data_stream = read(url) 527 | # break 528 | # except Exception: 529 | # if k == 4: 530 | # print(" Specific Time window timed out.") 531 | # return None 532 | 533 | # raise Exception ('Problem Requesting Data from OOI Server') 534 | 535 | hydrophone_data = HydrophoneData(data_stream[0].data, data_stream[0].stats, node) 536 | return hydrophone_data 537 | 538 | 539 | def ooipy_read( 540 | device, 541 | node, 542 | starttime, 543 | endtime, 544 | fmin=None, 545 | fmax=None, 546 | verbose=False, 547 | data_gap_mode=0, 548 | zero_mean=False, 549 | ): 550 | """ 551 | **this function is under development** 552 | 553 | General Purpose OOIpy read function. Parses input parameters to 554 | appropriate, device specific, read function. 555 | 556 | Parameters 557 | ---------- 558 | device : str 559 | Specifies device type. Valid option are 'broadband_hydrohpone' 560 | and 'low_frequency_hydrophone' 561 | node : str 562 | Specifies data acquisition device location. TODO add available 563 | options 564 | starttime : datetime.datetime 565 | Specifies start time of data requested 566 | endtime : datetime.datetime 567 | Specifies end time of data requested 568 | fmin : float 569 | Low frequency corner for filtering. If None are give, then no 570 | filtering happens. Broadband hydrophone data is filtered using 571 | Obspy. Low frequency hydrophone uses IRIS filtering. 572 | fmax : float 573 | High frequency corner for filtering. 574 | verbose : bool 575 | Specifies whether or not to print status update statements. 576 | data_gap_mode : int 577 | specifies how gaps in data are handled see documentation for 578 | get_acoustic_data 579 | 580 | Returns 581 | ------- 582 | hydrophone_data : HydrophoneData 583 | Object that stores hydrophone data. Similar to obspy trace. 584 | """ 585 | 586 | if device == "broadband_hydrophone": 587 | hydrophone_data = get_acoustic_data( 588 | starttime, 589 | endtime, 590 | node, 591 | fmin, 592 | fmax, 593 | verbose=verbose, 594 | data_gap_mode=data_gap_mode, 595 | ) 596 | elif device == "low_frequency_hydrophone": 597 | hydrophone_data = get_acoustic_data_LF( 598 | starttime, 599 | endtime, 600 | node, 601 | fmin=fmin, 602 | fmax=fmax, 603 | verbose=verbose, 604 | zero_mean=zero_mean, 605 | ) 606 | else: 607 | raise Exception("Invalid Device String") 608 | 609 | return hydrophone_data 610 | 611 | 612 | def __map_concurrency(func, iterator, args=(), max_workers=-1, verbose=False): 613 | # automatically set max_workers to 2x(available cores) 614 | if max_workers == -1: 615 | max_workers = 2 * mp.cpu_count() 616 | 617 | results = [None] * len(iterator) 618 | with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: 619 | # Start the load operations and mark each future with its index 620 | future_to_index = {executor.submit(func, i, *args): idx for idx, i in enumerate(iterator)} 621 | # Disable progress bar 622 | is_disabled = not verbose 623 | for future in tqdm( 624 | concurrent.futures.as_completed(future_to_index), 625 | total=len(iterator), 626 | disable=is_disabled, 627 | file=sys.stdout, 628 | ): 629 | idx = future_to_index[future] 630 | results[idx] = future.result() 631 | return results 632 | 633 | 634 | def __merge_singlecore(ls: list, merge_method: int = 0): 635 | """ 636 | merge a list of obspy traces into a single trace 637 | 638 | Parameters 639 | ---------- 640 | stream : list 641 | list of obspy traces 642 | merge_method : int 643 | see `obspy.Stream.merge() `__ passed to obspy.merge 645 | """ 646 | 647 | stream = obspy.Stream(ls) 648 | stream_merge = stream.merge(method=merge_method) 649 | stream_merge.id = ls[0].id 650 | return stream_merge 651 | 652 | 653 | def __read_mseed(url): 654 | # fname = os.path.basename(url) 655 | 656 | # removing try statement that abstracts errors 657 | # try: 658 | st = read(url, apply_calib=True) 659 | # except Exception: 660 | # print(f"Data Segment {url} Broken") 661 | # return None 662 | if isinstance(st, Stream): 663 | return st 664 | else: 665 | print(f"Problem Reading {url}") 666 | 667 | return None 668 | 669 | 670 | def __get_mseed_urls(day_str, node, verbose): 671 | """ 672 | get URLs for a specific day from OOI raw data server 673 | 674 | Parameters 675 | ---------- 676 | day_str : str 677 | date for which URLs are requested; format: yyyy/mm/dd, 678 | e.g. 2016/07/15 679 | node : str 680 | identifier or name of the hydrophone node 681 | verbose : bool 682 | print exceptions if True 683 | 684 | Returns 685 | ------- 686 | ([str], str) 687 | list of URLs, each URL refers to one data file. If no data is 688 | available for specified date, None is returned. 689 | """ 690 | 691 | try: 692 | if node == "LJ01D" or node == "Oregon_Shelf_Base_Seafloor": 693 | array = "/CE02SHBP" 694 | instrument = "/HYDBBA106" 695 | node_id = "/LJ01D" 696 | if node == "LJ01A" or node == "Oregon_Slope_Base_Seafloor": 697 | array = "/RS01SLBS" 698 | instrument = "/HYDBBA102" 699 | node_id = "/LJ01A" 700 | if node == "PC01A" or node == "Oregon_Slope_Base_Shallow": 701 | array = "/RS01SBPS" 702 | instrument = "/HYDBBA103" 703 | node_id = "/PC01A" 704 | if node == "PC03A" or node == "Axial_Base_Shallow": 705 | array = "/RS03AXPS" 706 | instrument = "/HYDBBA303" 707 | node_id = "/PC03A" 708 | if node == "LJ01C" or node == "Oregon_Offshore_Base_Seafloor": 709 | array = "/CE04OSBP" 710 | instrument = "/HYDBBA105" 711 | node_id = "/LJ01C" 712 | if node == "LJ03A" or node == "Axial_Base_Seafloor": 713 | array = "/RS03AXBS" 714 | instrument = "/HYDBBA302" 715 | node_id = "/LJ03A" 716 | 717 | mainurl = ( 718 | "https://rawdata.oceanobservatories.org/files" + array + node_id + instrument + day_str 719 | ) 720 | except Exception: 721 | raise Exception( 722 | "Invalid Location String " 723 | + node 724 | + ". Please use one " 725 | + "of the following node strings: " 726 | + "'Oregon_Shelf_Base_Seafloor' ('LJ01D'); ", 727 | "'Oregon_Slope_Base_Seafloor' ('LJ01A'); ", 728 | "'Oregon_Slope_Base_Shallow' ('PC01A'); ", 729 | "'Axial_Base_Shallow' ('PC03A'); ", 730 | "'Oregon_Offshore_Base_Seafloor' ('LJ01C'); ", 731 | "'Axial_Base_Seafloor' ('LJ03A')", 732 | ) 733 | 734 | FS = fsspec.filesystem("http") 735 | 736 | try: 737 | data_url_list = sorted( 738 | f["name"] 739 | for f in FS.ls(mainurl) 740 | if f["type"] == "file" and f["name"].endswith(".mseed") 741 | ) 742 | except Exception as e: 743 | if verbose: 744 | print("Client response: ", e) 745 | return None 746 | 747 | if not data_url_list: 748 | if verbose: 749 | print("No Data Available for Specified Time") 750 | return None 751 | 752 | return data_url_list 753 | 754 | 755 | def __build_LF_URL( 756 | node, 757 | starttime, 758 | endtime, 759 | bandpass_range=None, 760 | zero_mean=False, 761 | correct=False, 762 | channel=None, 763 | ): 764 | """ 765 | Build URL for Lowfrequency Data given the start time, end time, and 766 | node 767 | 768 | Parameters 769 | ---------- 770 | node : str 771 | node of low frequency hydrophone. Options include 772 | 'Easter_Caldera', ... 773 | starttime : datetime.datetime 774 | start of data segment requested 775 | endtime : datetime.datetime 776 | end of data segment requested 777 | bandpass_range : list 778 | list of length two specifying [flow, fhigh] in Hertz. If None 779 | are given, no bandpass will be added to data. 780 | zero_mean : bool 781 | specified whether mean should be removed from data 782 | correct : bool 783 | specifies whether to do sensitivity correction on hydrophone data 784 | channel : str 785 | channel string specifier ('HDH', 'HNE', 'HNN', 'HNZ') 786 | 787 | Returns 788 | ------- 789 | url : str 790 | url of specified data segment. Format will be in miniseed. 791 | """ 792 | 793 | network, station, location = __get_LF_locations_stats(node, channel) 794 | 795 | starttime = starttime.strftime("%Y-%m-%dT%H:%M:%S") 796 | endtime = endtime.strftime("%Y-%m-%dT%H:%M:%S") 797 | base_url = "http://service.iris.edu/irisws/timeseries/1/query?" 798 | netw_url = "net=" + network + "&" 799 | stat_url = "sta=" + station + "&" 800 | chan_url = "cha=" + channel + "&" 801 | strt_url = "start=" + starttime + "&" 802 | end_url = "end=" + endtime + "&" 803 | form_url = "format=miniseed&" 804 | loca_url = "loc=" + location 805 | if correct: 806 | corr_url = "&correct=true" 807 | else: 808 | corr_url = "" 809 | 810 | if bandpass_range is None: 811 | band_url = "" 812 | else: 813 | band_url = "bp=" + str(bandpass_range[0]) + "-" + str(bandpass_range[1]) + "&" 814 | if zero_mean: 815 | mean_url = "demean=true&" 816 | else: 817 | mean_url = "" 818 | url = ( 819 | base_url 820 | + netw_url 821 | + stat_url 822 | + chan_url 823 | + strt_url 824 | + end_url 825 | + mean_url 826 | + band_url 827 | + form_url 828 | + loca_url 829 | + corr_url 830 | ) 831 | return url 832 | 833 | 834 | def __get_LF_locations_stats(node, channel): 835 | network = "OO" 836 | location = "--" 837 | 838 | # only 200 Hz channels are supported 839 | if node == "Slope_Base" or node == "HYSB1": 840 | station = "HYSB1" 841 | if channel not in ["HDH", "HHN", "HHE", "HHZ", "HNN", "HNE", "HNZ"]: 842 | raise Exception( 843 | f"Invalid Channel String {channel} for node {node}.\n\ 844 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 845 | ) 846 | elif node == "Southern_Hydrate" or node == "HYS14": 847 | station = "HYS14" 848 | if channel not in ["HDH", "HHN", "HHE", "HHZ", "HNN", "HNE", "HNZ"]: 849 | raise Exception( 850 | f"Invalid Channel String {channel} for node {node}.\n\ 851 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 852 | ) 853 | elif node == "Axial_Base" or node == "AXBA1": 854 | station = "AXBA1" 855 | if channel not in ["HDH", "HHN", "HHE", "HHZ", "HNN", "HNE", "HNZ"]: 856 | raise Exception( 857 | f"Invalid Channel String {channel} for node {node}.\n\ 858 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 859 | ) 860 | elif node == "Central_Caldera" or node == "AXCC1": 861 | station = "AXCC1" 862 | if channel not in ["HDH", "HHN", "HHE", "HHZ", "HNN", "HNE", "HNZ"]: 863 | raise Exception( 864 | f"Invalid Channel String {channel} for node {node}.\n\ 865 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 866 | ) 867 | elif node == "Eastern_Caldera" or node == "AXEC2": 868 | station = "AXEC2" 869 | if channel not in ["HDH", "HHN", "HHE", "HHZ", "HNN", "HNE", "HNZ"]: 870 | raise Exception( 871 | f"Invalid Channel String {channel} for node {node}.\n\ 872 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 873 | ) 874 | elif node == "AXAS1": 875 | station = "AXAS1" 876 | if channel not in ["EHN", "EHE", "EHZ"]: 877 | raise Exception( 878 | f"Invalid Channel String {channel} for node {node}.\n\ 879 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 880 | ) 881 | elif node == "AXAS2": 882 | station = "AXAS2" 883 | if channel not in ["EHN", "EHE", "EHZ"]: 884 | raise Exception( 885 | f"Invalid Channel String {channel} for node {node}.\n\ 886 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 887 | ) 888 | elif node == "AXEC1": 889 | station = "AXEC1" 890 | if channel not in ["EHN", "EHE", "EHZ"]: 891 | raise Exception( 892 | f"Invalid Channel String {channel} for node {node}.\n\ 893 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 894 | ) 895 | elif node == "AXEC3": 896 | station = "AXEC3" 897 | if channel not in ["EHN", "EHE", "EHZ"]: 898 | raise Exception( 899 | f"Invalid Channel String {channel} for node {node}.\n\ 900 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 901 | ) 902 | elif node == "AXID1": 903 | station = "AXID1" 904 | if channel not in ["EHN", "EHE", "EHZ"]: 905 | raise Exception( 906 | f"Invalid Channel String {channel} for node {node}.\n\ 907 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 908 | ) 909 | elif node == "HYS11": 910 | station = "HYS11" 911 | if channel not in ["EHN", "EHE", "EHZ"]: 912 | raise Exception( 913 | f"Invalid Channel String {channel} for node {node}.\n\ 914 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 915 | ) 916 | elif node == "HYS12": 917 | station = "HYS12" 918 | if channel not in ["EHN", "EHE", "EHZ"]: 919 | raise Exception( 920 | f"Invalid Channel String {channel} for node {node}.\n\ 921 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 922 | ) 923 | elif node == "HYS13": 924 | station = "HYS13" 925 | if channel not in ["EHN", "EHE", "EHZ"]: 926 | raise Exception( 927 | f"Invalid Channel String {channel} for node {node}.\n\ 928 | see https://ds.iris.edu/mda/OO/ for available channels and nodes for OOI" 929 | ) 930 | 931 | else: 932 | raise Exception( 933 | f"Invalid Location String {node}. see https://ds.iris.edu/mda/OO/ for LF OOI nodes" 934 | ) 935 | 936 | return network, station, location 937 | -------------------------------------------------------------------------------- /src/ooipy/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/src/ooipy/scripts/__init__.py -------------------------------------------------------------------------------- /src/ooipy/scripts/download_hydrophone_data.py: -------------------------------------------------------------------------------- 1 | """ 2 | download_broadband.py 3 | John Ragland, June 2023 4 | 5 | download_broadband.py takes a csv file containing a list of 6 | sensors and time segments that you would like to download, 7 | and downloads them to your local machine. You can specify the file format 8 | that you want them to be saved. Supported file formats at this time 9 | include ['.mat', '.pkl', '.nc', '.wav']. 10 | 11 | example csv file: 12 | ----------------- 13 | ```csv 14 | node,start_time,end_time,file_format,downsample_factor 15 | LJ03A,2019-08-03T08:00:00,2019-08-03T08:01:00,wav,1 16 | AXBA1,2019-08-03T12:01:00,2019-08-03T12:02:00,wav,1 17 | ``` 18 | 19 | usage: 20 | ```console 21 | download_hydrophone_data --csv path/to/csv --output_path path/to/output 22 | ``` 23 | """ 24 | 25 | import argparse 26 | import sys 27 | 28 | import numpy as np 29 | import pandas as pd 30 | from tqdm import tqdm 31 | 32 | import ooipy 33 | 34 | 35 | def main(): 36 | hyd_type = { 37 | "LJ01D": "BB", 38 | "LJ01A": "BB", 39 | "PC01A": "BB", 40 | "PC03A": "BB", 41 | "LJ01C": "BB", 42 | "LJ03A": "BB", 43 | "AXBA1": "LF", 44 | "AXCC1": "LF", 45 | "AXEC2": "LF", 46 | "HYS14": "LF", 47 | "HYSB1": "LF", 48 | } 49 | 50 | # Create the argument parser 51 | parser = argparse.ArgumentParser() 52 | 53 | # Add command-line options 54 | parser.add_argument("--csv", help="file path to csv file") 55 | parser.add_argument("--output_path", help="file path to save files in") 56 | 57 | # Parse the command-line arguments 58 | args = parser.parse_args() 59 | 60 | # Check if the --path_to_csv option is present 61 | if args.csv is None: 62 | raise Exception("You must provide a path to the csv file, --csv ") 63 | if args.output_path is None: 64 | raise Exception( 65 | "You must provide a path to the output directory, --output_path " 66 | ) 67 | 68 | # Access the values of the command-line options 69 | df = pd.read_csv(args.csv) 70 | 71 | # estimate total download size and ask to proceed 72 | total_time = 0 73 | for k, item in df.iterrows(): 74 | total_time += (pd.Timestamp(item.end_time) - pd.Timestamp(item.start_time)).value / 1e9 75 | 76 | total_storage = total_time * 64e3 * 8 # 8 Bytes per sample 77 | 78 | def format_bytes(size): 79 | power = 2**10 # Power of 2^10 80 | n = 0 81 | units = ["B", "KB", "MB", "GB", "TB"] 82 | 83 | while size >= power and n < len(units) - 1: 84 | size /= power 85 | n += 1 86 | 87 | formatted_size = "{:.2f} {}".format(size, units[n]) 88 | return formatted_size 89 | 90 | print(f"total uncompressed download size: ~{format_bytes(total_storage)}") 91 | proceed = input("Do you want to proceed? (y/n): ") 92 | 93 | if proceed.lower() != "y": 94 | print("Exiting the script.") 95 | sys.exit(0) 96 | 97 | # download the data 98 | for k, item in tqdm(df.iterrows()): 99 | if item.node not in hyd_type.keys(): 100 | print(f"node {item.node} invalid, skipping") 101 | continue 102 | 103 | start_time_d = pd.Timestamp(item.start_time).to_pydatetime() 104 | end_time_d = pd.Timestamp(item.end_time).to_pydatetime() 105 | 106 | if hyd_type[item.node] == "LF": 107 | hdata = ooipy.get_acoustic_data_LF(start_time_d, end_time_d, item.node) 108 | else: 109 | hdata = ooipy.get_acoustic_data(start_time_d, end_time_d, item.node) 110 | 111 | if hdata is None: 112 | print(f"no data found for {item.node} between {start_time_d} and {end_time_d}") 113 | continue 114 | 115 | # fill masked values with mean 116 | hdata.data = np.ma.filled(hdata.data, np.mean(hdata.data.mean())) 117 | 118 | # downsample 119 | downsample_factor = item.downsample_factor 120 | if item.downsample_factor == 1: 121 | hdata_ds = hdata 122 | elif item.downsample_factor <= 16: 123 | hdata_ds = hdata.decimate(item.downsample_factor) 124 | else: 125 | hdata_ds = hdata 126 | while downsample_factor > 16: 127 | hdata_ds = hdata_ds.decimate(16) 128 | downsample_factor //= 16 129 | hdata_ds = hdata_ds.decimate(downsample_factor) 130 | 131 | # save 132 | op_path = args.output_path 133 | hdat_loc = hdata_ds.stats.location 134 | hdat_start_time = hdata_ds.stats.starttime.strftime("%Y%m%dT%H%M%S") 135 | hdat_end_time = hdata_ds.stats.endtime.strftime("%Y%m%dT%H%M%S") 136 | filename = f"{op_path}/{hdat_loc}_{hdat_start_time}_{hdat_end_time}" 137 | 138 | print(filename) 139 | hdata_ds.save( 140 | filename=filename, 141 | file_format=item.file_format, 142 | wav_kwargs={"norm": True}, 143 | ) 144 | -------------------------------------------------------------------------------- /src/ooipy/surface_buoy/__init__.py: -------------------------------------------------------------------------------- 1 | # TODO: 2 | -------------------------------------------------------------------------------- /src/ooipy/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/src/ooipy/tools/__init__.py -------------------------------------------------------------------------------- /src/ooipy/tools/ooiplotlib.py: -------------------------------------------------------------------------------- 1 | """ 2 | This modules provides functions for plotting spectrograms and power 3 | spectral density estimates. It extends the matplotlib.pyplot.plot 4 | function. 5 | """ 6 | 7 | # Import all dependencies 8 | import matplotlib 9 | import numpy as np 10 | from matplotlib import pyplot as plt 11 | 12 | 13 | def plot_ctd_profile(ctd_profile, **kwargs): 14 | """ 15 | Plot a :class:`ooipy.ctd.basic.CtdProfile` object using the 16 | matplotlib package. 17 | 18 | Parameters 19 | ---------- 20 | ctd_profile : :class:`ooipy.ctd.basic.CtdProfile` 21 | CtdProfile object to be plotted 22 | kwargs : 23 | See matplotlib documentation for list of arguments. Additional 24 | arguments are 25 | 26 | * plot : bool 27 | If False, figure will be closed. Can save time if only 28 | saving but not plotting is desired. Default is True 29 | * save : bool 30 | If True, figure will be saved under **filename**. Default is 31 | False 32 | * new_fig : bool 33 | If True, matplotlib will create a new figure. Default is 34 | True 35 | * filename : str 36 | filename of figure if saved. Default is "spectrogram.png" 37 | * figsize : (int, int) 38 | width and height of figure. Default is (16, 9) 39 | * title : str 40 | Title of plot. Default is 'CTD profile' 41 | * xlabel : str 42 | x-axis label of plot. Default is 'parameter' 43 | * ylabel : str 44 | y-axis label of plot. Default is 'depth' 45 | * show_variance : bool 46 | Indicates whether the variance should be plotted or not. 47 | Default is True 48 | * min_depth : int or float 49 | upper limit of vertical axis (depth axis). Default is the 50 | maximum of max(min(ctd_profile.depth_mean) - 10, 0) 51 | * max_depth : int or float 52 | lower limit of vertical axis (depth axis). Default is 53 | max(ctd_profile.depth_mean) + 10 54 | * dpi : int 55 | dots per inch, passed to matplotlib figure.savefig() 56 | * fontsize : int 57 | fontsize of saved plot, passed to matplotlib figure 58 | """ 59 | 60 | # check for keys 61 | if "plot" not in kwargs: 62 | kwargs["plot"] = True 63 | if "save" not in kwargs: 64 | kwargs["save"] = False 65 | if "new_fig" not in kwargs: 66 | kwargs["new_fig"] = True 67 | if "filename" not in kwargs: 68 | kwargs["filename"] = "ctd_profile.png" 69 | if "title" not in kwargs: 70 | kwargs["title"] = "CTD profile" 71 | if "xlabel" not in kwargs: 72 | kwargs["xlabel"] = "parameter" 73 | if "ylabel" not in kwargs: 74 | kwargs["ylabel"] = "depth" 75 | if "figsize" not in kwargs: 76 | kwargs["figsize"] = (16, 9) 77 | if "show_variance" not in kwargs: 78 | kwargs["show_variance"] = True 79 | if "linestyle" not in kwargs: 80 | kwargs["linestyle"] = "dashed" 81 | if "marker" not in kwargs: 82 | kwargs["marker"] = "o" 83 | if "markersize" not in kwargs: 84 | kwargs["markersize"] = 5 85 | if "color" not in kwargs: 86 | kwargs["color"] = "black" 87 | if "alpha" not in kwargs: 88 | kwargs["alpha"] = 0.5 89 | if "var_color" not in kwargs: 90 | kwargs["var_color"] = "gray" 91 | if "min_depth" not in kwargs: 92 | kwargs["min_depth"] = max(ctd_profile.depth_mean[0] - 10, 0) 93 | if "max_depth" not in kwargs: 94 | kwargs["max_depth"] = np.nanmax(ctd_profile.depth_mean) + 10 95 | if "dpi" not in kwargs: 96 | kwargs["dpi"] = 100 97 | if "fontsize" not in kwargs: 98 | kwargs["fontsize"] = 22 99 | 100 | # set backend for plotting/saving: 101 | if not kwargs["plot"]: 102 | matplotlib.use("Agg") 103 | font = {"size": kwargs["fontsize"]} 104 | matplotlib.rc("font", **font) 105 | 106 | # plot PSD object 107 | if kwargs["new_fig"]: 108 | fig, ax = plt.subplots(figsize=kwargs["figsize"]) 109 | plt.plot( 110 | ctd_profile.parameter_mean, 111 | ctd_profile.depth_mean, 112 | linestyle=kwargs["linestyle"], 113 | marker=kwargs["marker"], 114 | markersize=kwargs["markersize"], 115 | color=kwargs["color"], 116 | ) 117 | if kwargs["show_variance"]: 118 | y1 = ctd_profile.parameter_mean - 2 * np.sqrt(ctd_profile.parameter_var) 119 | y2 = ctd_profile.parameter_mean + 2 * np.sqrt(ctd_profile.parameter_var) 120 | plt.fill_betweenx( 121 | ctd_profile.depth_mean, 122 | y1, 123 | y2, 124 | alpha=kwargs["alpha"], 125 | color=kwargs["var_color"], 126 | ) 127 | plt.ylim([kwargs["max_depth"], kwargs["min_depth"]]) 128 | plt.ylabel(kwargs["ylabel"]) 129 | plt.xlabel(kwargs["xlabel"]) 130 | plt.title(kwargs["title"]) 131 | plt.grid(True) 132 | 133 | if kwargs["save"]: 134 | plt.savefig(kwargs["filename"], bbox_inches="tight", dpi=kwargs["dpi"]) 135 | 136 | if not kwargs["plot"]: 137 | plt.close(fig) 138 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ocean-Data-Lab/ooipy/ed9cb2822b20c249af19bdd3d0e74b3dc3bd6c74/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_request.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit tests for request module 3 | """ 4 | 5 | import datetime 6 | 7 | import numpy as np 8 | 9 | import ooipy.request.ctd_request as ctd_request # noqa 10 | import ooipy.request.hydrophone_request as hyd_request 11 | from ooipy.ctd.basic import CtdData # noqa 12 | from ooipy.hydrophone.basic import HydrophoneData 13 | 14 | 15 | def test_get_acoustic_data(): 16 | # 1. case: 100% data coverage 17 | start_time = datetime.datetime(2017, 3, 10, 0, 0, 0) 18 | end_time = datetime.datetime(2017, 3, 10, 0, 5, 0) 19 | node = "PC01A" 20 | 21 | data = hyd_request.get_acoustic_data(start_time, end_time, node, gapless_merge=False) 22 | 23 | assert isinstance(data, HydrophoneData) 24 | assert isinstance(data.data, (np.ndarray, np.ma.core.MaskedArray)) 25 | 26 | diff_start = abs((start_time - data.stats.starttime.datetime).microseconds) 27 | diff_end = abs((end_time - data.stats.endtime.datetime).microseconds) 28 | 29 | assert diff_start <= 100 30 | assert diff_end <= 100 31 | 32 | # 2. case: 0% data coverage 33 | start_time = datetime.datetime(2017, 10, 10, 15, 30, 0) 34 | end_time = datetime.datetime(2017, 10, 10, 15, 35, 0) 35 | node = "LJ01C" 36 | 37 | data = hyd_request.get_acoustic_data( 38 | start_time, end_time, node, append=False, gapless_merge=False 39 | ) 40 | 41 | assert data is None 42 | 43 | # 3. case: partial data coverage (data available until 15:17:50) 44 | start_time = datetime.datetime(2017, 10, 10, 15, 15, 0) 45 | end_time = datetime.datetime(2017, 10, 10, 15, 20, 0) 46 | node = "LJ01C" 47 | 48 | data = hyd_request.get_acoustic_data( 49 | start_time, end_time, node, append=False, gapless_merge=False 50 | ) 51 | 52 | assert isinstance(data, HydrophoneData) 53 | assert isinstance(data.data, (np.ndarray, np.ma.core.MaskedArray)) 54 | 55 | diff_start = abs((start_time - data.stats.starttime.datetime).microseconds) 56 | diff_end = abs((end_time - data.stats.endtime.datetime).microseconds) 57 | assert diff_start <= 100 58 | assert diff_end > 100 59 | 60 | # 4. case: 0% data coverage for entire day (directory does not exists) 61 | start_time = datetime.datetime(2019, 11, 1, 0, 0, 0) 62 | end_time = datetime.datetime(2019, 11, 1, 0, 5, 0) 63 | node = "LJ01D" 64 | 65 | data = hyd_request.get_acoustic_data( 66 | start_time, end_time, node, append=False, gapless_merge=False 67 | ) 68 | 69 | assert data is None 70 | 71 | 72 | def test_get_acoustic_data_LF(): 73 | start_time = datetime.datetime(2017, 3, 10, 0, 0, 0) 74 | end_time = datetime.datetime(2017, 3, 10, 0, 5, 0) 75 | node = "AXBA1" 76 | 77 | hdata = hyd_request.get_acoustic_data_LF(start_time, end_time, node) 78 | 79 | assert type(hdata) is HydrophoneData 80 | assert type(hdata.data) is np.ndarray 81 | 82 | 83 | def test_hydrophone_node_names(): 84 | node_arr = [ 85 | "Oregon_Shelf_Base_Seafloor", 86 | "Oregon_Slope_Base_Seafloor", 87 | "Oregon_Slope_Base_Shallow", 88 | "Axial_Base_Shallow", 89 | "Oregon_Offshore_Base_Seafloor", 90 | "Axial_Base_Seafloor", 91 | ] 92 | node_id_arr = ["LJ01D", "LJ01A", "PC01A", "PC03A", "LJ01C", "LJ03A"] 93 | starttime = datetime.datetime(2017, 3, 20, 0, 0, 0) # time of first sample 94 | endtime = datetime.datetime(2017, 3, 20, 0, 0, 1) # time of last sample 95 | for item in node_arr: 96 | hyd_data = hyd_request.get_acoustic_data(starttime, endtime, node=item, gapless_merge=False) 97 | assert hyd_data.stats.location in node_id_arr 98 | node_arr = [ 99 | "Slope_Base", 100 | "Southern_Hydrate", 101 | "Axial_Base", 102 | "Central_Caldera", 103 | "Eastern_Caldera", 104 | ] 105 | node_id_arr = ["HYSB1", "HYS14", "AXBA1", "AXCC1", "AXEC2"] 106 | for item in node_arr: 107 | hyd_data = hyd_request.get_acoustic_data_LF(starttime, endtime, node=item) 108 | assert hyd_data.stats.location in node_id_arr 109 | --------------------------------------------------------------------------------