├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── azarr ├── __init__.py ├── core.py ├── storage.py ├── storage_c.py ├── storage_fake.py ├── storage_js.py └── tests │ ├── __init__.py │ ├── conftest.py │ ├── test.zarr │ ├── .zgroup │ └── var │ │ ├── .zarray │ │ ├── 1.0 │ │ └── 1.1 │ └── test_core.py ├── example ├── index.html └── notebook.ipynb ├── setup.cfg └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | known_third_party = aiohttp,numpy,pyodide,pytest,requests,setuptools,zarr 3 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: > 2 | (?x)^( 3 | \.tox/.* 4 | )$ 5 | repos: 6 | 7 | - repo: https://github.com/pre-commit/pre-commit-hooks 8 | rev: v3.4.0 9 | hooks: 10 | - id: trailing-whitespace 11 | - id: end-of-file-fixer 12 | - id: check-docstring-first 13 | - id: check-json 14 | - id: check-yaml 15 | - repo: https://github.com/ambv/black 16 | rev: 22.3.0 17 | hooks: 18 | - id: black 19 | - repo: https://gitlab.com/pycqa/flake8 20 | rev: 3.8.4 21 | hooks: 22 | - id: flake8 23 | - repo: https://github.com/asottile/seed-isort-config 24 | rev: v2.2.0 25 | hooks: 26 | - id: seed-isort-config 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Martin Durant 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # async-zarr 2 | 3 | Hack wrapper around zarr to make data access async 4 | 5 | To run normal example: 6 | 7 | ```bash 8 | jupyter notebook ./example/notebook.ipynb 9 | ``` 10 | 11 | To run pyscript example: 12 | 13 | ```bash 14 | pip install -e . 15 | python ./azarr/tests/conftest.py 16 | open http://localhost:8000/example/index.html # browse to this location 17 | ``` 18 | 19 | And copy the cells from the notebook into the REPL cells. 20 | -------------------------------------------------------------------------------- /azarr/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import open_group, open, open_consolidated 2 | 3 | __version__ = "0.0.1" 4 | __all__ = ["open_group", "open", "open_consolidated"] 5 | -------------------------------------------------------------------------------- /azarr/core.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import zarr 3 | from zarr.core import check_array_shape, check_fields, ensure_ndarray_like 4 | from zarr.hierarchy import ( 5 | ContainsArrayError, 6 | GroupNotFoundError, 7 | normalize_storage_path, 8 | ) 9 | from zarr.convenience import PathNotFoundError 10 | from zarr.storage import ConsolidatedMetadataStore, contains_array, contains_group 11 | 12 | 13 | from .storage import SyncStore, ASyncStore 14 | 15 | 16 | class AGroup(zarr.Group): 17 | def __getitem__(self, item): 18 | path = self._item_path(item) 19 | if contains_array(self._store, path): 20 | return AArray( 21 | self._store, 22 | read_only=self._read_only, 23 | path=path, 24 | chunk_store=self._chunk_store, 25 | synchronizer=self._synchronizer, 26 | cache_attrs=self.attrs.cache, 27 | ) 28 | elif contains_group(self._store, path): 29 | return AGroup( 30 | self._store, 31 | read_only=self._read_only, 32 | path=path, 33 | chunk_store=self._chunk_store, 34 | cache_attrs=self.attrs.cache, 35 | synchronizer=self._synchronizer, 36 | ) 37 | else: 38 | raise KeyError(item) 39 | 40 | 41 | class AArray(zarr.Array): 42 | async def _get_selection(self, indexer, out=None, fields=None): 43 | 44 | # We iterate over all chunks which overlap the selection and thus contain data 45 | # that needs to be extracted. Each chunk is processed in turn, extracting the 46 | # necessary data and storing into the correct location in the output array. 47 | 48 | # N.B., it is an important optimisation that we only visit chunks which overlap 49 | # the selection. This minimises the number of iterations in the main for loop. 50 | 51 | # check fields are sensible 52 | out_dtype = check_fields(fields, self._dtype) 53 | 54 | # determine output shape 55 | out_shape = indexer.shape 56 | 57 | # setup output array 58 | if out is None: 59 | out = np.empty(out_shape, dtype=out_dtype, order=self._order) 60 | else: 61 | check_array_shape("out", out, out_shape) 62 | 63 | lchunk_coords, lchunk_selection, lout_selection = zip(*indexer) 64 | await self._chunk_getitems( 65 | lchunk_coords, 66 | lchunk_selection, 67 | out, 68 | lout_selection, 69 | drop_axes=indexer.drop_axes, 70 | fields=fields, 71 | ) 72 | 73 | if out.shape: 74 | return out 75 | else: 76 | return out[()] 77 | 78 | async def _chunk_getitems( 79 | self, 80 | lchunk_coords, 81 | lchunk_selection, 82 | out, 83 | lout_selection, 84 | drop_axes=None, 85 | fields=None, 86 | ): 87 | """As _chunk_getitem, but for lists of chunks 88 | 89 | This gets called where the storage supports ``getitems``, so that 90 | it can decide how to fetch the keys, allowing concurrency. 91 | """ 92 | out_is_ndarray = True 93 | try: 94 | out = ensure_ndarray_like(out) 95 | except TypeError: # pragma: no cover 96 | out_is_ndarray = False 97 | 98 | ckeys = [self._chunk_key(ch) for ch in lchunk_coords] 99 | partial_read_decode = False 100 | cdatas = await self.chunk_store.getitems(ckeys, on_error="omit") 101 | for ckey, chunk_select, out_select in zip( 102 | ckeys, lchunk_selection, lout_selection 103 | ): 104 | if ckey in cdatas: 105 | self._process_chunk( 106 | out, 107 | cdatas[ckey], 108 | chunk_select, 109 | drop_axes, 110 | out_is_ndarray, 111 | fields, 112 | out_select, 113 | partial_read_decode=partial_read_decode, 114 | ) 115 | else: 116 | # check exception type 117 | if self._fill_value is not None: 118 | if fields: 119 | fill_value = self._fill_value[fields] 120 | else: 121 | fill_value = self._fill_value 122 | out[out_select] = fill_value 123 | 124 | 125 | def open_group( 126 | store, 127 | mode="r", 128 | cache_attrs=True, 129 | synchronizer=None, 130 | path=None, 131 | chunk_store=None, 132 | **_, 133 | ): 134 | """Open a group using file-mode-like semantics. 135 | 136 | Parameters 137 | ---------- 138 | store : MutableMapping or string, optional 139 | Store or path to directory in file system or name of zip file. 140 | mode : {'r', 'r+', 'a', 'w', 'w-'}, optional 141 | Persistence mode: 'r' means read only (must exist); 'r+' means 142 | read/write (must exist); 'a' means read/write (create if doesn't 143 | exist); 'w' means create (overwrite if exists); 'w-' means create 144 | (fail if exists). 145 | cache_attrs : bool, optional 146 | If True (default), user attributes will be cached for attribute read 147 | operations. If False, user attributes are reloaded from the store prior 148 | to all attribute read operations. 149 | synchronizer : object, optional 150 | Array synchronizer. 151 | path : string, optional 152 | Group path within store. 153 | chunk_store : MutableMapping or string, optional 154 | Store or path to directory in file system or name of zip file. 155 | storage_options : dict 156 | If using an fsspec URL to create the store, these will be passed to 157 | the backend implementation. Ignored otherwise. 158 | 159 | Returns 160 | ------- 161 | g : zarr.hierarchy.Group 162 | 163 | Examples 164 | -------- 165 | >>> import zarr 166 | >>> root = zarr.open_group('data/example.zarr', mode='w') 167 | >>> foo = root.create_group('foo') 168 | >>> bar = root.create_group('bar') 169 | >>> root 170 | 171 | >>> root2 = zarr.open_group('data/example.zarr', mode='a') 172 | >>> root2 173 | 174 | >>> root == root2 175 | True 176 | 177 | """ 178 | # handle polymorphic store arg 179 | if isinstance(store, str): 180 | store = SyncStore(store) 181 | if chunk_store is None: 182 | chunk_store = ASyncStore(store.prefix) 183 | 184 | path = normalize_storage_path(path) 185 | 186 | if mode == "r": 187 | if not contains_group(store, path=path): 188 | if contains_array(store, path=path): 189 | raise ContainsArrayError(path) 190 | raise GroupNotFoundError(path) 191 | 192 | else: 193 | # although could have http PUT, leaving this example read-only for now 194 | raise NotImplementedError 195 | read_only = mode == "r" 196 | 197 | return AGroup( 198 | store, 199 | read_only=read_only, 200 | cache_attrs=cache_attrs, 201 | synchronizer=synchronizer, 202 | path=path, 203 | chunk_store=chunk_store, 204 | ) 205 | 206 | 207 | def open(store=None, mode="r", *, path=None, chunk_store=None, **kwargs): 208 | """Convenience function to open a group or array using file-mode-like semantics. 209 | 210 | Parameters 211 | ---------- 212 | store : Store or string, optional 213 | Store or path to directory in file system or name of zip file. 214 | mode : 'r', optional 215 | Persistence mode: 'r' means read only (must exist); 'r+' means 216 | read/write (must exist); 'a' means read/write (create if doesn't 217 | exist); 'w' means create (overwrite if exists); 'w-' means create 218 | (fail if exists). 219 | path : str or None, optional 220 | The path within the store to open. 221 | **kwargs 222 | Additional parameters are passed through to :func:`zarr.creation.open_array` or 223 | :func:`zarr.hierarchy.open_group`. 224 | 225 | Returns 226 | ------- 227 | z : :class:`zarr.core.Array` or :class:`zarr.hierarchy.Group` 228 | Array or group, depending on what exists in the given store. 229 | 230 | See Also 231 | -------- 232 | zarr.creation.open_array, zarr.hierarchy.open_group 233 | 234 | """ 235 | 236 | if isinstance(store, str): 237 | store = SyncStore(store) 238 | if chunk_store is None: 239 | chunk_store = ASyncStore(store.prefix) 240 | 241 | path = normalize_storage_path(path) 242 | kwargs["path"] = path 243 | 244 | if contains_array(store, path): 245 | return AArray(store, read_only=True, chunk_store=chunk_store, **kwargs) 246 | elif contains_group(store, path): 247 | return open_group(store, mode=mode, chunk_store=chunk_store, **kwargs) 248 | else: 249 | raise PathNotFoundError(path) 250 | 251 | 252 | def open_consolidated(store, metadata_key=".zmetadata", mode="r", **kwargs): 253 | """Open group using metadata previously consolidated into a single key. 254 | 255 | This is an optimised method for opening a Zarr group, where instead of 256 | traversing the group/array hierarchy by accessing the metadata keys at 257 | each level, a single key contains all of the metadata for everything. 258 | For remote data sources where the overhead of accessing a key is large 259 | compared to the time to read data. 260 | 261 | The group accessed must have already had its metadata consolidated into a 262 | single key using the function :func:`consolidate_metadata`. 263 | 264 | This optimised method only works in modes which do not change the 265 | metadata, although the data may still be written/updated. 266 | 267 | Parameters 268 | ---------- 269 | store : MutableMapping or string 270 | Store or path to directory in file system or name of zip file. 271 | metadata_key : str 272 | Key to read the consolidated metadata from. The default (.zmetadata) 273 | corresponds to the default used by :func:`consolidate_metadata`. 274 | mode : {'r', 'r+'}, optional 275 | Persistence mode: 'r' means read only (must exist); 'r+' means 276 | read/write (must exist) although only writes to data are allowed, 277 | changes to metadata including creation of new arrays or group 278 | are not allowed. 279 | **kwargs 280 | Additional parameters are passed through to :func:`zarr.creation.open_array` or 281 | :func:`zarr.hierarchy.open_group`. 282 | 283 | Returns 284 | ------- 285 | g : :class:`zarr.hierarchy.Group` 286 | Group instance, opened with the consolidated metadata. 287 | 288 | See Also 289 | -------- 290 | consolidate_metadata 291 | 292 | """ 293 | 294 | # normalize parameters 295 | if isinstance(store, str): 296 | store = SyncStore(store) 297 | chunk_store = kwargs.pop("chunk_store", None) 298 | if chunk_store is None: 299 | chunk_store = ASyncStore(store.prefix) 300 | 301 | path = kwargs.pop("path", None) 302 | 303 | # setup metadata store 304 | meta_store = ConsolidatedMetadataStore(store, metadata_key=metadata_key) 305 | return open( 306 | store=meta_store, chunk_store=chunk_store, mode=mode, path=path, **kwargs 307 | ) 308 | -------------------------------------------------------------------------------- /azarr/storage.py: -------------------------------------------------------------------------------- 1 | try: 2 | from .storage_js import SyncStore, ASyncStore 3 | except ImportError: 4 | from .storage_c import SyncStore, ASyncStore 5 | 6 | 7 | __all__ = ["SyncStore", "ASyncStore"] 8 | -------------------------------------------------------------------------------- /azarr/storage_c.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import weakref 3 | 4 | import aiohttp 5 | import requests 6 | 7 | try: 8 | from zarr.storage import BaseStore 9 | except ImportError: 10 | BaseStore = object 11 | 12 | 13 | class SyncStore(BaseStore): 14 | def __init__(self, prefix): 15 | self.prefix = prefix 16 | self.session = requests.Session() 17 | 18 | def __getitem__(self, key): 19 | url = "/".join([self.prefix, key]) 20 | try: 21 | r = self.session.get(url) 22 | if r.ok: 23 | out = r.content 24 | return out 25 | except Exception: 26 | pass 27 | raise KeyError 28 | 29 | __delitem__ = __iter__ = __len__ = __setitem__ = None 30 | 31 | 32 | class ASyncStore(BaseStore): 33 | def __init__(self, prefix): 34 | self.prefix = prefix 35 | self.session = None 36 | 37 | async def get_session(self): 38 | if self.session is None: 39 | self.session = aiohttp.ClientSession() 40 | weakref.finalize(self, self.close, self.session) 41 | return self.session 42 | 43 | async def getitems(self, keys, **_): 44 | urls = ["/".join([self.prefix, k]) for k in keys] 45 | session = await self.get_session() 46 | data = await asyncio.gather(*[get(session, url) for url in urls]) 47 | out = {k: o for k, o in zip(keys, data) if o} 48 | return out 49 | 50 | __delitem__ = __getitem__ = __iter__ = __len__ = __setitem__ = None 51 | 52 | @staticmethod 53 | def close(session): 54 | connector = getattr(session, "_connector", None) 55 | if connector is not None: 56 | # close after loop is dead 57 | connector._close() 58 | 59 | 60 | async def get(session, url): 61 | try: 62 | r = await session.get(url) 63 | if r.ok: 64 | return await r.read() 65 | except Exception as e: 66 | print(e) 67 | return None 68 | finally: 69 | if "r" in locals(): 70 | await r.__aexit__(None, None, None) 71 | -------------------------------------------------------------------------------- /azarr/storage_fake.py: -------------------------------------------------------------------------------- 1 | from zarr.storage import BaseStore 2 | 3 | 4 | class FakeGetter(BaseStore): 5 | def __init__(self, *args, **kwargs): 6 | self.needed_keys = set() 7 | super().__init__(*args, **kwargs) 8 | 9 | async def getitems(self, items, **kwargs): 10 | self.needed_keys.update(items) 11 | return {} 12 | 13 | __delitem__ = __getitem__ = __iter__ = __len__ = __setitem__ = None 14 | -------------------------------------------------------------------------------- /azarr/storage_js.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import pyodide.http 4 | 5 | try: 6 | from zarr.storage import BaseStore 7 | except ImportError: 8 | BaseStore = object 9 | 10 | 11 | class SyncStore(BaseStore): 12 | def __init__(self, prefix): 13 | self.prefix = prefix 14 | 15 | def __getitem__(self, key): 16 | url = "/".join([self.prefix, key]) 17 | try: 18 | out = pyodide.http.open_url(url).read() 19 | if out and "nosuchkey" not in out.lower() and "error" not in out.lower(): 20 | return out 21 | except Exception as e: 22 | print(e) 23 | raise KeyError 24 | 25 | __delitem__ = __iter__ = __len__ = __setitem__ = None 26 | 27 | 28 | class ASyncStore(BaseStore): 29 | def __init__(self, prefix): 30 | self.prefix = prefix 31 | 32 | async def getitems(self, keys, **_): 33 | urls = ["/".join([self.prefix, k]) for k in keys] 34 | data = await asyncio.gather(*[get(url) for url in urls]) 35 | return {k: o for k, o in zip(keys, data) if o} 36 | 37 | __delitem__ = __getitem__ = __iter__ = __len__ = __setitem__ = None 38 | 39 | 40 | async def get(url): 41 | try: 42 | r = await pyodide.http.pyfetch(url) 43 | if r.ok: 44 | return await r.bytes() 45 | except Exception as e: 46 | print(e) 47 | return None 48 | -------------------------------------------------------------------------------- /azarr/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/martindurant/async-zarr/b8bba4237fc3bcb022a05505cd6111095fc7bfa4/azarr/tests/__init__.py -------------------------------------------------------------------------------- /azarr/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import http.server 3 | import subprocess 4 | 5 | import pytest 6 | import requests 7 | import time 8 | 9 | here = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) 10 | 11 | 12 | class CORSRequestHandler(http.server.SimpleHTTPRequestHandler): 13 | def __init__(self, *args, **kwargs): 14 | kwargs.pop("directory", None) 15 | super().__init__(*args, directory=here, **kwargs) 16 | 17 | def send_head(self): 18 | path = self.translate_path(self.path) 19 | if os.path.isfile(path): 20 | return super().send_head() 21 | self.send_response(404) 22 | self.end_headers() 23 | 24 | def end_headers(self): 25 | self.send_header("Access-Control-Allow-Origin", "*") 26 | super().end_headers() 27 | 28 | 29 | @pytest.fixture(scope="session") 30 | def server(): 31 | P = subprocess.Popen(["python", __file__]) 32 | timeout = 5 33 | url = "http://localhost:8000/azarr/tests/test.zarr" 34 | while True: 35 | try: 36 | assert requests.get(url + "/.zgroup").ok 37 | yield url 38 | break 39 | except Exception: 40 | time.sleep(0.1) 41 | timeout -= 0.1 42 | assert timeout > 0 43 | P.terminate() 44 | 45 | 46 | if __name__ == "__main__": 47 | http.server.test(CORSRequestHandler) 48 | -------------------------------------------------------------------------------- /azarr/tests/test.zarr/.zgroup: -------------------------------------------------------------------------------- 1 | { 2 | "zarr_format": 2 3 | } 4 | -------------------------------------------------------------------------------- /azarr/tests/test.zarr/var/.zarray: -------------------------------------------------------------------------------- 1 | { 2 | "chunks": [ 3 | 5, 4 | 5 5 | ], 6 | "compressor": null, 7 | "dtype": " 2 | 3 | 4 | 5 | 6 | 7 | Async Zarr Playground 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 | 18 | - zarr 19 | - matplotlib 20 | 21 | 22 | import asyncio 23 | 24 | import micropip 25 | await micropip.install("http://localhost:8000/dist/azarr-0.0.1-py3-none-any.whl") 26 | import zarr 27 | print(f"zarr version = {zarr.__version__}") 28 | import azarr 29 | azarr.__version__ 30 | 31 |
32 | 33 |
34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /example/notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "c255444e", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import azarr" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "id": "877f84f6", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "# see https://pangeo-forge.org/dashboard/feedstock/88\n", 21 | "url = 'https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/EOBS-feedstock/eobs-surface-downwelling.zarr'" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 3, 27 | "id": "8d8648f1", 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "z = azarr.open_consolidated(url)" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 4, 37 | "id": "14a398cf", 38 | "metadata": {}, 39 | "outputs": [ 40 | { 41 | "data": { 42 | "text/plain": [ 43 | "['latitude', 'longitude', 'qq', 'time']" 44 | ] 45 | }, 46 | "execution_count": 4, 47 | "metadata": {}, 48 | "output_type": "execute_result" 49 | } 50 | ], 51 | "source": [ 52 | "list(z)" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 5, 58 | "id": "26eb51c1", 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "data": { 63 | "text/plain": [ 64 | "" 65 | ] 66 | }, 67 | "execution_count": 5, 68 | "metadata": {}, 69 | "output_type": "execute_result" 70 | } 71 | ], 72 | "source": [ 73 | "z.latitude" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 6, 79 | "id": "579c853f", 80 | "metadata": {}, 81 | "outputs": [ 82 | { 83 | "data": { 84 | "text/plain": [ 85 | "" 86 | ] 87 | }, 88 | "execution_count": 6, 89 | "metadata": {}, 90 | "output_type": "execute_result" 91 | } 92 | ], 93 | "source": [ 94 | "z.latitude[:]" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 7, 100 | "id": "8b444563", 101 | "metadata": {}, 102 | "outputs": [ 103 | { 104 | "data": { 105 | "text/plain": [ 106 | "0.3319680690765381" 107 | ] 108 | }, 109 | "execution_count": 7, 110 | "metadata": {}, 111 | "output_type": "execute_result" 112 | } 113 | ], 114 | "source": [ 115 | "# get two three arrays with a single context switch to async\n", 116 | "# and only one latency wait\n", 117 | "import asyncio\n", 118 | "import time\n", 119 | "\n", 120 | "t0 = time.time()\n", 121 | "lat, lot, t = await asyncio.gather(\n", 122 | " z.latitude[:], \n", 123 | " z.longitude[:],\n", 124 | " z.time[:]\n", 125 | ")\n", 126 | "t1 = time.time()\n", 127 | "t1 - t0" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": 8, 133 | "id": "b23cc442", 134 | "metadata": {}, 135 | "outputs": [ 136 | { 137 | "data": { 138 | "text/plain": [ 139 | "array([25.04986065, 25.14986065, 25.24986065, 25.34986065, 25.44986065,\n", 140 | " 25.54986065, 25.64986065, 25.74986065, 25.84986065, 25.94986065,\n", 141 | " 26.04986065, 26.14986065, 26.24986065, 26.34986065, 26.44986065,\n", 142 | " 26.54986064, 26.64986064, 26.74986064, 26.84986064, 26.94986064,\n", 143 | " 27.04986064, 27.14986064, 27.24986064, 27.34986064, 27.44986064,\n", 144 | " 27.54986064, 27.64986064, 27.74986064, 27.84986064, 27.94986064,\n", 145 | " 28.04986064, 28.14986064, 28.24986064, 28.34986064, 28.44986064,\n", 146 | " 28.54986064, 28.64986064, 28.74986064, 28.84986064, 28.94986064,\n", 147 | " 29.04986063, 29.14986063, 29.24986063, 29.34986063, 29.44986063,\n", 148 | " 29.54986063, 29.64986063, 29.74986063, 29.84986063, 29.94986063,\n", 149 | " 30.04986063, 30.14986063, 30.24986063, 30.34986063, 30.44986063,\n", 150 | " 30.54986063, 30.64986063, 30.74986063, 30.84986063, 30.94986063,\n", 151 | " 31.04986063, 31.14986063, 31.24986063, 31.34986063, 31.44986063,\n", 152 | " 31.54986062, 31.64986062, 31.74986062, 31.84986062, 31.94986062,\n", 153 | " 32.04986062, 32.14986062, 32.24986062, 32.34986062, 32.44986062,\n", 154 | " 32.54986062, 32.64986062, 32.74986062, 32.84986062, 32.94986062,\n", 155 | " 33.04986062, 33.14986062, 33.24986062, 33.34986062, 33.44986062,\n", 156 | " 33.54986062, 33.64986062, 33.74986062, 33.84986062, 33.94986062,\n", 157 | " 34.04986061, 34.14986061, 34.24986061, 34.34986061, 34.44986061,\n", 158 | " 34.54986061, 34.64986061, 34.74986061, 34.84986061, 34.94986061])" 159 | ] 160 | }, 161 | "execution_count": 8, 162 | "metadata": {}, 163 | "output_type": "execute_result" 164 | } 165 | ], 166 | "source": [ 167 | "lat[:100]" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": 9, 173 | "id": "391dd82b", 174 | "metadata": {}, 175 | "outputs": [ 176 | { 177 | "data": { 178 | "text/plain": [ 179 | "((25933, 460, 700), (40, 460, 700))" 180 | ] 181 | }, 182 | "execution_count": 9, 183 | "metadata": {}, 184 | "output_type": "execute_result" 185 | } 186 | ], 187 | "source": [ 188 | "z.qq.shape, z.qq.chunks" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 10, 194 | "id": "8bf24946", 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "# or opposite ends of the same array\n", 199 | "slice1, slice2 = await asyncio.gather(\n", 200 | " z.qq[:2, 200:250, 300:350],\n", 201 | " z.qq[-1, 200:250, 300:350]\n", 202 | ")" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": 11, 208 | "id": "cb6fccd2", 209 | "metadata": {}, 210 | "outputs": [ 211 | { 212 | "data": { 213 | "text/plain": [ 214 | "array([[78., 77., 75., ..., 49., 46., 46.],\n", 215 | " [76., 75., 74., ..., 53., 47., 46.],\n", 216 | " [73., 73., 72., ..., 56., 48., 47.],\n", 217 | " ...,\n", 218 | " [59., 59., 58., ..., 24., 27., 30.],\n", 219 | " [52., 52., 51., ..., 23., 26., 28.],\n", 220 | " [45., 45., 44., ..., 22., 25., 27.]], dtype=float32)" 221 | ] 222 | }, 223 | "execution_count": 11, 224 | "metadata": {}, 225 | "output_type": "execute_result" 226 | } 227 | ], 228 | "source": [ 229 | "slice2" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": null, 235 | "id": "04912aa9", 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [] 239 | } 240 | ], 241 | "metadata": { 242 | "kernelspec": { 243 | "display_name": "Python 3 (ipykernel)", 244 | "language": "python", 245 | "name": "python3" 246 | }, 247 | "language_info": { 248 | "codemirror_mode": { 249 | "name": "ipython", 250 | "version": 3 251 | }, 252 | "file_extension": ".py", 253 | "mimetype": "text/x-python", 254 | "name": "python", 255 | "nbconvert_exporter": "python", 256 | "pygments_lexer": "ipython3", 257 | "version": "3.9.13" 258 | } 259 | }, 260 | "nbformat": 4, 261 | "nbformat_minor": 5 262 | } 263 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | long_description: file: README.rst 3 | 4 | [versioneer] 5 | VCS = git 6 | style = pep440 7 | versionfile_source = fsspec/_version.py 8 | versionfile_build = fsspec/_version.py 9 | tag_prefix = "" 10 | 11 | [flake8] 12 | exclude = .tox,build,docs/source/conf.py,versioneer.py,fsspec/_version 13 | max-line-length = 88 14 | ignore = 15 | # Assigning lambda expression 16 | E731 17 | # Ambiguous variable names 18 | E741 19 | # line break before binary operator 20 | W503 21 | # whitespace before : 22 | E203 23 | # redefs 24 | F811 25 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | 6 | setup( 7 | name="azarr", 8 | version="0.0.1", 9 | classifiers=[ 10 | "Programming Language :: Python :: 3.9", 11 | "Programming Language :: Python :: 3.10", 12 | ], 13 | description="", 14 | long_description="", 15 | long_description_content_type="text/markdown", 16 | url="", 17 | project_urls={}, 18 | maintainer="Martin Durant", 19 | maintainer_email="mdurant@anaconda.com", 20 | license="BSD", 21 | keywords="file", 22 | packages=["azarr"], 23 | python_requires=">=3.8", 24 | install_requires=["zarr"], 25 | zip_safe=False, 26 | ) 27 | --------------------------------------------------------------------------------