├── .mise.toml ├── object-store ├── python │ └── object_store │ │ ├── py.typed │ │ ├── arrow.py │ │ ├── __init__.py │ │ └── _internal.pyi ├── README.md ├── Cargo.toml ├── src │ └── lib.rs ├── pyproject.toml └── tests │ ├── test_object_store.py │ └── test_arrow.py ├── poetry.toml ├── Cargo.toml ├── docs ├── reference.md └── index.md ├── .vscode ├── settings.json └── cspell.json ├── justfile ├── object-store-internal ├── Cargo.toml └── src │ ├── utils.rs │ ├── builder.rs │ ├── file.rs │ └── lib.rs ├── mkdocs.yaml ├── .editorconfig ├── .pre-commit-config.yaml ├── pyproject.toml ├── .github └── workflows │ ├── release.yaml │ └── ci.yaml ├── examples └── object_store.ipynb ├── README.md ├── LICENSE └── Cargo.lock /.mise.toml: -------------------------------------------------------------------------------- 1 | [tools] 2 | python = "3.11" 3 | -------------------------------------------------------------------------------- /object-store/python/object_store/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = ["object-store", "object-store-internal"] 4 | -------------------------------------------------------------------------------- /docs/reference.md: -------------------------------------------------------------------------------- 1 | ::: object_store.ObjectStore 2 | 3 | ::: object_store.arrow.ArrowFileSystemHandler 4 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true, 3 | "python.formatting.blackArgs": ["--line-length=120"] 4 | } 5 | -------------------------------------------------------------------------------- /object-store/README.md: -------------------------------------------------------------------------------- 1 | # object-store 2 | 3 | A common abstraction for high performance access to various storage backends. 4 | -------------------------------------------------------------------------------- /object-store/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "object-store-python" 3 | version = "0.2.0" 4 | edition = "2021" 5 | authors = ["Robert Pack "] 6 | description = "A generic object store interface for uniformly interacting with AWS S3, Google Cloud Storage, Azure Storage and local files." 7 | 8 | [dependencies] 9 | object-store-internal = { path = "../object-store-internal" } 10 | pyo3 = { version = "0.21", features = [ 11 | "extension-module", 12 | "abi3", 13 | "abi3-py38", 14 | ] } 15 | 16 | [lib] 17 | name = "object_store" 18 | crate-type = ["cdylib"] 19 | 20 | [package.metadata.maturin] 21 | name = "object_store._internal" 22 | -------------------------------------------------------------------------------- /object-store/src/lib.rs: -------------------------------------------------------------------------------- 1 | use object_store_internal::{ 2 | ArrowFileSystemHandler, ObjectInputFile, ObjectOutputStream, PyClientOptions, PyListResult, 3 | PyObjectMeta, PyObjectStore, PyPath, 4 | }; 5 | use pyo3::prelude::*; 6 | 7 | #[pymodule] 8 | fn _internal(_py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { 9 | // Register the python classes 10 | m.add_class::()?; 11 | m.add_class::()?; 12 | m.add_class::()?; 13 | m.add_class::()?; 14 | m.add_class::()?; 15 | m.add_class::()?; 16 | m.add_class::()?; 17 | m.add_class::()?; 18 | 19 | Ok(()) 20 | } 21 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Object Stores 2 | 3 | ```py 4 | from object_store import ObjectStore 5 | 6 | store = ObjectStore(".") 7 | 8 | store.put("path", b"data") 9 | loaded = store.get("path") 10 | 11 | assert loaded == b"data" 12 | ``` 13 | 14 | ## Integrations 15 | 16 | ### pyarrow 17 | 18 | ```py 19 | from pathlib import Path 20 | 21 | import numpy as np 22 | import pyarrow as pa 23 | import pyarrow.fs as fs 24 | import pyarrow.dataset as ds 25 | import pyarrow.parquet as pq 26 | 27 | from object_store import ArrowFileSystemHandler 28 | 29 | table = pa.table({"a": range(10), "b": np.random.randn(10), "c": [1, 2] * 5}) 30 | 31 | base = Path.cwd() 32 | store = fs.PyFileSystem(ArrowFileSystemHandler(str(base.absolute()))) 33 | 34 | pq.write_table(table.slice(0, 5), "data/data1.parquet", filesystem=store) 35 | pq.write_table(table.slice(5, 10), "data/data2.parquet", filesystem=store) 36 | ``` 37 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | set dotenv-load := true 2 | 3 | _default: 4 | just --list 5 | 6 | # initialize repository 7 | init: 8 | poetry install --no-root 9 | poetry run pip install --upgrade pip 10 | poetry run pre-commit install 11 | 12 | # build development version of packages 13 | develop: 14 | poetry run maturin develop -m object-store/Cargo.toml --extras=pyarrow 15 | 16 | build: 17 | poetry run maturin build -m object-store/Cargo.toml --release 18 | 19 | # run automatic code formatters 20 | fix: 21 | poetry run black . 22 | poetry run ruff --fix . 23 | 24 | # run object-store python tests 25 | test-py: 26 | poetry run pytest object-store/ --benchmark-autosave --cov 27 | 28 | # run object-store rust tests 29 | test-rs: 30 | cargo test 31 | 32 | # run all tests 33 | test: test-rs test-py 34 | 35 | # serve the documentation 36 | serve: 37 | poetry run mkdocs serve 38 | -------------------------------------------------------------------------------- /object-store-internal/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "object-store-internal" 3 | version = "0.2.0" 4 | edition = "2021" 5 | authors = ["Robert Pack "] 6 | description = "A generic object store interface for uniformly interacting with AWS S3, Google Cloud Storage, Azure Storage and local files." 7 | 8 | [dependencies] 9 | async-trait = "0.1.57" 10 | bytes = "1.2.1" 11 | futures = "0.3" 12 | once_cell = "1.12.0" 13 | object_store = { version = "0.10.2", features = ["azure", "aws", "gcp"] } 14 | percent-encoding = "2" 15 | pyo3 = { version = "0.21", default-features = false, features = ["macros"] } 16 | pyo3-asyncio-0-21 = { version = "0.21", features = ["tokio-runtime"] } 17 | thiserror = "1.0.34" 18 | tokio = { version = "1.0", features = [ 19 | "macros", 20 | "rt", 21 | "rt-multi-thread", 22 | "sync", 23 | ] } 24 | url = "2.3" 25 | 26 | # reqwest is pulled in by object store, but not used by python binding itself 27 | # for binary wheel best practice, statically link openssl 28 | reqwest = { version = "*", features = ["native-tls-vendored"] } 29 | 30 | [lib] 31 | crate-type = ["rlib"] 32 | -------------------------------------------------------------------------------- /mkdocs.yaml: -------------------------------------------------------------------------------- 1 | site_name: object-store-python 2 | repo_url: https://github.com/roeap/object-store-python 3 | 4 | nav: 5 | - Home: index.md 6 | - Reference: reference.md 7 | 8 | theme: 9 | name: material 10 | palette: 11 | # Palette toggle for light mode 12 | - media: "(prefers-color-scheme: light)" 13 | scheme: default 14 | 15 | toggle: 16 | icon: material/lightbulb 17 | name: Switch to dark mode 18 | 19 | # Palette toggle for dark mode 20 | - media: "(prefers-color-scheme: dark)" 21 | scheme: slate 22 | toggle: 23 | icon: material/lightbulb-outline 24 | name: Switch to system preference 25 | 26 | markdown_extensions: 27 | - admonition 28 | - pymdownx.highlight: 29 | anchor_linenums: true 30 | linenums: true 31 | - pymdownx.inlinehilite 32 | - pymdownx.snippets 33 | - pymdownx.superfences 34 | 35 | plugins: 36 | - search 37 | - mkdocstrings: 38 | handlers: 39 | python: 40 | options: 41 | show_root_heading: true 42 | show_root_full_path: false 43 | show_bases: false 44 | -------------------------------------------------------------------------------- /object-store/python/object_store/arrow.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List, Optional 2 | 3 | import pyarrow as pa 4 | import pyarrow.fs as fs 5 | 6 | from ._internal import ArrowFileSystemHandler as _ArrowFileSystemHandler 7 | 8 | 9 | # NOTE the order of inheritance is important to make sure the right methods are overwritten. 10 | # _ArrowFileSystemHandler mus be the first element in the inherited classes, we need to also 11 | # inherit form fs.FileSystemHandler to pass pyarrow's type checks. 12 | class ArrowFileSystemHandler(_ArrowFileSystemHandler, fs.FileSystemHandler): 13 | def open_input_file(self, path: str) -> "pa.PythonFile": 14 | return pa.PythonFile(_ArrowFileSystemHandler.open_input_file(self, path)) 15 | 16 | def open_input_stream(self, path: str) -> "pa.PythonFile": 17 | return pa.PythonFile(_ArrowFileSystemHandler.open_input_file(self, path)) 18 | 19 | def open_output_stream(self, path: str, metadata: Optional[Dict[str, str]] = None) -> "pa.PythonFile": 20 | return pa.PythonFile(_ArrowFileSystemHandler.open_output_stream(self, path, metadata)) 21 | 22 | def get_file_info_selector(self, selector: fs.FileSelector) -> List["fs.FileInfo"]: 23 | return _ArrowFileSystemHandler.get_file_info_selector( 24 | self, selector.base_dir, selector.allow_not_found, selector.recursive 25 | ) 26 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: https://EditorConfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 2 8 | end_of_line = lf 9 | charset = utf-8 10 | trim_trailing_whitespace = true 11 | insert_final_newline = true 12 | 13 | [*.{py,pyi}] 14 | indent_size = 4 15 | max_line_length = 100 16 | multi_line_output = 3 17 | include_trailing_comma = True 18 | force_grid_wrap = 0 19 | use_parentheses = True 20 | ensure_newline_before_comments = True 21 | 22 | [*.rs] 23 | end_of_line = lf 24 | charset = utf-8 25 | trim_trailing_whitespace = true 26 | indent_style = space 27 | indent_size = 4 28 | 29 | [*.html] 30 | indent_size = 2 31 | 32 | [*.json] 33 | indent_size = 2 34 | insert_final_newline = ignore 35 | 36 | # Minified JavaScript files shouldn't be changed 37 | [**.min.js] 38 | indent_style = ignore 39 | insert_final_newline = ignore 40 | 41 | [{Makefile,**.mk}] 42 | # Use tabs for indentation (Makefiles require tabs) 43 | indent_style = tab 44 | # Use Unix line endings 45 | end_of_line = lf 46 | # The files are utf-8 encoded 47 | charset = utf-8 48 | 49 | # Batch files use tabs for indentation 50 | [*.bat] 51 | indent_style = tab 52 | 53 | [*.md] 54 | trim_trailing_whitespace = true 55 | insert_final_newline = false 56 | 57 | [*.cmd] 58 | end_of_line = crlf 59 | 60 | [*.env] 61 | end_of_line = crlf 62 | 63 | [*.pem] 64 | insert_final_newline = false 65 | 66 | [*.crt] 67 | insert_final_newline = false 68 | 69 | [*.j2] 70 | insert_final_newline = false 71 | 72 | [justfile] 73 | indent_size = 4 74 | indent_style = space 75 | -------------------------------------------------------------------------------- /object-store/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["maturin>=1.4,<2.0", "typing_extensions"] 3 | build-backend = "maturin" 4 | 5 | [project] 6 | name = "object-store-python" 7 | version = "0.1.10" 8 | description = "A generic object store interface for uniformly interacting with AWS S3, Google Cloud Storage, Azure Storage and local files." 9 | requires-python = ">=3.8" 10 | readme = "README.md" 11 | keywords = ["object-store", "azure", "aws", "gcp"] 12 | authors = [{ name = "Robert Pack", email = "robstar.pack@gmail.com" }] 13 | classifiers = [ 14 | "Development Status :: 3 - Alpha", 15 | "Programming Language :: Python :: 3 :: Only", 16 | "Programming Language :: Python :: 3.8", 17 | "Programming Language :: Python :: 3.9", 18 | "Programming Language :: Python :: 3.10", 19 | "Programming Language :: Python :: 3.11", 20 | "Programming Language :: Rust", 21 | "Intended Audience :: Developers", 22 | "License :: OSI Approved :: Apache Software License", 23 | ] 24 | 25 | [project.optional-dependencies] 26 | pyarrow = ["pyarrow>=7.0"] 27 | 28 | [project.urls] 29 | Documentation = "https://github.com/roeap/object-store-python#readme" 30 | Repository = "https://github.com/roeap/object-store-python" 31 | 32 | [tool.maturin] 33 | features = ["pyo3/extension-module"] 34 | module-name = "object_store._internal" 35 | python-source = "python" 36 | sdist-include = ["Cargo.lock"] 37 | 38 | [tool.coverage.report] 39 | fail_under = 85 40 | exclude_lines = [ 41 | "pragma: no cover", 42 | "@overload", 43 | "except ImportError", 44 | "if TYPE_CHECKING:", 45 | "from typing_extensions import ", 46 | ] 47 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_install_hook_types: [pre-commit, commit-msg] 2 | default_stages: [commit, push] 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v4.6.0 6 | hooks: 7 | - id: check-case-conflict 8 | - id: check-merge-conflict 9 | - id: end-of-file-fixer 10 | - id: mixed-line-ending 11 | - id: trailing-whitespace 12 | 13 | - repo: https://github.com/commitizen-tools/commitizen 14 | rev: v3.28.0 15 | hooks: 16 | - id: commitizen 17 | stages: [commit-msg] 18 | 19 | - repo: https://github.com/astral-sh/ruff-pre-commit 20 | rev: v0.5.6 21 | hooks: 22 | - id: ruff 23 | types_or: [python, pyi] 24 | args: [--fix] 25 | - id: ruff-format 26 | types_or: [python, pyi, jupyter] 27 | 28 | - repo: https://github.com/pre-commit/mirrors-prettier 29 | rev: v4.0.0-alpha.8 30 | hooks: 31 | - id: prettier 32 | 33 | - repo: https://github.com/python-poetry/poetry 34 | rev: "1.8.0" 35 | hooks: 36 | - id: poetry-check 37 | 38 | - repo: https://github.com/RobertCraigie/pyright-python 39 | rev: v1.1.374 40 | hooks: 41 | - id: pyright 42 | 43 | - repo: local 44 | hooks: 45 | - id: just 46 | name: just 47 | language: system 48 | entry: just --fmt --unstable --check 49 | files: ^justfile$ 50 | pass_filenames: false 51 | types: 52 | - file 53 | 54 | - id: rustfmt 55 | name: Rust Format 56 | language: system 57 | entry: bash -c "cargo +stable fmt --all -- --check" 58 | files: ^.*\.rs$ 59 | types: 60 | - file 61 | - rust 62 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "object-stores" 3 | version = "0.2.0" 4 | description = "Python bindings and integrations for the rust object_store crate." 5 | authors = ["Robert Pack "] 6 | license = "MIT" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.9,<3.13" 10 | object-store-python = { path = "object-store/", develop = true, extras = [ 11 | "pyarrow", 12 | "mlflow", 13 | ] } 14 | 15 | [tool.poetry.group.dev.dependencies] 16 | maturin = "^1.7" 17 | pre-commit = "^3.8" 18 | pytest = "^8" 19 | pytest-datadir = ">=1.3.1" 20 | ipykernel = ">=6.15.2" 21 | pytest-cov = "^5.0.0" 22 | ruff = "^0.5.6" 23 | pytest-benchmark = "^4.0.0" 24 | 25 | [tool.poetry.group.examples.dependencies] 26 | duckdb = "^1" 27 | 28 | [tool.poetry.group.docs.dependencies] 29 | mkdocs = "^1.4" 30 | mkdocs-material = "^9" 31 | mkdocstrings = { version = ">=0.25", extras = ["python"] } 32 | 33 | [tool.black] 34 | color = true 35 | line-length = 120 36 | target-version = ['py38', 'py39', 'py310', 'py311'] 37 | include = '\.pyi?$' 38 | 39 | [tool.ruff] 40 | exclude = [ 41 | '__pycache__', 42 | '.git', 43 | '.ipynb_checkpoints', 44 | '.venv', 45 | '.tox', 46 | '.mypy_cache', 47 | '.pytest_cache', 48 | '.vscode', 49 | '.github', 50 | 'build', 51 | 'dist', 52 | 'typestubs', 53 | '*.pyi', 54 | ] 55 | line-length = 120 56 | 57 | [tool.ruff.lint] 58 | ignore = ['E501'] 59 | select = ['B', 'C4', 'E', 'F', "I", "S", 'W'] 60 | 61 | [tool.ruff.lint.per-file-ignores] 62 | # allow asserts in test files (bandit) 63 | "test_*" = ["S101"] 64 | 65 | [tool.ruff.lint.isort] 66 | known-first-party = ["object_store"] 67 | 68 | [tool.pyright] 69 | reportUnnecessaryTypeIgnoreComment = true 70 | venvPath = "." 71 | venv = ".venv" 72 | 73 | [tool.coverage.report] 74 | fail_under = 85 75 | exclude_lines = [ 76 | "pragma: no cover", 77 | "@overload", 78 | "except ImportError", 79 | "if TYPE_CHECKING:", 80 | "from typing_extensions import ", 81 | ] 82 | -------------------------------------------------------------------------------- /.vscode/cspell.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2", 3 | "ignorePaths": [], 4 | "dictionaryDefinitions": [], 5 | "dictionaries": [], 6 | "words": [], 7 | "ignoreWords": [ 8 | "Datetime", 9 | "Hasher", 10 | "Introspector", 11 | "Kubernetes", 12 | "Kusto", 13 | "Noneable", 14 | "Pageable", 15 | "Popen", 16 | "RUSTFLAGS", 17 | "Timespan", 18 | "addinivalue", 19 | "antd", 20 | "betterproto", 21 | "canonicalize", 22 | "chrono", 23 | "classmethod", 24 | "clippy", 25 | "cloudpickle", 26 | "codegen", 27 | "dadc", 28 | "dagit", 29 | "dagster", 30 | "dataclass", 31 | "datadir", 32 | "datafusion", 33 | "datalake", 34 | "dataplane", 35 | "datasource", 36 | "debuginfo", 37 | "deps", 38 | "devel", 39 | "dotenv", 40 | "filetime", 41 | "getattr", 42 | "getfixturevalue", 43 | "grpclib", 44 | "hmac", 45 | "httpx", 46 | "inlinehilite", 47 | "isatty", 48 | "isort", 49 | "itertools", 50 | "jdbc", 51 | "justfile", 52 | "kwargs", 53 | "linenums", 54 | "maturin", 55 | "mkdir", 56 | "mkdocs", 57 | "mkdocstrings", 58 | "mlflow", 59 | "mlflowclient", 60 | "mlfusion", 61 | "mlserver", 62 | "multiprocess", 63 | "nanos", 64 | "nbytes", 65 | "numpy", 66 | "opentelemetry", 67 | "pathlib", 68 | "petgraph", 69 | "plotly", 70 | "prost", 71 | "proto", 72 | "protobuf", 73 | "protoc", 74 | "pyarrow", 75 | "pyclass", 76 | "pydantic", 77 | "pyerr", 78 | "pyfunction", 79 | "pymdownx", 80 | "pymethods", 81 | "pymodule", 82 | "pyproject", 83 | "pyright", 84 | "quicktype", 85 | "randn", 86 | "readline", 87 | "readlines", 88 | "repr", 89 | "reqwest", 90 | "richcmp", 91 | "ritelinked", 92 | "rustfmt", 93 | "rustup", 94 | "schemafy", 95 | "seekable", 96 | "sklearn", 97 | "sqlparser", 98 | "structopt", 99 | "subschema", 100 | "superfences", 101 | "tempdir", 102 | "tempfile", 103 | "thiserror", 104 | "tolerations", 105 | "toposort", 106 | "typer", 107 | "unpartitioned", 108 | "venv", 109 | "virtualenvs", 110 | "walkdir", 111 | "weakref" 112 | ], 113 | "import": [] 114 | } 115 | -------------------------------------------------------------------------------- /object-store-internal/src/utils.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use bytes::Bytes; 4 | use futures::future::{join_all, BoxFuture, FutureExt}; 5 | use futures::{StreamExt, TryStreamExt}; 6 | use object_store::path::Path; 7 | use object_store::{DynObjectStore, ListResult, ObjectMeta, Result as ObjectStoreResult}; 8 | 9 | /// List directory 10 | pub async fn flatten_list_stream( 11 | storage: &DynObjectStore, 12 | prefix: Option<&Path>, 13 | ) -> ObjectStoreResult> { 14 | storage.list(prefix).try_collect::>().await 15 | } 16 | 17 | pub async fn walk_tree( 18 | storage: Arc, 19 | path: &Path, 20 | recursive: bool, 21 | ) -> ObjectStoreResult { 22 | list_with_delimiter_recursive(storage, [path.clone()], recursive).await 23 | } 24 | 25 | fn list_with_delimiter_recursive( 26 | storage: Arc, 27 | paths: impl IntoIterator, 28 | recursive: bool, 29 | ) -> BoxFuture<'static, ObjectStoreResult> { 30 | let mut tasks = vec![]; 31 | for path in paths { 32 | let store = storage.clone(); 33 | let prefix = path.clone(); 34 | let handle = 35 | tokio::task::spawn(async move { store.list_with_delimiter(Some(&prefix)).await }); 36 | tasks.push(handle); 37 | } 38 | 39 | async move { 40 | let mut results = join_all(tasks) 41 | .await 42 | .into_iter() 43 | .collect::, _>>()? 44 | .into_iter() 45 | .collect::, _>>()? 46 | .into_iter() 47 | .fold( 48 | ListResult { 49 | common_prefixes: vec![], 50 | objects: vec![], 51 | }, 52 | |mut acc, res| { 53 | acc.common_prefixes.extend(res.common_prefixes); 54 | acc.objects.extend(res.objects); 55 | acc 56 | }, 57 | ); 58 | 59 | if recursive && !results.common_prefixes.is_empty() { 60 | let more_result = list_with_delimiter_recursive( 61 | storage.clone(), 62 | results.common_prefixes.clone(), 63 | recursive, 64 | ) 65 | .await?; 66 | results.common_prefixes.extend(more_result.common_prefixes); 67 | results.objects.extend(more_result.objects); 68 | } 69 | 70 | Ok(results) 71 | } 72 | .boxed() 73 | } 74 | 75 | pub async fn delete_dir(storage: &DynObjectStore, prefix: &Path) -> ObjectStoreResult<()> { 76 | // TODO batch delete would be really useful now... 77 | let mut stream = storage.list(Some(prefix)); 78 | while let Some(maybe_meta) = stream.next().await { 79 | let meta = maybe_meta?; 80 | storage.delete(&meta.location).await?; 81 | } 82 | Ok(()) 83 | } 84 | 85 | /// get bytes from a location 86 | pub async fn get_bytes(storage: &DynObjectStore, path: &Path) -> ObjectStoreResult { 87 | storage.get(path).await?.bytes().await 88 | } 89 | -------------------------------------------------------------------------------- /object-store/tests/test_object_store.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from pathlib import Path 4 | 5 | import pytest 6 | 7 | from object_store import ObjectStore 8 | from object_store import Path as ObjectStorePath 9 | 10 | 11 | @pytest.fixture 12 | def object_store(datadir: Path) -> tuple[ObjectStore, Path]: 13 | return ObjectStore(str(datadir)), datadir 14 | 15 | 16 | def test_put_get_delete_list(object_store: tuple[ObjectStore, Path]): 17 | store, _ = object_store 18 | 19 | files = store.list() 20 | assert len(files) == 0 21 | 22 | expected_data = b"arbitrary data" 23 | location = ObjectStorePath("test_dir/test_file.json") 24 | store.put("test_dir/test_file.json", expected_data) 25 | 26 | files = store.list() 27 | assert len(files) == 1 28 | assert files[0].location == location 29 | 30 | files = store.list(ObjectStorePath("/")) 31 | assert len(files) == 1 32 | assert files[0].location == location 33 | 34 | result = store.list_with_delimiter() 35 | assert len(result.objects) == 0 36 | assert len(result.common_prefixes) == 1 37 | assert result.common_prefixes[0] == ObjectStorePath("test_dir") 38 | 39 | result = store.list_with_delimiter(ObjectStorePath("/")) 40 | assert len(result.objects) == 0 41 | assert len(result.common_prefixes) == 1 42 | assert result.common_prefixes[0] == ObjectStorePath("test_dir") 43 | 44 | files = store.list(ObjectStorePath("test_dir")) 45 | assert len(files) == 1 46 | assert files[0].location == location 47 | 48 | files = store.list(ObjectStorePath("something")) 49 | assert len(files) == 0 50 | 51 | data = store.get(location) 52 | assert data == expected_data 53 | 54 | range_result = store.get_range(location, 3, 4) 55 | assert range_result == expected_data[3:7] 56 | 57 | with pytest.raises(Exception): # noqa: B017 58 | store.get_range(location, 200, 100) 59 | 60 | head = store.head(location) 61 | assert head.location == location 62 | assert head.size == len(expected_data) 63 | 64 | store.delete(location) 65 | 66 | files = store.list() 67 | assert len(files) == 0 68 | 69 | with pytest.raises(FileNotFoundError): 70 | store.get(location) 71 | 72 | with pytest.raises(FileNotFoundError): 73 | store.head(location) 74 | 75 | 76 | def test_rename_and_copy(object_store: tuple[ObjectStore, Path]): 77 | store, _ = object_store 78 | 79 | path1 = ObjectStorePath("test1") 80 | path2 = ObjectStorePath("test2") 81 | contents1 = b"cats" 82 | contents2 = b"dogs" 83 | 84 | # copy() make both objects identical 85 | store.put(path1, contents1) 86 | store.put(path2, contents2) 87 | store.copy(path1, path2) 88 | new_contents = store.get(path2) 89 | assert new_contents == contents1 90 | 91 | # rename() copies contents and deletes original 92 | store.put(path1, contents1) 93 | store.put(path2, contents2) 94 | store.rename(path1, path2) 95 | new_contents = store.get(path2) 96 | assert new_contents == contents1 97 | with pytest.raises(FileNotFoundError): 98 | store.get(path1) 99 | 100 | store.delete(path2) 101 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release to PyPI 2 | 3 | on: 4 | push: 5 | tags: ["v*"] 6 | 7 | env: 8 | MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} 9 | 10 | jobs: 11 | validate-release-tag: 12 | name: Validate git tag 13 | runs-on: ubuntu-20.04 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: compare git tag with cargo metadata 18 | working-directory: object-store/ 19 | run: | 20 | PUSHED_TAG=${GITHUB_REF##*/} 21 | CURR_VER=$( grep version Cargo.toml | head -n 1 | awk '{print $3}' | tr -d '"' ) 22 | if [[ "${PUSHED_TAG}" != "v${CURR_VER}" ]]; then 23 | echo "Cargo metadata has version set to ${CURR_VER}, but got pushed tag ${PUSHED_TAG}." 24 | exit 1 25 | fi 26 | 27 | release-pypi-mac: 28 | needs: validate-release-tag 29 | name: PyPI release on Mac 30 | runs-on: macos-11 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | target: [x86_64-apple-darwin, aarch64-apple-darwin] 36 | 37 | steps: 38 | - uses: actions/checkout@v3 39 | - run: rm object-store/README.md && cp README.md object-store/README.md 40 | 41 | - uses: actions/setup-python@v2 42 | with: 43 | python-version: "3.10" 44 | 45 | - uses: PyO3/maturin-action@v1 46 | name: Publish to pypi (without sdist) 47 | with: 48 | target: ${{ matrix.target }} 49 | command: publish 50 | args: -m object-store/Cargo.toml --no-sdist 51 | 52 | release-pypi-mac-universal2: 53 | needs: validate-release-tag 54 | name: PyPI release on Mac universal 2 55 | runs-on: macos-latest 56 | 57 | steps: 58 | - uses: actions/checkout@v3 59 | - run: rm object-store/README.md && cp README.md object-store/README.md 60 | 61 | - uses: actions/setup-python@v2 62 | with: 63 | python-version: "3.10" 64 | 65 | - uses: PyO3/maturin-action@v1 66 | name: Publish to pypi (without sdist) 67 | with: 68 | target: ${{ matrix.target }} 69 | command: publish 70 | args: -m object-store/Cargo.toml --no-sdist --universal2 71 | 72 | release-pypi-windows: 73 | needs: validate-release-tag 74 | name: PyPI release on Windows 75 | runs-on: windows-2019 76 | 77 | steps: 78 | - uses: actions/checkout@v3 79 | - run: rm object-store/README.md && cp README.md object-store/README.md 80 | 81 | - uses: actions/setup-python@v2 82 | with: 83 | python-version: "3.10" 84 | 85 | - uses: PyO3/maturin-action@v1 86 | name: Publish to pypi (without sdist) 87 | with: 88 | target: x86_64-pc-windows-msvc 89 | command: publish 90 | args: -m object-store/Cargo.toml --no-sdist 91 | 92 | release-pypi-manylinux: 93 | needs: validate-release-tag 94 | name: PyPI release manylinux 95 | runs-on: ubuntu-latest 96 | 97 | steps: 98 | - uses: actions/checkout@v3 99 | - run: rm object-store/README.md && cp README.md object-store/README.md 100 | 101 | - uses: actions/setup-python@v2 102 | with: 103 | python-version: "3.10" 104 | 105 | - uses: PyO3/maturin-action@v1 106 | name: Publish manylinux to pypi x86_64 (with sdist) 107 | with: 108 | target: x86_64-unknown-linux-gnu 109 | command: publish 110 | args: -m object-store/Cargo.toml 111 | 112 | - uses: PyO3/maturin-action@v1 113 | name: Publish manylinux to pypi aarch64 (without sdist) 114 | with: 115 | target: aarch64-unknown-linux-gnu 116 | command: publish 117 | args: -m object-store/Cargo.toml --no-sdist 118 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | env: 10 | CARGO_INCREMENTAL: 0 11 | CARGO_NET_RETRY: 10 12 | RUSTUP_MAX_RETRIES: 10 13 | 14 | jobs: 15 | cargo_build: 16 | name: "cargo build" 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v3 20 | - uses: actions-rs/toolchain@v1 21 | with: 22 | profile: minimal 23 | toolchain: stable 24 | override: true 25 | - uses: actions/cache@v3 26 | env: 27 | cache-name: cache-cargo 28 | with: 29 | path: | 30 | ~/.cargo/registry 31 | ~/.cargo/git 32 | key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} 33 | restore-keys: | 34 | ${{ runner.os }}-build-${{ env.cache-name }}- 35 | ${{ runner.os }}-build- 36 | ${{ runner.os }}- 37 | - run: cargo build --all --release 38 | 39 | cargo_fmt: 40 | name: "cargo fmt" 41 | runs-on: ubuntu-latest 42 | steps: 43 | - uses: actions/checkout@v3 44 | - uses: actions-rs/toolchain@v1 45 | with: 46 | profile: minimal 47 | toolchain: stable 48 | override: true 49 | components: rustfmt 50 | - uses: actions/cache@v3 51 | env: 52 | cache-name: cache-cargo 53 | with: 54 | path: | 55 | ~/.cargo/registry 56 | ~/.cargo/git 57 | key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} 58 | restore-keys: | 59 | ${{ runner.os }}-build-${{ env.cache-name }}- 60 | ${{ runner.os }}-build- 61 | ${{ runner.os }}- 62 | - run: cargo fmt --all --check 63 | 64 | cargo_clippy: 65 | name: "cargo clippy" 66 | runs-on: ubuntu-latest 67 | steps: 68 | - uses: actions/checkout@v3 69 | - uses: actions-rs/toolchain@v1 70 | with: 71 | profile: minimal 72 | toolchain: stable 73 | override: true 74 | components: clippy 75 | - uses: actions/cache@v3 76 | env: 77 | cache-name: cache-cargo 78 | with: 79 | path: | 80 | ~/.cargo/registry 81 | ~/.cargo/git 82 | key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} 83 | restore-keys: | 84 | ${{ runner.os }}-build-${{ env.cache-name }}- 85 | ${{ runner.os }}-build- 86 | ${{ runner.os }}- 87 | - run: cargo clippy --workspace --all-targets --all-features -- -D warnings 88 | 89 | cargo_test: 90 | name: "cargo test" 91 | runs-on: ubuntu-latest 92 | steps: 93 | - uses: actions/checkout@v3 94 | - uses: actions-rs/toolchain@v1 95 | with: 96 | profile: minimal 97 | toolchain: stable 98 | override: true 99 | - uses: actions/cache@v3 100 | env: 101 | cache-name: cache-cargo 102 | with: 103 | path: | 104 | ~/.cargo/registry 105 | ~/.cargo/git 106 | key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} 107 | restore-keys: | 108 | ${{ runner.os }}-build-${{ env.cache-name }}- 109 | ${{ runner.os }}-build- 110 | ${{ runner.os }}- 111 | - run: cargo test --all 112 | 113 | maturin_build: 114 | name: "maturin build" 115 | runs-on: ubuntu-latest 116 | steps: 117 | - uses: actions/checkout@v3 118 | - uses: actions-rs/toolchain@v1 119 | with: 120 | profile: minimal 121 | toolchain: stable 122 | override: true 123 | - uses: actions/setup-python@v4 124 | with: 125 | python-version: "3.10" 126 | - run: pip install maturin 127 | - uses: actions/cache@v3 128 | env: 129 | cache-name: cache-cargo 130 | with: 131 | path: | 132 | ~/.cargo/registry 133 | ~/.cargo/git 134 | key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} 135 | restore-keys: | 136 | ${{ runner.os }}-build-${{ env.cache-name }}- 137 | ${{ runner.os }}-build- 138 | ${{ runner.os }}- 139 | - run: maturin build -m object-store/Cargo.toml 140 | -------------------------------------------------------------------------------- /object-store/tests/test_arrow.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections import Counter 4 | from pathlib import Path 5 | 6 | import numpy as np 7 | import pyarrow as pa 8 | import pyarrow.dataset as ds 9 | import pyarrow.fs as fs 10 | import pyarrow.parquet as pq 11 | import pytest 12 | 13 | from object_store.arrow import ArrowFileSystemHandler 14 | 15 | 16 | @pytest.fixture 17 | def file_systems(datadir: Path): 18 | store = fs.PyFileSystem(ArrowFileSystemHandler(str(datadir.absolute()))) 19 | arrow_fs = fs.SubTreeFileSystem(str(datadir.absolute()), fs.LocalFileSystem()) 20 | return (store, arrow_fs) 21 | 22 | 23 | @pytest.fixture 24 | def table_data(): 25 | return pa.Table.from_arrays([pa.array([1, 2, 3]), pa.array(["a", "b", "c"])], names=["int", "str"]) 26 | 27 | 28 | def test_file_info(file_systems: tuple[fs.PyFileSystem, fs.SubTreeFileSystem], table_data): 29 | store, arrow_fs = file_systems 30 | file_path = "table.parquet" 31 | 32 | pq.write_table(table_data, file_path, filesystem=arrow_fs) 33 | 34 | info = store.get_file_info(file_path) 35 | arrow_info = arrow_fs.get_file_info(file_path) 36 | 37 | assert type(info) is type(arrow_info) 38 | assert info.path == arrow_info.path 39 | assert info.type == arrow_info.type 40 | assert info.size == arrow_info.size 41 | assert info.mtime_ns == arrow_info.mtime_ns 42 | assert info.mtime == arrow_info.mtime 43 | 44 | 45 | def test_get_file_info_selector(file_systems: tuple[fs.PyFileSystem, fs.SubTreeFileSystem]): 46 | store, arrow_fs = file_systems 47 | table = pa.table({"a": range(10), "b": np.random.randn(10), "c": [1, 2] * 5}) 48 | partitioning = ds.partitioning(pa.schema([("c", pa.int64())]), flavor="hive") 49 | ds.write_dataset( 50 | table, 51 | "/", 52 | partitioning=partitioning, 53 | format="parquet", 54 | filesystem=arrow_fs, 55 | ) 56 | 57 | selector = fs.FileSelector("/", recursive=True) 58 | infos = store.get_file_info(selector) 59 | arrow_infos = arrow_fs.get_file_info(selector) 60 | 61 | assert Counter([i.type for i in infos]) == Counter([i.type for i in arrow_infos]) 62 | 63 | 64 | def test_open_input_file(file_systems: tuple[fs.PyFileSystem, fs.SubTreeFileSystem], table_data): 65 | store, arrow_fs = file_systems 66 | file_path = "table.parquet" 67 | 68 | pq.write_table(table_data, file_path, filesystem=arrow_fs) 69 | 70 | file = store.open_input_file(file_path) 71 | arrow_file = arrow_fs.open_input_file(file_path) 72 | 73 | # Check the metadata 74 | assert file.mode == arrow_file.mode 75 | assert file.closed == arrow_file.closed 76 | assert file.size() == arrow_file.size() 77 | assert file.isatty() == arrow_file.isatty() 78 | assert file.readable() == arrow_file.readable() 79 | assert file.seekable() == arrow_file.seekable() 80 | 81 | # Check reading the same content 82 | assert file.read() == arrow_file.read() 83 | # Check subsequent read (should return no data anymore) 84 | assert file.read() == arrow_file.read() 85 | assert file.read() == b"" 86 | 87 | file = store.open_input_file(file_path) 88 | arrow_file = arrow_fs.open_input_file(file_path) 89 | 90 | # Check seeking works as expected 91 | assert file.tell() == arrow_file.tell() 92 | assert file.seek(2) == arrow_file.seek(2) 93 | assert file.tell() == arrow_file.tell() 94 | assert file.tell() == 2 95 | 96 | # check reading works as expected 97 | assert file.read(10) == arrow_file.read(10) 98 | assert file.read1(10) == arrow_file.read1(10) 99 | assert file.read_at(10, 0) == arrow_file.read_at(10, 0) 100 | 101 | 102 | def test_read_table(file_systems: tuple[fs.PyFileSystem, fs.SubTreeFileSystem], table_data): 103 | store, arrow_fs = file_systems 104 | file_path = "table.parquet" 105 | 106 | pq.write_table(table_data, file_path, filesystem=arrow_fs) 107 | 108 | table = pq.read_table(file_path, filesystem=store) 109 | arrow_table = pq.read_table(file_path, filesystem=arrow_fs) 110 | 111 | assert isinstance(table, pa.Table) 112 | assert table.equals(arrow_table) 113 | 114 | 115 | def test_read_dataset(file_systems: tuple[fs.PyFileSystem, fs.SubTreeFileSystem]): 116 | store, arrow_fs = file_systems 117 | table = pa.table({"a": range(10), "b": np.random.randn(10), "c": [1, 2] * 5}) 118 | 119 | pq.write_table(table.slice(0, 5), "data1.parquet", filesystem=arrow_fs) 120 | pq.write_table(table.slice(5, 10), "data2.parquet", filesystem=arrow_fs) 121 | 122 | dataset = ds.dataset("/", format="parquet", filesystem=store) 123 | ds_table = dataset.to_table() 124 | 125 | assert table.schema == dataset.schema 126 | assert table.equals(ds_table) 127 | 128 | 129 | def test_write_table(file_systems: tuple[fs.PyFileSystem, fs.SubTreeFileSystem]): 130 | store, _ = file_systems 131 | table = pa.table({"a": range(10), "b": np.random.randn(10), "c": [1, 2] * 5}) 132 | 133 | pq.write_table(table.slice(0, 5), "data1.parquet", filesystem=store) 134 | pq.write_table(table.slice(5, 10), "data2.parquet", filesystem=store) 135 | 136 | dataset = ds.dataset("/", format="parquet", filesystem=store) 137 | ds_table = dataset.to_table() 138 | 139 | assert table.schema == ds_table.schema 140 | assert table.equals(ds_table) 141 | 142 | 143 | def test_write_partitioned_dataset(file_systems: tuple[fs.PyFileSystem, fs.SubTreeFileSystem]): 144 | store, arrow_fs = file_systems 145 | table = pa.table({"a": range(10), "b": np.random.randn(10), "c": [1, 2] * 5}) 146 | 147 | partitioning = ds.partitioning(pa.schema([("c", pa.int64())]), flavor="hive") 148 | ds.write_dataset( 149 | table, 150 | "/", 151 | partitioning=partitioning, 152 | format="parquet", 153 | filesystem=store, 154 | ) 155 | 156 | dataset = ds.dataset("/", format="parquet", filesystem=arrow_fs, partitioning=partitioning) 157 | ds_table = dataset.to_table().select(["a", "b", "c"]) 158 | 159 | dataset = ds.dataset("/", format="parquet", filesystem=store, partitioning=partitioning) 160 | ds_table2 = dataset.to_table().select(["a", "b", "c"]) 161 | 162 | assert table.schema == ds_table.schema 163 | assert table.schema == ds_table2.schema 164 | assert table.shape == ds_table.shape 165 | assert table.shape == ds_table2.shape 166 | -------------------------------------------------------------------------------- /examples/object_store.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from pathlib import Path as PythonPath\n", 10 | "\n", 11 | "import numpy as np\n", 12 | "import pyarrow as pa\n", 13 | "import pyarrow.dataset as ds\n", 14 | "import pyarrow.fs as fs\n", 15 | "import pyarrow.parquet as pq\n", 16 | "\n", 17 | "from object_store.arrow import ArrowFileSystemHandler\n", 18 | "\n", 19 | "table = pa.table({\"a\": range(10), \"b\": np.random.randn(10), \"c\": [1, 2] * 5})\n", 20 | "\n", 21 | "base = PythonPath.cwd()\n", 22 | "store = fs.PyFileSystem(ArrowFileSystemHandler(str(base.absolute())))\n", 23 | "arrow_fs = fs.SubTreeFileSystem(str(base.absolute()), fs.LocalFileSystem())\n", 24 | "\n", 25 | "pq.write_table(table.slice(0, 5), \"data/data1.parquet\", filesystem=store)\n", 26 | "pq.write_table(table.slice(5, 10), \"data/data2.parquet\", filesystem=store)\n", 27 | "\n", 28 | "dataset = ds.dataset(\"data\", format=\"parquet\", filesystem=store)" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "from pathlib import Path as PythonPath\n", 38 | "\n", 39 | "import numpy as np\n", 40 | "import pyarrow as pa\n", 41 | "import pyarrow.dataset as ds\n", 42 | "import pyarrow.fs as fs\n", 43 | "import pyarrow.parquet as pq\n", 44 | "\n", 45 | "from object_store.arrow import ArrowFileSystemHandler\n", 46 | "\n", 47 | "table = pa.table({\"a\": range(10), \"b\": np.random.randn(10), \"c\": [1, 2] * 5})\n", 48 | "\n", 49 | "base = PythonPath.cwd()\n", 50 | "store = ArrowFileSystemHandler(str(base.absolute()))\n", 51 | "\n", 52 | "import pickle\n", 53 | "\n", 54 | "with PythonPath(\"asd.pkl\").open(\"wb\") as handle:\n", 55 | " pickle.dump(store, handle)\n", 56 | "\n", 57 | "with PythonPath(\"asd.pkl\").open(\"rb\") as handle:\n", 58 | " store_pkl = pickle.load(handle)\n", 59 | "\n", 60 | "store_pkl.get_file_info([\"asd.pkl\"])" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "from object_store import ObjectMeta, ObjectStore\n", 70 | "\n", 71 | "# we use an in-memory store for demonstration purposes.\n", 72 | "# data will not be persisted and is not shared across store instances\n", 73 | "store = ObjectStore(\"memory://\")\n", 74 | "\n", 75 | "store.put(\"data\", b\"some data\")\n", 76 | "\n", 77 | "data = store.get(\"data\")\n", 78 | "assert data == b\"some data\"\n", 79 | "\n", 80 | "blobs = store.list()\n", 81 | "\n", 82 | "meta: ObjectMeta = store.head(\"data\")\n", 83 | "\n", 84 | "range = store.get_range(\"data\", start=0, length=4)\n", 85 | "assert range == b\"some\"\n", 86 | "\n", 87 | "store.copy(\"data\", \"copied\")\n", 88 | "copied = store.get(\"copied\")\n", 89 | "assert copied == data" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": null, 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [ 98 | "import duckdb\n", 99 | "\n", 100 | "con = duckdb.connect()\n", 101 | "results = con.execute(\"SELECT * FROM dataset WHERE c = 2\").arrow()\n", 102 | "\n", 103 | "results.shape" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | "visited_paths = []\n", 113 | "\n", 114 | "\n", 115 | "def file_visitor(written_file):\n", 116 | " visited_paths.append(written_file)\n", 117 | "\n", 118 | "\n", 119 | "partitioning = ds.partitioning(pa.schema([(\"c\", pa.int64())]), flavor=\"hive\")\n", 120 | "ds.write_dataset(\n", 121 | " table,\n", 122 | " \"partitioned\",\n", 123 | " partitioning=partitioning,\n", 124 | " format=\"parquet\",\n", 125 | " filesystem=store,\n", 126 | " file_visitor=file_visitor,\n", 127 | ")\n", 128 | "\n", 129 | "len(visited_paths)" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "partitioning = ds.partitioning(pa.schema([(\"c\", pa.int64())]), flavor=\"hive\")\n", 139 | "dataset_part = ds.dataset(\"/partitioned\", format=\"parquet\", filesystem=store, partitioning=partitioning)\n", 140 | "dataset_part.schema" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "from object_store import ObjectStore\n", 150 | "\n", 151 | "store = ObjectStore(\"az://delta-rs\", options={\"account_name\": \"mlfusiondev\", \"use_azure_cli\": \"true\"})\n", 152 | "\n", 153 | "store.list()" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": null, 159 | "metadata": {}, 160 | "outputs": [], 161 | "source": [ 162 | "import os\n", 163 | "\n", 164 | "import pyarrow.fs as pa_fs\n", 165 | "\n", 166 | "from object_store import ClientOptions\n", 167 | "from object_store.arrow import ArrowFileSystemHandler\n", 168 | "\n", 169 | "storage_options = {\n", 170 | " \"account_name\": os.environ[\"AZURE_STORAGE_ACCOUNT_NAME\"],\n", 171 | " \"account_key\": os.environ[\"AZURE_STORAGE_ACCOUNT_KEY\"],\n", 172 | "}\n", 173 | "\n", 174 | "filesystem = pa_fs.PyFileSystem(ArrowFileSystemHandler(\"adl://simple\", storage_options, ClientOptions()))\n", 175 | "filesystem.get_file_info([\"part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet\"])" 176 | ] 177 | } 178 | ], 179 | "metadata": { 180 | "kernelspec": { 181 | "display_name": ".venv", 182 | "language": "python", 183 | "name": "python3" 184 | }, 185 | "language_info": { 186 | "codemirror_mode": { 187 | "name": "ipython", 188 | "version": 3 189 | }, 190 | "file_extension": ".py", 191 | "mimetype": "text/x-python", 192 | "name": "python", 193 | "nbconvert_exporter": "python", 194 | "pygments_lexer": "ipython3", 195 | "version": "3.10.6" 196 | }, 197 | "orig_nbformat": 4, 198 | "vscode": { 199 | "interpreter": { 200 | "hash": "9d6ce819d12cb3dc1d584870253e9f5e189fd2e2773823a6ff4f2c218d69ebab" 201 | } 202 | } 203 | }, 204 | "nbformat": 4, 205 | "nbformat_minor": 2 206 | } 207 | -------------------------------------------------------------------------------- /object-store/python/object_store/__init__.py: -------------------------------------------------------------------------------- 1 | from io import BytesIO 2 | from typing import List, Optional, Union 3 | 4 | # NOTE aliasing the imports with 'as' makes them public in the eyes 5 | # of static code checkers. Thus we avoid listing them with __all__ = ... 6 | from ._internal import ClientOptions as ClientOptions 7 | from ._internal import ListResult as ListResult 8 | from ._internal import ObjectMeta as ObjectMeta 9 | from ._internal import ObjectStore as _ObjectStore 10 | from ._internal import Path as Path 11 | 12 | try: 13 | import importlib.metadata as importlib_metadata 14 | except ImportError: 15 | import importlib_metadata # type: ignore 16 | 17 | __version__ = importlib_metadata.version("object-store-python") 18 | 19 | PathLike = Union[str, List[str], Path] 20 | BytesLike = Union[bytes, BytesIO] 21 | 22 | DELIMITER = "/" 23 | 24 | 25 | def _as_path(raw: PathLike) -> Path: 26 | if isinstance(raw, str): 27 | return Path(raw) 28 | if isinstance(raw, list): 29 | return Path(DELIMITER.join(raw)) 30 | if isinstance(raw, Path): 31 | return raw 32 | raise ValueError(f"Cannot convert type '{type(raw)}' to type Path.") 33 | 34 | 35 | def _as_bytes(raw: BytesLike) -> bytes: 36 | if isinstance(raw, bytes): 37 | return raw 38 | if isinstance(raw, BytesIO): 39 | return raw.read() 40 | raise ValueError(f"Cannot convert type '{type(raw)}' to type bytes.") 41 | 42 | 43 | class ObjectStore(_ObjectStore): 44 | """A uniform API for interacting with object storage services and local files. 45 | 46 | backed by the Rust object_store crate.""" 47 | 48 | def head(self, location: PathLike) -> ObjectMeta: 49 | """Return the metadata for the specified location. 50 | 51 | Args: 52 | location (PathLike): path / key to storage location 53 | 54 | Returns: 55 | ObjectMeta: metadata for object at location 56 | """ 57 | return super().head(_as_path(location)) 58 | 59 | def get(self, location: PathLike) -> bytes: 60 | """Return the bytes that are stored at the specified location. 61 | 62 | Args: 63 | location (PathLike): path / key to storage location 64 | 65 | Returns: 66 | bytes: raw data stored in location 67 | """ 68 | return super().get(_as_path(location)) 69 | 70 | def get_range(self, location: PathLike, start: int, length: int) -> bytes: 71 | """Return the bytes that are stored at the specified location in the given byte range. 72 | 73 | Args: 74 | location (PathLike): path / key to storage location 75 | start (int): zero-based start index 76 | length (int): length of the byte range 77 | 78 | Returns: 79 | bytes: raw data range stored in location 80 | """ 81 | return super().get_range(_as_path(location), start, length) 82 | 83 | def put(self, location: PathLike, bytes: BytesLike) -> None: 84 | """Save the provided bytes to the specified location. 85 | 86 | Args: 87 | location (PathLike): path / key to storage location 88 | bytes (BytesLike): data to be written to location 89 | """ 90 | return super().put(_as_path(location), _as_bytes(bytes)) 91 | 92 | def delete(self, location: PathLike) -> None: 93 | """Delete the object at the specified location. 94 | 95 | Args: 96 | location (PathLike): path / key to storage location 97 | """ 98 | return super().delete(_as_path(location)) 99 | 100 | def list(self, prefix: Optional[PathLike] = None) -> List[ObjectMeta]: 101 | """List all the objects with the given prefix. 102 | 103 | Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix 104 | of `foo/bar/x` but not of `foo/bar_baz/x`. 105 | 106 | Args: 107 | prefix (PathLike | None, optional): path prefix to filter limit list results. Defaults to None. 108 | 109 | Returns: 110 | list[ObjectMeta]: ObjectMeta for all objects under the listed path 111 | """ 112 | prefix_ = _as_path(prefix) if prefix else None 113 | return super().list(prefix_) 114 | 115 | def list_with_delimiter(self, prefix: Optional[PathLike] = None) -> ListResult: 116 | """List objects with the given prefix and an implementation specific 117 | delimiter. Returns common prefixes (directories) in addition to object 118 | metadata. 119 | 120 | Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix 121 | of `foo/bar/x` but not of `foo/bar_baz/x`. 122 | 123 | Args: 124 | prefix (PathLike | None, optional): path prefix to filter limit list results. Defaults to None. 125 | 126 | Returns: 127 | list[ObjectMeta]: ObjectMeta for all objects under the listed path 128 | """ 129 | prefix_ = _as_path(prefix) if prefix else None 130 | return super().list_with_delimiter(prefix_) 131 | 132 | def copy(self, src: PathLike, dst: PathLike) -> None: 133 | """Copy an object from one path to another in the same object store. 134 | 135 | If there exists an object at the destination, it will be overwritten. 136 | 137 | Args: 138 | src (PathLike): source path 139 | dst (PathLike): destination path 140 | """ 141 | return super().copy(_as_path(src), _as_path(dst)) 142 | 143 | def copy_if_not_exists(self, src: PathLike, dst: PathLike) -> None: 144 | """Copy an object from one path to another, only if destination is empty. 145 | 146 | Will return an error if the destination already has an object. 147 | 148 | Args: 149 | src (PathLike): source path 150 | dst (PathLike): destination path 151 | """ 152 | return super().copy_if_not_exists(_as_path(src), _as_path(dst)) 153 | 154 | def rename(self, src: PathLike, dst: PathLike) -> None: 155 | """Move an object from one path to another in the same object store. 156 | 157 | By default, this is implemented as a copy and then delete source. It may not 158 | check when deleting source that it was the same object that was originally copied. 159 | 160 | If there exists an object at the destination, it will be overwritten. 161 | 162 | Args: 163 | src (PathLike): source path 164 | dst (PathLike): destination path 165 | """ 166 | return super().rename(_as_path(src), _as_path(dst)) 167 | 168 | def rename_if_not_exists(self, src: PathLike, dst: PathLike) -> None: 169 | """Move an object from one path to another in the same object store. 170 | 171 | Will return an error if the destination already has an object. 172 | 173 | Args: 174 | src (PathLike): source path 175 | dst (PathLike): destination path 176 | """ 177 | return super().rename_if_not_exists(_as_path(src), _as_path(dst)) 178 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # object-store-python 2 | 3 | [![CI][ci-img]][ci-link] 4 | [![code style: black][black-img]][black-link] 5 | ![PyPI](https://img.shields.io/pypi/v/object-store-python) 6 | [![PyPI - Downloads][pypi-img]][pypi-link] 7 | 8 | Python bindings and integrations for the excellent [`object_store`][object-store] crate. 9 | The main idea is to provide a common interface to various storage backends including the 10 | objects stores from most major cloud providers. The APIs are very focussed and taylored 11 | towards modern cloud native applications by hiding away many features (and complexities) 12 | encountered in full fledges file systems. 13 | 14 | Among the included backend are: 15 | 16 | - Amazon S3 and S3 compliant APIs 17 | - Google Cloud Storage Buckets 18 | - Azure Blob Gen1 and Gen2 accounts (including ADLS Gen2) 19 | - local storage 20 | - in-memory store 21 | 22 | ## Installation 23 | 24 | The `object-store-python` package is available on PyPI and can be installed via 25 | 26 | ```sh 27 | poetry add object-store-python 28 | ``` 29 | 30 | or using pip 31 | 32 | ```sh 33 | pip install object-store-python 34 | ``` 35 | 36 | ## Usage 37 | 38 | The main [`ObjectStore`](#object-store-python) API mirrors the native [`object_store`][object-store] 39 | implementation, with some slight adjustments for ease of use in python programs. 40 | 41 | ### `ObjectStore` api 42 | 43 | ```py 44 | from object_store import ObjectStore, ObjectMeta, Path 45 | 46 | # we use an in-memory store for demonstration purposes. 47 | # data will not be persisted and is not shared across store instances 48 | store = ObjectStore("memory://") 49 | 50 | store.put(Path("data"), b"some data") 51 | 52 | data = store.get("data") 53 | assert data == b"some data" 54 | 55 | blobs = store.list() 56 | 57 | meta = store.head("data") 58 | 59 | range = store.get_range("data", start=0, length=4) 60 | assert range == b"some" 61 | 62 | store.copy("data", "copied") 63 | copied = store.get("copied") 64 | assert copied == data 65 | ``` 66 | 67 | #### Async api 68 | 69 | ```py 70 | from object_store import ObjectStore, ObjectMeta, Path 71 | 72 | # we use an in-memory store for demonstration purposes. 73 | # data will not be persisted and is not shared across store instances 74 | store = ObjectStore("memory://") 75 | 76 | path = Path("data") 77 | await store.put_async(path, b"some data") 78 | 79 | data = await store.get_async(path) 80 | assert data == b"some data" 81 | 82 | blobs = await store.list_async() 83 | 84 | meta = await store.head_async(path) 85 | 86 | range = await store.get_range_async(path, start=0, length=4) 87 | assert range == b"some" 88 | 89 | await store.copy_async(Path("data"), Path("copied")) 90 | copied = await store.get_async(Path("copied")) 91 | assert copied == data 92 | ``` 93 | 94 | ### Configuration 95 | 96 | As much as possible we aim to make access to various storage backends dependent 97 | only on runtime configuration. The kind of service is always derived from the 98 | url used to specifiy the storage location. Some basic configuration can also be 99 | derived from the url string, dependent on the chosen url format. 100 | 101 | ```py 102 | from object_store import ObjectStore 103 | 104 | storage_options = { 105 | "azure_storage_account_name": "", 106 | "azure_client_id": "", 107 | "azure_client_secret": "", 108 | "azure_tenant_id": "" 109 | } 110 | 111 | store = ObjectStore("az://", storage_options) 112 | ``` 113 | 114 | We can provide the same configuration via the environment. 115 | 116 | ```py 117 | import os 118 | from object_store import ObjectStore 119 | 120 | os.environ["AZURE_STORAGE_ACCOUNT_NAME"] = "" 121 | os.environ["AZURE_CLIENT_ID"] = "" 122 | os.environ["AZURE_CLIENT_SECRET"] = "" 123 | os.environ["AZURE_TENANT_ID"] = "" 124 | 125 | store = ObjectStore("az://") 126 | ``` 127 | 128 | #### Azure 129 | 130 | The recommended url format is `az:///` and Azure always requieres 131 | `azure_storage_account_name` to be configured. 132 | 133 | - [shared key][azure-key] 134 | - `azure_storage_account_key` 135 | - [service principal][azure-ad] 136 | - `azure_client_id` 137 | - `azure_client_secret` 138 | - `azure_tenant_id` 139 | - [shared access signature][azure-sas] 140 | - `azure_storage_sas_key` (as provided by StorageExplorer) 141 | - bearer token 142 | - `azure_storage_token` 143 | - [managed identity][azure-managed] 144 | - if using user assigned identity one of `azure_client_id`, `azure_object_id`, `azure_msi_resource_id` 145 | - if no other credential can be created, managed identity will be tried 146 | - [workload identity][azure-workload] 147 | - `azure_client_id` 148 | - `azure_tenant_id` 149 | - `azure_federated_token_file` 150 | 151 | #### S3 152 | 153 | The recommended url format is `s3:///` S3 storage always requires a 154 | region to be specified via one of `aws_region` or `aws_default_region`. 155 | 156 | - [access key][aws-key] 157 | - `aws_access_key_id` 158 | - `aws_secret_access_key` 159 | - [session token][aws-sts] 160 | - `aws_session_token` 161 | - [imds instance metadata][aws-imds] 162 | - `aws_metadata_endpoint` 163 | - [profile][aws-profile] 164 | - `aws_profile` 165 | 166 | AWS supports [virtual hosting of buckets][aws-virtual], which can be configured by setting 167 | `aws_virtual_hosted_style_request` to "true". 168 | 169 | When an alternative implementation or a mocked service like localstack is used, the service 170 | endpoint needs to be explicitly specified via `aws_endpoint`. 171 | 172 | #### GCS 173 | 174 | The recommended url format is `gs:///`. 175 | 176 | - service account 177 | - `google_service_account` 178 | 179 | ### with `pyarrow` 180 | 181 | ```py 182 | from pathlib import Path 183 | 184 | import numpy as np 185 | import pyarrow as pa 186 | import pyarrow.fs as fs 187 | import pyarrow.dataset as ds 188 | import pyarrow.parquet as pq 189 | 190 | from object_store import ArrowFileSystemHandler 191 | 192 | table = pa.table({"a": range(10), "b": np.random.randn(10), "c": [1, 2] * 5}) 193 | 194 | base = Path.cwd() 195 | store = fs.PyFileSystem(ArrowFileSystemHandler(str(base.absolute()))) 196 | 197 | pq.write_table(table.slice(0, 5), "data/data1.parquet", filesystem=store) 198 | pq.write_table(table.slice(5, 10), "data/data2.parquet", filesystem=store) 199 | 200 | dataset = ds.dataset("data", format="parquet", filesystem=store) 201 | ``` 202 | 203 | ## Development 204 | 205 | ### Prerequisites 206 | 207 | - [poetry](https://python-poetry.org/docs/) 208 | - [Rust toolchain](https://www.rust-lang.org/tools/install) 209 | - [just](https://github.com/casey/just#readme) 210 | 211 | ### Running tests 212 | 213 | If you do not have [`just`](https://github.com/casey/just#readme) installed and do not wish to install it, 214 | have a look at the [`justfile`](https://github.com/roeap/object-store-python/blob/main/justfile) to see the raw commands. 215 | 216 | To set up the development environment, and install a dev version of the native package just run: 217 | 218 | ```sh 219 | just init 220 | ``` 221 | 222 | This will also configure [`pre-commit`](https://pre-commit.com/) hooks in the repository. 223 | 224 | To run the rust as well as python tests: 225 | 226 | ```sh 227 | just test 228 | ``` 229 | 230 | [object-store]: https://crates.io/crates/object_store 231 | [pypi-img]: https://img.shields.io/pypi/dm/object-store-python 232 | [pypi-link]: https://pypi.org/project/object-store-python/ 233 | [ci-img]: https://github.com/roeap/object-store-python/actions/workflows/ci.yaml/badge.svg 234 | [ci-link]: https://github.com/roeap/object-store-python/actions/workflows/ci.yaml 235 | [black-img]: https://img.shields.io/badge/code%20style-black-000000.svg 236 | [black-link]: https://github.com/psf/black 237 | [aws-virtual]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html 238 | [azure-managed]: https://learn.microsoft.com/en-gb/azure/app-service/overview-managed-identity 239 | [azure-sas]: https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview 240 | [azure-ad]: https://learn.microsoft.com/en-us/azure/storage/blobs/authorize-access-azure-active-directory 241 | [azure-key]: https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key 242 | [azure-workload]: https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview 243 | [aws-imds]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html 244 | [aws-profile]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html 245 | [aws-sts]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html 246 | [aws-key]: https://docs.aws.amazon.com/accounts/latest/reference/credentials-access-keys-best-practices.html 247 | -------------------------------------------------------------------------------- /object-store-internal/src/builder.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::sync::Arc; 3 | 4 | use object_store::aws::{AmazonS3, AmazonS3Builder}; 5 | use object_store::azure::{MicrosoftAzure, MicrosoftAzureBuilder}; 6 | use object_store::gcp::{GoogleCloudStorage, GoogleCloudStorageBuilder}; 7 | use object_store::local::LocalFileSystem; 8 | use object_store::memory::InMemory; 9 | use object_store::path::Path; 10 | use object_store::prefix::PrefixStore; 11 | use object_store::{ 12 | ClientOptions, DynObjectStore, Error as ObjectStoreError, Result as ObjectStoreResult, 13 | RetryConfig, 14 | }; 15 | use url::Url; 16 | 17 | enum ObjectStoreKind { 18 | Local, 19 | InMemory, 20 | S3, 21 | Google, 22 | Azure, 23 | } 24 | 25 | impl ObjectStoreKind { 26 | pub fn parse_url(url: &Url) -> ObjectStoreResult { 27 | match url.scheme() { 28 | "file" => Ok(ObjectStoreKind::Local), 29 | "memory" => Ok(ObjectStoreKind::InMemory), 30 | "az" | "abfs" | "abfss" | "azure" | "wasb" | "adl" => Ok(ObjectStoreKind::Azure), 31 | "s3" | "s3a" => Ok(ObjectStoreKind::S3), 32 | "gs" => Ok(ObjectStoreKind::Google), 33 | "https" => { 34 | let host = url.host_str().unwrap_or_default(); 35 | if host.contains("amazonaws.com") { 36 | Ok(ObjectStoreKind::S3) 37 | } else if host.contains("dfs.core.windows.net") 38 | || host.contains("blob.core.windows.net") 39 | { 40 | Ok(ObjectStoreKind::Azure) 41 | } else { 42 | Err(ObjectStoreError::NotImplemented) 43 | } 44 | } 45 | _ => Err(ObjectStoreError::NotImplemented), 46 | } 47 | } 48 | } 49 | 50 | enum ObjectStoreImpl { 51 | Local(LocalFileSystem), 52 | InMemory(InMemory), 53 | Azure(MicrosoftAzure), 54 | S3(AmazonS3), 55 | Gcp(GoogleCloudStorage), 56 | } 57 | 58 | impl ObjectStoreImpl { 59 | pub fn into_prefix(self, prefix: Path) -> Arc { 60 | match self { 61 | ObjectStoreImpl::Local(store) => Arc::new(PrefixStore::new(store, prefix)), 62 | ObjectStoreImpl::InMemory(store) => Arc::new(PrefixStore::new(store, prefix)), 63 | ObjectStoreImpl::Azure(store) => Arc::new(PrefixStore::new(store, prefix)), 64 | ObjectStoreImpl::S3(store) => Arc::new(PrefixStore::new(store, prefix)), 65 | ObjectStoreImpl::Gcp(store) => Arc::new(PrefixStore::new(store, prefix)), 66 | } 67 | } 68 | 69 | pub fn into_store(self) -> Arc { 70 | match self { 71 | ObjectStoreImpl::Local(store) => Arc::new(store), 72 | ObjectStoreImpl::InMemory(store) => Arc::new(store), 73 | ObjectStoreImpl::Azure(store) => Arc::new(store), 74 | ObjectStoreImpl::S3(store) => Arc::new(store), 75 | ObjectStoreImpl::Gcp(store) => Arc::new(store), 76 | } 77 | } 78 | } 79 | 80 | #[derive(Debug, Clone)] 81 | pub struct ObjectStoreBuilder { 82 | url: String, 83 | prefix: Option, 84 | path_as_prefix: bool, 85 | options: HashMap, 86 | client_options: Option, 87 | retry_config: Option, 88 | } 89 | 90 | impl ObjectStoreBuilder { 91 | pub fn new(url: impl Into) -> Self { 92 | Self { 93 | url: url.into(), 94 | prefix: None, 95 | path_as_prefix: false, 96 | options: Default::default(), 97 | client_options: None, 98 | retry_config: None, 99 | } 100 | } 101 | 102 | pub fn with_options, impl Into)>>( 103 | mut self, 104 | options: I, 105 | ) -> Self { 106 | self.options 107 | .extend(options.into_iter().map(|(k, v)| (k.into(), v.into()))); 108 | self 109 | } 110 | 111 | pub fn with_option(mut self, key: impl Into, value: impl Into) -> Self { 112 | self.options.insert(key.into(), value.into()); 113 | self 114 | } 115 | 116 | pub fn with_prefix(mut self, prefix: impl Into) -> Self { 117 | self.prefix = Some(prefix.into()); 118 | self 119 | } 120 | 121 | pub fn with_path_as_prefix(mut self, path_as_prefix: bool) -> Self { 122 | self.path_as_prefix = path_as_prefix; 123 | self 124 | } 125 | 126 | pub fn with_client_options(mut self, options: ClientOptions) -> Self { 127 | self.client_options = Some(options); 128 | self 129 | } 130 | 131 | pub fn with_retry_config(mut self, retry_config: RetryConfig) -> Self { 132 | self.retry_config = Some(retry_config); 133 | self 134 | } 135 | 136 | pub fn build(mut self) -> ObjectStoreResult> { 137 | let maybe_url = Url::parse(&self.url); 138 | let url = 139 | match maybe_url { 140 | Ok(url) => Ok(url), 141 | Err(url::ParseError::RelativeUrlWithoutBase) => { 142 | let abs_path = std::fs::canonicalize(std::path::PathBuf::from(&self.url)) 143 | .map_err(|err| ObjectStoreError::Generic { 144 | store: "Generic", 145 | source: Box::new(err), 146 | })?; 147 | Url::parse(&format!("file://{}", abs_path.to_str().unwrap())).map_err(|err| { 148 | ObjectStoreError::Generic { 149 | store: "Generic", 150 | source: Box::new(err), 151 | } 152 | }) 153 | } 154 | Err(err) => Err(ObjectStoreError::Generic { 155 | store: "Generic", 156 | source: Box::new(err), 157 | }), 158 | }?; 159 | let root_store = match ObjectStoreKind::parse_url(&url)? { 160 | ObjectStoreKind::Local => ObjectStoreImpl::Local(LocalFileSystem::new()), 161 | ObjectStoreKind::InMemory => ObjectStoreImpl::InMemory(InMemory::new()), 162 | ObjectStoreKind::Azure => { 163 | let mut store_builder = MicrosoftAzureBuilder::new().with_url(url.clone()); 164 | 165 | for (key, value) in self.options.iter() { 166 | store_builder = store_builder.with_config(key.parse()?, value); 167 | } 168 | 169 | let store = store_builder 170 | .with_client_options(self.client_options.clone().unwrap_or_default()) 171 | .with_retry(self.retry_config.clone().unwrap_or_default()) 172 | .build() 173 | .or_else(|_| { 174 | let mut store_builder = 175 | MicrosoftAzureBuilder::from_env().with_url(url.clone()); 176 | 177 | for (key, value) in self.options.iter() { 178 | store_builder = store_builder.with_config(key.parse()?, value); 179 | } 180 | 181 | store_builder 182 | .with_client_options(self.client_options.clone().unwrap_or_default()) 183 | .with_retry(self.retry_config.clone().unwrap_or_default()) 184 | .build() 185 | })?; 186 | ObjectStoreImpl::Azure(store) 187 | } 188 | ObjectStoreKind::S3 => { 189 | let mut store_builder = AmazonS3Builder::new().with_url(url.clone()); 190 | 191 | for (key, value) in self.options.iter() { 192 | store_builder = store_builder.with_config(key.parse()?, value); 193 | } 194 | 195 | let store = store_builder 196 | .with_client_options(self.client_options.clone().unwrap_or_default()) 197 | .with_retry(self.retry_config.clone().unwrap_or_default()) 198 | .build() 199 | .or_else(|_| { 200 | let mut store_builder = AmazonS3Builder::from_env().with_url(url.clone()); 201 | 202 | for (key, value) in self.options.iter() { 203 | store_builder = store_builder.with_config(key.parse()?, value); 204 | } 205 | 206 | store_builder 207 | .with_client_options(self.client_options.unwrap_or_default()) 208 | .with_retry(self.retry_config.unwrap_or_default()) 209 | .build() 210 | })?; 211 | ObjectStoreImpl::S3(store) 212 | } 213 | ObjectStoreKind::Google => { 214 | let mut store_builder = GoogleCloudStorageBuilder::new().with_url(url.clone()); 215 | 216 | for (key, value) in self.options.iter() { 217 | store_builder = store_builder.with_config(key.parse()?, value); 218 | } 219 | 220 | let store = store_builder 221 | .with_client_options(self.client_options.clone().unwrap_or_default()) 222 | .with_retry(self.retry_config.clone().unwrap_or_default()) 223 | .build() 224 | .or_else(|_| { 225 | let mut store_builder = 226 | GoogleCloudStorageBuilder::from_env().with_url(url.clone()); 227 | 228 | for (key, value) in self.options.iter() { 229 | store_builder = store_builder.with_config(key.parse()?, value); 230 | } 231 | 232 | store_builder 233 | .with_client_options(self.client_options.unwrap_or_default()) 234 | .with_retry(self.retry_config.unwrap_or_default()) 235 | .build() 236 | })?; 237 | ObjectStoreImpl::Gcp(store) 238 | } 239 | }; 240 | 241 | if self.path_as_prefix && !url.path().is_empty() && self.prefix.is_none() { 242 | self.prefix = Some(Path::from(url.path())) 243 | } 244 | 245 | if let Some(prefix) = self.prefix { 246 | Ok(root_store.into_prefix(prefix)) 247 | } else { 248 | Ok(root_store.into_store()) 249 | } 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2022 Robert Pack 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /object-store/python/object_store/_internal.pyi: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | if TYPE_CHECKING: 6 | import pyarrow.fs as fs 7 | 8 | class Path: 9 | def __init__(self, raw: str | list[str]) -> None: ... 10 | def child(self, part: str) -> Path: ... 11 | 12 | class ObjectMeta: 13 | """The metadata that describes an object.""" 14 | 15 | @property 16 | def size(self) -> int: 17 | """The size in bytes of the object""" 18 | 19 | @property 20 | def location(self) -> Path: 21 | """The full path to the object""" 22 | 23 | @property 24 | def last_modified(self) -> int: 25 | """The last modified time""" 26 | 27 | class ListResult: 28 | """Result of a list call that includes objects and prefixes (directories)""" 29 | 30 | @property 31 | def common_prefixes(self) -> list[Path]: 32 | """Prefixes that are common (like directories)""" 33 | 34 | @property 35 | def objects(self) -> list[ObjectMeta]: 36 | """Object metadata for the listing""" 37 | 38 | class ClientOptions: 39 | """HTTP client configuration for remote object stores""" 40 | 41 | @property 42 | def user_agent(self) -> str | None: 43 | """Sets the User-Agent header to be used by this client 44 | 45 | Default is based on the version of this crate 46 | """ 47 | 48 | @property 49 | def default_content_type(self) -> str | None: 50 | """Set the default CONTENT_TYPE for uploads""" 51 | 52 | @property 53 | def proxy_url(self) -> str | None: 54 | """Set an HTTP proxy to use for requests""" 55 | 56 | @property 57 | def allow_http(self) -> bool: 58 | """Sets what protocol is allowed. 59 | 60 | If `allow_http` is : 61 | * false (default): Only HTTPS ise allowed 62 | * true: HTTP and HTTPS are allowed 63 | """ 64 | 65 | @property 66 | def allow_insecure(self) -> bool: 67 | """Allows connections to invalid SSL certificates 68 | * false (default): Only valid HTTPS certificates are allowed 69 | * true: All HTTPS certificates are allowed 70 | 71 | # Warning 72 | 73 | You should think very carefully before using this method. If 74 | invalid certificates are trusted, *any* certificate for *any* site 75 | will be trusted for use. This includes expired certificates. This 76 | introduces significant vulnerabilities, and should only be used 77 | as a last resort or for testing. 78 | """ 79 | 80 | @property 81 | def timeout(self) -> int: 82 | """Set a request timeout (seconds) 83 | 84 | The timeout is applied from when the request starts connecting until the 85 | response body has finished 86 | """ 87 | 88 | @property 89 | def connect_timeout(self) -> int: 90 | """Set a timeout (seconds) for only the connect phase of a Client""" 91 | 92 | @property 93 | def pool_idle_timeout(self) -> int: 94 | """Set the pool max idle timeout (seconds) 95 | 96 | This is the length of time an idle connection will be kept alive 97 | 98 | Default is 90 seconds 99 | """ 100 | 101 | @property 102 | def pool_max_idle_per_host(self) -> int: 103 | """Set the maximum number of idle connections per host 104 | 105 | Default is no limit""" 106 | 107 | @property 108 | def http2_keep_alive_interval(self) -> int: 109 | """Sets an interval for HTTP2 Ping frames should be sent to keep a connection alive. 110 | 111 | Default is disabled 112 | """ 113 | 114 | @property 115 | def http2_keep_alive_timeout(self) -> int: 116 | """Sets a timeout for receiving an acknowledgement of the keep-alive ping. 117 | 118 | If the ping is not acknowledged within the timeout, the connection will be closed. 119 | Does nothing if http2_keep_alive_interval is disabled. 120 | 121 | Default is disabled 122 | """ 123 | 124 | @property 125 | def http2_keep_alive_while_idle(self) -> bool: 126 | """Enable HTTP2 keep alive pings for idle connections 127 | 128 | If disabled, keep-alive pings are only sent while there are open request/response 129 | streams. If enabled, pings are also sent when no streams are active 130 | 131 | Default is disabled 132 | """ 133 | 134 | @property 135 | def http1_only(self) -> bool: 136 | """Only use http1 connections""" 137 | 138 | @property 139 | def http2_only(self) -> bool: 140 | """Only use http2 connections""" 141 | 142 | class ObjectStore: 143 | """A uniform API for interacting with object storage services and local files.""" 144 | 145 | def __init__( 146 | self, root: str, options: dict[str, str] | None = None, client_options: ClientOptions | None = None 147 | ) -> None: ... 148 | def get(self, location: Path) -> bytes: 149 | """Return the bytes that are stored at the specified location.""" 150 | 151 | async def get_async(self, location: Path) -> bytes: 152 | """Return the bytes that are stored at the specified location.""" 153 | 154 | def get_range(self, location: Path, start: int, length: int) -> bytes: 155 | """Return the bytes that are stored at the specified location in the given byte range.""" 156 | 157 | async def get_range_async(self, location: Path, start: int, length: int) -> bytes: 158 | """Return the bytes that are stored at the specified location in the given byte range.""" 159 | 160 | def put(self, location: Path, bytes: bytes) -> None: 161 | """Save the provided bytes to the specified location.""" 162 | 163 | async def put_async(self, location: Path, bytes: bytes) -> None: 164 | """Save the provided bytes to the specified location.""" 165 | 166 | def list(self, prefix: Path | None) -> list[ObjectMeta]: 167 | """List all the objects with the given prefix. 168 | 169 | Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix 170 | of `foo/bar/x` but not of `foo/bar_baz/x`. 171 | """ 172 | 173 | async def list_async(self, prefix: Path | None) -> list[ObjectMeta]: 174 | """List all the objects with the given prefix. 175 | 176 | Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix 177 | of `foo/bar/x` but not of `foo/bar_baz/x`. 178 | """ 179 | 180 | def head(self, location: Path) -> ObjectMeta: 181 | """Return the metadata for the specified location""" 182 | 183 | async def head_async(self, location: Path) -> ObjectMeta: 184 | """Return the metadata for the specified location""" 185 | 186 | def list_with_delimiter(self, prefix: Path | None) -> ListResult: 187 | """List objects with the given prefix and an implementation specific 188 | delimiter. Returns common prefixes (directories) in addition to object 189 | metadata. 190 | 191 | Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix 192 | of `foo/bar/x` but not of `foo/bar_baz/x`. 193 | """ 194 | 195 | async def list_with_delimiter_async(self, prefix: Path | None) -> ListResult: 196 | """List objects with the given prefix and an implementation specific 197 | delimiter. Returns common prefixes (directories) in addition to object 198 | metadata. 199 | 200 | Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix 201 | of `foo/bar/x` but not of `foo/bar_baz/x`. 202 | """ 203 | 204 | def delete(self, location: Path) -> None: 205 | """Delete the object at the specified location.""" 206 | 207 | async def delete_async(self, location: Path) -> None: 208 | """Delete the object at the specified location.""" 209 | 210 | def copy(self, src: Path, dst: Path) -> None: 211 | """Copy an object from one path to another in the same object store. 212 | 213 | If there exists an object at the destination, it will be overwritten. 214 | """ 215 | 216 | async def copy_async(self, src: Path, dst: Path) -> None: 217 | """Copy an object from one path to another in the same object store. 218 | 219 | If there exists an object at the destination, it will be overwritten. 220 | """ 221 | 222 | def copy_if_not_exists(self, src: Path, dst: Path) -> None: 223 | """Copy an object from one path to another, only if destination is empty. 224 | 225 | Will return an error if the destination already has an object. 226 | """ 227 | 228 | async def copy_if_not_exists_async(self, src: Path, dst: Path) -> None: 229 | """Copy an object from one path to another, only if destination is empty. 230 | 231 | Will return an error if the destination already has an object. 232 | """ 233 | 234 | def rename(self, src: Path, dst: Path) -> None: 235 | """Move an object from one path to another in the same object store. 236 | 237 | By default, this is implemented as a copy and then delete source. It may not 238 | check when deleting source that it was the same object that was originally copied. 239 | 240 | If there exists an object at the destination, it will be overwritten. 241 | """ 242 | 243 | async def rename_async(self, src: Path, dst: Path) -> None: 244 | """Move an object from one path to another in the same object store. 245 | 246 | By default, this is implemented as a copy and then delete source. It may not 247 | check when deleting source that it was the same object that was originally copied. 248 | 249 | If there exists an object at the destination, it will be overwritten. 250 | """ 251 | 252 | def rename_if_not_exists(self, src: Path, dst: Path) -> None: 253 | """Move an object from one path to another in the same object store. 254 | 255 | Will return an error if the destination already has an object. 256 | """ 257 | 258 | async def rename_if_not_exists_async(self, src: Path, dst: Path) -> None: 259 | """Move an object from one path to another in the same object store. 260 | 261 | Will return an error if the destination already has an object. 262 | """ 263 | 264 | class ObjectInputFile: 265 | @property 266 | def closed(self) -> bool: ... 267 | @property 268 | def mode(self) -> str: ... 269 | def isatty(self) -> bool: ... 270 | def readable(self) -> bool: ... 271 | def seekable(self) -> bool: ... 272 | def tell(self) -> int: ... 273 | def size(self) -> int: ... 274 | def seek(self, position: int, whence: int) -> int: ... 275 | def read(self, nbytes: int) -> bytes: ... 276 | 277 | class ObjectOutputStream: 278 | @property 279 | def closed(self) -> bool: ... 280 | @property 281 | def mode(self) -> str: ... 282 | def isatty(self) -> bool: ... 283 | def readable(self) -> bool: ... 284 | def seekable(self) -> bool: ... 285 | def writable(self) -> bool: ... 286 | def tell(self) -> int: ... 287 | def size(self) -> int: ... 288 | def seek(self, position: int, whence: int) -> int: ... 289 | def read(self, nbytes: int) -> bytes: ... 290 | def write(self, data: bytes) -> int: ... 291 | 292 | class ArrowFileSystemHandler: 293 | """Implementation of pyarrow.fs.FileSystemHandler for use with pyarrow.fs.PyFileSystem""" 294 | 295 | def __init__( 296 | self, root: str, options: dict[str, str] | None = None, client_options: ClientOptions | None = None 297 | ) -> None: ... 298 | def copy_file(self, src: str, dst: str) -> None: 299 | """Copy a file. 300 | 301 | If the destination exists and is a directory, an error is returned. Otherwise, it is replaced. 302 | """ 303 | 304 | def create_dir(self, path: str, *, recursive: bool = True) -> None: 305 | """Create a directory and subdirectories. 306 | 307 | This function succeeds if the directory already exists. 308 | """ 309 | 310 | def delete_dir(self, path: str) -> None: 311 | """Delete a directory and its contents, recursively.""" 312 | 313 | def delete_file(self, path: str) -> None: 314 | """Delete a file.""" 315 | 316 | def equals(self, other) -> bool: ... 317 | def delete_dir_contents(self, path: str, *, accept_root_dir: bool = False, missing_dir_ok: bool = False) -> None: 318 | """Delete a directory's contents, recursively. 319 | 320 | Like delete_dir, but doesn't delete the directory itself. 321 | """ 322 | 323 | def get_file_info(self, paths: list[str]) -> list[fs.FileInfo]: 324 | """Get info for the given files. 325 | 326 | A non-existing or unreachable file returns a FileStat object and has a FileType of value NotFound. 327 | An exception indicates a truly exceptional condition (low-level I/O error, etc.). 328 | """ 329 | 330 | def get_file_info_selector( 331 | self, base_dir: str, allow_not_found: bool = False, recursive: bool = False 332 | ) -> list[fs.FileInfo]: 333 | """Get info for the given files. 334 | 335 | A non-existing or unreachable file returns a FileStat object and has a FileType of value NotFound. 336 | An exception indicates a truly exceptional condition (low-level I/O error, etc.). 337 | """ 338 | 339 | def move_file(self, src: str, dest: str) -> None: 340 | """Move / rename a file or directory. 341 | 342 | If the destination exists: - if it is a non-empty directory, an error is returned - otherwise, 343 | if it has the same type as the source, it is replaced - otherwise, behavior is 344 | unspecified (implementation-dependent). 345 | """ 346 | 347 | def normalize_path(self, path: str) -> str: 348 | """Normalize filesystem path.""" 349 | 350 | def open_input_file(self, path: str) -> ObjectInputFile: 351 | """Open an input file for random access reading.""" 352 | 353 | def open_output_stream(self, path: str, metadata: dict[str, str] | None = None) -> ObjectOutputStream: 354 | """Open an output stream for sequential writing.""" 355 | -------------------------------------------------------------------------------- /object-store-internal/src/file.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::sync::Arc; 3 | 4 | use crate::builder::ObjectStoreBuilder; 5 | use crate::utils::{delete_dir, walk_tree}; 6 | use crate::{ObjectStoreError, PyClientOptions}; 7 | 8 | use object_store::path::Path; 9 | use object_store::{DynObjectStore, Error as InnerObjectStoreError, ListResult, MultipartUpload}; 10 | use pyo3::exceptions::{PyNotImplementedError, PyValueError}; 11 | use pyo3::prelude::*; 12 | use pyo3::types::{IntoPyDict, PyBytes}; 13 | use tokio::runtime::Runtime; 14 | 15 | #[pyclass(subclass, weakref)] 16 | #[derive(Debug, Clone)] 17 | pub struct ArrowFileSystemHandler { 18 | inner: Arc, 19 | rt: Arc, 20 | root_url: String, 21 | options: Option>, 22 | } 23 | 24 | #[pymethods] 25 | impl ArrowFileSystemHandler { 26 | #[new] 27 | #[pyo3(signature = (root, options = None, client_options = None))] 28 | fn new( 29 | root: String, 30 | options: Option>, 31 | client_options: Option, 32 | ) -> PyResult { 33 | let client_options = client_options.unwrap_or_default(); 34 | let inner = ObjectStoreBuilder::new(root.clone()) 35 | .with_path_as_prefix(true) 36 | .with_options(options.clone().unwrap_or_default()) 37 | .with_client_options(client_options.client_options()?) 38 | .with_retry_config(client_options.retry_config()?) 39 | .build() 40 | .map_err(ObjectStoreError::from)?; 41 | Ok(Self { 42 | root_url: root, 43 | inner, 44 | rt: Arc::new(Runtime::new()?), 45 | options, 46 | }) 47 | } 48 | 49 | fn get_type_name(&self) -> String { 50 | "object-store".into() 51 | } 52 | 53 | fn normalize_path(&self, path: String) -> PyResult { 54 | let path = Path::parse(path).map_err(ObjectStoreError::from)?; 55 | Ok(path.to_string()) 56 | } 57 | 58 | fn copy_file(&self, src: String, dest: String) -> PyResult<()> { 59 | let from_path = Path::from(src); 60 | let to_path = Path::from(dest); 61 | self.rt 62 | .block_on(self.inner.copy(&from_path, &to_path)) 63 | .map_err(ObjectStoreError::from)?; 64 | Ok(()) 65 | } 66 | 67 | fn create_dir(&self, _path: String, _recursive: bool) -> PyResult<()> { 68 | // TODO creating a dir should be a no-op with object_store, right? 69 | Ok(()) 70 | } 71 | 72 | fn delete_dir(&self, path: String) -> PyResult<()> { 73 | let path = Path::from(path); 74 | self.rt 75 | .block_on(delete_dir(self.inner.as_ref(), &path)) 76 | .map_err(ObjectStoreError::from)?; 77 | Ok(()) 78 | } 79 | 80 | fn delete_file(&self, path: String) -> PyResult<()> { 81 | let path = Path::from(path); 82 | self.rt 83 | .block_on(self.inner.delete(&path)) 84 | .map_err(ObjectStoreError::from)?; 85 | Ok(()) 86 | } 87 | 88 | fn equals(&self, other: &ArrowFileSystemHandler) -> PyResult { 89 | Ok(format!("{:?}", self) == format!("{:?}", other)) 90 | } 91 | 92 | fn get_file_info<'py>( 93 | &self, 94 | paths: Vec, 95 | py: Python<'py>, 96 | ) -> PyResult>> { 97 | let fs = PyModule::import_bound(py, "pyarrow.fs")?; 98 | let file_types = fs.getattr("FileType")?; 99 | 100 | let to_file_info = |loc: String, type_: Bound<'_, PyAny>, kwargs: HashMap<&str, i64>| { 101 | fs.call_method( 102 | "FileInfo", 103 | (loc, type_), 104 | Some(&kwargs.into_py_dict_bound(py)), 105 | ) 106 | }; 107 | 108 | let mut infos = Vec::new(); 109 | for file_path in paths { 110 | let path = Path::from(file_path); 111 | let listed = self 112 | .rt 113 | .block_on(self.inner.list_with_delimiter(Some(&path))) 114 | .map_err(ObjectStoreError::from)?; 115 | 116 | // TODO is there a better way to figure out if we are in a directory? 117 | if listed.objects.is_empty() && listed.common_prefixes.is_empty() { 118 | let maybe_meta = self.rt.block_on(self.inner.head(&path)); 119 | match maybe_meta { 120 | Ok(meta) => { 121 | let kwargs = HashMap::from([ 122 | ("size", meta.size as i64), 123 | ( 124 | "mtime_ns", 125 | meta.last_modified.timestamp_nanos_opt().unwrap(), 126 | ), 127 | ]); 128 | infos.push(to_file_info( 129 | meta.location.to_string(), 130 | file_types.getattr("File")?, 131 | kwargs, 132 | )?); 133 | } 134 | Err(object_store::Error::NotFound { .. }) => { 135 | infos.push(to_file_info( 136 | path.to_string(), 137 | file_types.getattr("NotFound")?, 138 | HashMap::new(), 139 | )?); 140 | } 141 | Err(err) => { 142 | return Err(ObjectStoreError::from(err).into()); 143 | } 144 | } 145 | } else { 146 | infos.push(to_file_info( 147 | path.to_string(), 148 | file_types.getattr("Directory")?, 149 | HashMap::new(), 150 | )?); 151 | } 152 | } 153 | 154 | Ok(infos) 155 | } 156 | 157 | #[pyo3(signature = (base_dir, allow_not_found = false, recursive = false))] 158 | fn get_file_info_selector<'py>( 159 | &self, 160 | base_dir: String, 161 | allow_not_found: bool, 162 | recursive: bool, 163 | py: Python<'py>, 164 | ) -> PyResult>> { 165 | let fs = PyModule::import_bound(py, "pyarrow.fs")?; 166 | let file_types = fs.getattr("FileType")?; 167 | 168 | let to_file_info = 169 | |loc: String, type_: Bound<'_, pyo3::PyAny>, kwargs: HashMap<&str, i64>| { 170 | fs.call_method( 171 | "FileInfo", 172 | (loc, type_), 173 | Some(&kwargs.into_py_dict_bound(py)), 174 | ) 175 | }; 176 | 177 | let path = Path::from(base_dir); 178 | let list_result = match self 179 | .rt 180 | .block_on(walk_tree(self.inner.clone(), &path, recursive)) 181 | { 182 | Ok(res) => Ok(res), 183 | Err(InnerObjectStoreError::NotFound { path, source }) => { 184 | if allow_not_found { 185 | Ok(ListResult { 186 | common_prefixes: vec![], 187 | objects: vec![], 188 | }) 189 | } else { 190 | Err(InnerObjectStoreError::NotFound { path, source }) 191 | } 192 | } 193 | Err(err) => Err(err), 194 | } 195 | .map_err(ObjectStoreError::from)?; 196 | 197 | let mut infos = vec![]; 198 | infos.extend( 199 | list_result 200 | .common_prefixes 201 | .into_iter() 202 | .map(|p| { 203 | to_file_info( 204 | p.to_string(), 205 | file_types.getattr("Directory")?, 206 | HashMap::new(), 207 | ) 208 | }) 209 | .collect::, _>>()?, 210 | ); 211 | infos.extend( 212 | list_result 213 | .objects 214 | .into_iter() 215 | .map(|meta| { 216 | let kwargs = HashMap::from([ 217 | ("size", meta.size as i64), 218 | ( 219 | "mtime_ns", 220 | meta.last_modified.timestamp_nanos_opt().unwrap(), 221 | ), 222 | ]); 223 | to_file_info( 224 | meta.location.to_string(), 225 | file_types.getattr("File")?, 226 | kwargs, 227 | ) 228 | }) 229 | .collect::, _>>()?, 230 | ); 231 | 232 | Ok(infos) 233 | } 234 | 235 | fn move_file(&self, src: String, dest: String) -> PyResult<()> { 236 | let from_path = Path::from(src); 237 | let to_path = Path::from(dest); 238 | // TODO check the if not exists semantics 239 | self.rt 240 | .block_on(self.inner.rename(&from_path, &to_path)) 241 | .map_err(ObjectStoreError::from)?; 242 | Ok(()) 243 | } 244 | 245 | fn open_input_file(&self, path: String) -> PyResult { 246 | let path = Path::from(path); 247 | let file = self 248 | .rt 249 | .block_on(ObjectInputFile::try_new( 250 | self.rt.clone(), 251 | self.inner.clone(), 252 | path, 253 | )) 254 | .map_err(ObjectStoreError::from)?; 255 | Ok(file) 256 | } 257 | 258 | #[pyo3(signature = (path, metadata = None))] 259 | fn open_output_stream( 260 | &self, 261 | path: String, 262 | #[allow(unused)] metadata: Option>, 263 | ) -> PyResult { 264 | let path = Path::from(path); 265 | let file = self 266 | .rt 267 | .block_on(ObjectOutputStream::try_new( 268 | self.rt.clone(), 269 | self.inner.clone(), 270 | path, 271 | )) 272 | .map_err(ObjectStoreError::from)?; 273 | Ok(file) 274 | } 275 | 276 | pub fn __getnewargs__(&self) -> PyResult<(String, Option>)> { 277 | Ok((self.root_url.clone(), self.options.clone())) 278 | } 279 | } 280 | 281 | // TODO the C++ implementation track an internal lock on all random access files, DO we need this here? 282 | // TODO add buffer to store data ... 283 | #[pyclass(weakref)] 284 | #[derive(Debug, Clone)] 285 | pub struct ObjectInputFile { 286 | store: Arc, 287 | rt: Arc, 288 | path: Path, 289 | content_length: i64, 290 | #[pyo3(get)] 291 | closed: bool, 292 | pos: i64, 293 | #[pyo3(get)] 294 | mode: String, 295 | } 296 | 297 | impl ObjectInputFile { 298 | pub async fn try_new( 299 | rt: Arc, 300 | store: Arc, 301 | path: Path, 302 | ) -> Result { 303 | // Issue a HEAD Object to get the content-length and ensure any 304 | // errors (e.g. file not found) don't wait until the first read() call. 305 | let meta = store.head(&path).await?; 306 | let content_length = meta.size as i64; 307 | // TODO make sure content length is valid 308 | // https://github.com/apache/arrow/blob/f184255cbb9bf911ea2a04910f711e1a924b12b8/cpp/src/arrow/filesystem/s3fs.cc#L1083 309 | Ok(Self { 310 | store, 311 | rt, 312 | path, 313 | content_length, 314 | closed: false, 315 | pos: 0, 316 | mode: "rb".into(), 317 | }) 318 | } 319 | 320 | fn check_closed(&self) -> Result<(), ObjectStoreError> { 321 | if self.closed { 322 | return Err(ObjectStoreError::Common( 323 | "Operation on closed stream".into(), 324 | )); 325 | } 326 | 327 | Ok(()) 328 | } 329 | 330 | fn check_position(&self, position: i64, action: &str) -> Result<(), ObjectStoreError> { 331 | if position < 0 { 332 | return Err(ObjectStoreError::Common(format!( 333 | "Cannot {} for negative position.", 334 | action 335 | ))); 336 | } 337 | if position > self.content_length { 338 | return Err(ObjectStoreError::Common(format!( 339 | "Cannot {} past end of file.", 340 | action 341 | ))); 342 | } 343 | Ok(()) 344 | } 345 | } 346 | 347 | #[pymethods] 348 | impl ObjectInputFile { 349 | fn close(&mut self) -> PyResult<()> { 350 | self.closed = true; 351 | Ok(()) 352 | } 353 | 354 | fn isatty(&self) -> PyResult { 355 | Ok(false) 356 | } 357 | 358 | fn readable(&self) -> PyResult { 359 | Ok(true) 360 | } 361 | 362 | fn seekable(&self) -> PyResult { 363 | Ok(true) 364 | } 365 | 366 | fn writable(&self) -> PyResult { 367 | Ok(false) 368 | } 369 | 370 | fn tell(&self) -> PyResult { 371 | self.check_closed()?; 372 | Ok(self.pos) 373 | } 374 | 375 | fn size(&self) -> PyResult { 376 | self.check_closed()?; 377 | Ok(self.content_length) 378 | } 379 | 380 | #[pyo3(signature = (offset, whence = 0))] 381 | fn seek(&mut self, offset: i64, whence: i64) -> PyResult { 382 | self.check_closed()?; 383 | self.check_position(offset, "seek")?; 384 | match whence { 385 | // reference is start of the stream (the default); offset should be zero or positive 386 | 0 => { 387 | self.pos = offset; 388 | } 389 | // reference is current stream position; offset may be negative 390 | 1 => { 391 | self.pos += offset; 392 | } 393 | // reference is end of the stream; offset is usually negative 394 | 2 => { 395 | self.pos = self.content_length - offset; 396 | } 397 | _ => { 398 | return Err(PyValueError::new_err( 399 | "'whence' must be between 0 <= whence <= 2.", 400 | )); 401 | } 402 | } 403 | Ok(self.pos) 404 | } 405 | 406 | #[pyo3(signature = (nbytes = None))] 407 | fn read(&mut self, nbytes: Option) -> PyResult> { 408 | self.check_closed()?; 409 | let range = match nbytes { 410 | Some(len) => { 411 | let end = i64::min(self.pos + len, self.content_length) as usize; 412 | std::ops::Range { 413 | start: self.pos as usize, 414 | end, 415 | } 416 | } 417 | _ => std::ops::Range { 418 | start: self.pos as usize, 419 | end: self.content_length as usize, 420 | }, 421 | }; 422 | let nbytes = (range.end - range.start) as i64; 423 | self.pos += nbytes; 424 | let data = if nbytes > 0 { 425 | self.rt 426 | .block_on(self.store.get_range(&self.path, range)) 427 | .map_err(ObjectStoreError::from)? 428 | } else { 429 | "".into() 430 | }; 431 | Python::with_gil(|py| Ok(PyBytes::new_bound(py, data.as_ref()).into_py(py))) 432 | } 433 | 434 | fn fileno(&self) -> PyResult<()> { 435 | Err(PyNotImplementedError::new_err("'fileno' not implemented")) 436 | } 437 | 438 | fn truncate(&self) -> PyResult<()> { 439 | Err(PyNotImplementedError::new_err("'truncate' not implemented")) 440 | } 441 | 442 | fn readline(&self, _size: Option) -> PyResult<()> { 443 | Err(PyNotImplementedError::new_err("'readline' not implemented")) 444 | } 445 | 446 | fn readlines(&self, _hint: Option) -> PyResult<()> { 447 | Err(PyNotImplementedError::new_err( 448 | "'readlines' not implemented", 449 | )) 450 | } 451 | } 452 | 453 | // TODO the C++ implementation track an internal lock on all random access files, DO we need this here? 454 | // TODO add buffer to store data ... 455 | #[pyclass(weakref)] 456 | pub struct ObjectOutputStream { 457 | pub store: Arc, 458 | rt: Arc, 459 | pub path: Path, 460 | writer: Box, 461 | pos: i64, 462 | #[pyo3(get)] 463 | closed: bool, 464 | #[pyo3(get)] 465 | mode: String, 466 | } 467 | 468 | impl ObjectOutputStream { 469 | pub async fn try_new( 470 | rt: Arc, 471 | store: Arc, 472 | path: Path, 473 | ) -> Result { 474 | match store.put_multipart(&path).await { 475 | Ok(writer) => Ok(Self { 476 | store, 477 | rt, 478 | path, 479 | writer, 480 | pos: 0, 481 | closed: false, 482 | mode: "wb".into(), 483 | }), 484 | Err(err) => Err(ObjectStoreError::ObjectStore(err)), 485 | } 486 | } 487 | 488 | fn check_closed(&self) -> Result<(), ObjectStoreError> { 489 | if self.closed { 490 | return Err(ObjectStoreError::Common( 491 | "Operation on closed stream".into(), 492 | )); 493 | } 494 | 495 | Ok(()) 496 | } 497 | } 498 | 499 | #[pymethods] 500 | impl ObjectOutputStream { 501 | fn close(&mut self) -> PyResult<()> { 502 | self.closed = true; 503 | match self.rt.block_on(self.writer.complete()) { 504 | Ok(_) => Ok(()), 505 | Err(err) => { 506 | self.rt 507 | .block_on(self.writer.abort()) 508 | .map_err(ObjectStoreError::from)?; 509 | Err(ObjectStoreError::from(err).into()) 510 | } 511 | } 512 | } 513 | 514 | fn isatty(&self) -> PyResult { 515 | Ok(false) 516 | } 517 | 518 | fn readable(&self) -> PyResult { 519 | Ok(false) 520 | } 521 | 522 | fn seekable(&self) -> PyResult { 523 | Ok(false) 524 | } 525 | 526 | fn writable(&self) -> PyResult { 527 | Ok(true) 528 | } 529 | 530 | fn tell(&self) -> PyResult { 531 | self.check_closed()?; 532 | Ok(self.pos) 533 | } 534 | 535 | fn size(&self) -> PyResult { 536 | self.check_closed()?; 537 | Err(PyNotImplementedError::new_err("'size' not implemented")) 538 | } 539 | 540 | fn seek(&mut self, _offset: i64, _whence: i64) -> PyResult { 541 | self.check_closed()?; 542 | Err(PyNotImplementedError::new_err("'seek' not implemented")) 543 | } 544 | 545 | fn read(&mut self, _nbytes: Option) -> PyResult<()> { 546 | self.check_closed()?; 547 | Err(PyNotImplementedError::new_err("'read' not implemented")) 548 | } 549 | 550 | fn write(&mut self, data: Bound<'_, PyBytes>) -> PyResult { 551 | self.check_closed()?; 552 | let bytes = data.as_bytes().to_vec(); 553 | let len = bytes.len() as i64; 554 | match self.rt.block_on(self.writer.put_part(bytes.into())) { 555 | Ok(_) => Ok(len), 556 | Err(err) => { 557 | self.rt 558 | .block_on(self.writer.abort()) 559 | .map_err(ObjectStoreError::from)?; 560 | Err(ObjectStoreError::from(err).into()) 561 | } 562 | } 563 | } 564 | 565 | fn flush(&mut self) -> PyResult<()> { 566 | match self.rt.block_on(self.writer.complete()) { 567 | Ok(_) => Ok(()), 568 | Err(err) => { 569 | self.rt 570 | .block_on(self.writer.abort()) 571 | .map_err(ObjectStoreError::from)?; 572 | Err(ObjectStoreError::from(err).into()) 573 | } 574 | } 575 | } 576 | 577 | fn fileno(&self) -> PyResult<()> { 578 | Err(PyNotImplementedError::new_err("'fileno' not implemented")) 579 | } 580 | 581 | fn truncate(&self) -> PyResult<()> { 582 | Err(PyNotImplementedError::new_err("'truncate' not implemented")) 583 | } 584 | 585 | fn readline(&self, _size: Option) -> PyResult<()> { 586 | Err(PyNotImplementedError::new_err("'readline' not implemented")) 587 | } 588 | 589 | fn readlines(&self, _hint: Option) -> PyResult<()> { 590 | Err(PyNotImplementedError::new_err( 591 | "'readlines' not implemented", 592 | )) 593 | } 594 | } 595 | -------------------------------------------------------------------------------- /object-store-internal/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod builder; 2 | mod file; 3 | mod utils; 4 | 5 | use pyo3_asyncio_0_21 as pyo3_asyncio; 6 | use std::borrow::Cow; 7 | use std::collections::HashMap; 8 | use std::fmt; 9 | use std::sync::Arc; 10 | use std::time::Duration; 11 | 12 | pub use crate::file::{ArrowFileSystemHandler, ObjectInputFile, ObjectOutputStream}; 13 | use crate::utils::{flatten_list_stream, get_bytes}; 14 | 15 | use object_store::path::{Error as PathError, Path}; 16 | use object_store::{ 17 | BackoffConfig, ClientOptions, DynObjectStore, Error as InnerObjectStoreError, ListResult, 18 | ObjectMeta, RetryConfig, 19 | }; 20 | use pyo3::exceptions::{ 21 | PyException, PyFileExistsError, PyFileNotFoundError, PyNotImplementedError, 22 | }; 23 | use pyo3::prelude::*; 24 | use pyo3::PyErr; 25 | use tokio::runtime::Runtime; 26 | 27 | pub use builder::ObjectStoreBuilder; 28 | 29 | #[derive(Debug)] 30 | pub enum ObjectStoreError { 31 | ObjectStore(InnerObjectStoreError), 32 | Common(String), 33 | Python(PyErr), 34 | IO(std::io::Error), 35 | Task(tokio::task::JoinError), 36 | Path(PathError), 37 | InputValue(String), 38 | } 39 | 40 | impl fmt::Display for ObjectStoreError { 41 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 42 | match self { 43 | ObjectStoreError::ObjectStore(e) => write!(f, "ObjectStore error: {:?}", e), 44 | ObjectStoreError::Python(e) => write!(f, "Python error {:?}", e), 45 | ObjectStoreError::Path(e) => write!(f, "Path error {:?}", e), 46 | ObjectStoreError::IO(e) => write!(f, "IOError error {:?}", e), 47 | ObjectStoreError::Task(e) => write!(f, "Task error {:?}", e), 48 | ObjectStoreError::Common(e) => write!(f, "{}", e), 49 | ObjectStoreError::InputValue(e) => write!(f, "Invalid input value: {}", e), 50 | } 51 | } 52 | } 53 | 54 | impl From for ObjectStoreError { 55 | fn from(err: InnerObjectStoreError) -> ObjectStoreError { 56 | ObjectStoreError::ObjectStore(err) 57 | } 58 | } 59 | 60 | impl From for ObjectStoreError { 61 | fn from(err: PathError) -> ObjectStoreError { 62 | ObjectStoreError::Path(err) 63 | } 64 | } 65 | 66 | impl From for ObjectStoreError { 67 | fn from(err: tokio::task::JoinError) -> ObjectStoreError { 68 | ObjectStoreError::Task(err) 69 | } 70 | } 71 | 72 | impl From for ObjectStoreError { 73 | fn from(err: std::io::Error) -> ObjectStoreError { 74 | ObjectStoreError::IO(err) 75 | } 76 | } 77 | 78 | impl From for ObjectStoreError { 79 | fn from(err: PyErr) -> ObjectStoreError { 80 | ObjectStoreError::Python(err) 81 | } 82 | } 83 | 84 | impl From for PyErr { 85 | fn from(err: ObjectStoreError) -> PyErr { 86 | match err { 87 | ObjectStoreError::Python(py_err) => py_err, 88 | ObjectStoreError::ObjectStore(store_err) => match store_err { 89 | InnerObjectStoreError::NotFound { .. } => { 90 | PyFileNotFoundError::new_err(store_err.to_string()) 91 | } 92 | InnerObjectStoreError::AlreadyExists { .. } => { 93 | PyFileExistsError::new_err(store_err.to_string()) 94 | } 95 | _ => PyException::new_err(store_err.to_string()), 96 | }, 97 | _ => PyException::new_err(err.to_string()), 98 | } 99 | } 100 | } 101 | 102 | #[pyclass(name = "Path", subclass)] 103 | #[derive(Clone)] 104 | pub struct PyPath(Path); 105 | 106 | impl From for Path { 107 | fn from(path: PyPath) -> Self { 108 | path.0 109 | } 110 | } 111 | 112 | impl From for PyPath { 113 | fn from(path: Path) -> Self { 114 | Self(path) 115 | } 116 | } 117 | 118 | #[pymethods] 119 | impl PyPath { 120 | #[new] 121 | fn new(path: String) -> PyResult { 122 | Ok(Self(Path::parse(path).map_err(ObjectStoreError::from)?)) 123 | } 124 | 125 | /// Creates a new child of this [`Path`] 126 | fn child(&self, part: String) -> Self { 127 | Self(self.0.child(part)) 128 | } 129 | 130 | fn __str__(&self) -> String { 131 | self.0.to_string() 132 | } 133 | 134 | fn __richcmp__(&self, other: PyPath, cmp: pyo3::basic::CompareOp) -> PyResult { 135 | match cmp { 136 | pyo3::basic::CompareOp::Eq => Ok(self.0 == other.0), 137 | pyo3::basic::CompareOp::Ne => Ok(self.0 != other.0), 138 | _ => Err(PyNotImplementedError::new_err( 139 | "Only == and != are supported.", 140 | )), 141 | } 142 | } 143 | } 144 | 145 | #[pyclass(name = "ObjectMeta", subclass)] 146 | #[derive(Clone)] 147 | pub struct PyObjectMeta(ObjectMeta); 148 | 149 | impl From for PyObjectMeta { 150 | fn from(meta: ObjectMeta) -> Self { 151 | Self(meta) 152 | } 153 | } 154 | 155 | #[pymethods] 156 | impl PyObjectMeta { 157 | #[getter] 158 | fn location(&self) -> PyPath { 159 | self.0.location.clone().into() 160 | } 161 | 162 | #[getter] 163 | fn size(&self) -> usize { 164 | self.0.size 165 | } 166 | 167 | #[getter] 168 | fn last_modified(&self) -> i64 { 169 | self.0.last_modified.timestamp() 170 | } 171 | 172 | fn __str__(&self) -> String { 173 | format!("{:?}", self.0) 174 | } 175 | 176 | fn __repr__(&self) -> String { 177 | format!("{:?}", self.0) 178 | } 179 | 180 | fn __richcmp__(&self, other: PyObjectMeta, cmp: pyo3::basic::CompareOp) -> PyResult { 181 | match cmp { 182 | pyo3::basic::CompareOp::Eq => Ok(self.0 == other.0), 183 | pyo3::basic::CompareOp::Ne => Ok(self.0 != other.0), 184 | _ => Err(PyNotImplementedError::new_err( 185 | "Only == and != are supported.", 186 | )), 187 | } 188 | } 189 | } 190 | 191 | #[pyclass(name = "ListResult", subclass)] 192 | pub struct PyListResult(ListResult); 193 | 194 | #[pymethods] 195 | impl PyListResult { 196 | #[getter] 197 | fn common_prefixes(&self) -> Vec { 198 | self.0 199 | .common_prefixes 200 | .iter() 201 | .cloned() 202 | .map(PyPath::from) 203 | .collect() 204 | } 205 | 206 | #[getter] 207 | fn objects(&self) -> Vec { 208 | self.0 209 | .objects 210 | .iter() 211 | .cloned() 212 | .map(PyObjectMeta::from) 213 | .collect() 214 | } 215 | } 216 | 217 | impl From for PyListResult { 218 | fn from(result: ListResult) -> Self { 219 | Self(result) 220 | } 221 | } 222 | 223 | #[pyclass(name = "ClientOptions")] 224 | #[derive(Debug, Clone, Default)] 225 | pub struct PyClientOptions { 226 | #[pyo3(get, set)] 227 | user_agent: Option, 228 | #[pyo3(get, set)] 229 | content_type_map: HashMap, 230 | #[pyo3(get, set)] 231 | default_content_type: Option, 232 | // default_headers: Option, 233 | #[pyo3(get, set)] 234 | proxy_url: Option, 235 | #[pyo3(get, set)] 236 | allow_http: bool, 237 | #[pyo3(get, set)] 238 | allow_insecure: bool, 239 | #[pyo3(get, set)] 240 | timeout: Option, 241 | #[pyo3(get, set)] 242 | connect_timeout: Option, 243 | #[pyo3(get, set)] 244 | pool_idle_timeout: Option, 245 | #[pyo3(get, set)] 246 | pool_max_idle_per_host: Option, 247 | #[pyo3(get, set)] 248 | http2_keep_alive_interval: Option, 249 | #[pyo3(get, set)] 250 | http2_keep_alive_timeout: Option, 251 | #[pyo3(get, set)] 252 | http2_keep_alive_while_idle: bool, 253 | #[pyo3(get, set)] 254 | http1_only: bool, 255 | #[pyo3(get, set)] 256 | http2_only: bool, 257 | #[pyo3(get, set)] 258 | retry_init_backoff: Option, 259 | #[pyo3(get, set)] 260 | retry_max_backoff: Option, 261 | #[pyo3(get, set)] 262 | retry_backoff_base: Option, 263 | #[pyo3(get, set)] 264 | retry_max_retries: Option, 265 | #[pyo3(get, set)] 266 | retry_timeout: Option, 267 | } 268 | 269 | impl PyClientOptions { 270 | fn client_options(&self) -> Result { 271 | let mut options = ClientOptions::new() 272 | .with_allow_http(self.allow_http) 273 | .with_allow_invalid_certificates(self.allow_insecure); 274 | if let Some(user_agent) = &self.user_agent { 275 | options = options.with_user_agent( 276 | user_agent 277 | .clone() 278 | .try_into() 279 | .map_err(|_| ObjectStoreError::InputValue(user_agent.into()))?, 280 | ); 281 | } 282 | if let Some(default_content_type) = &self.default_content_type { 283 | options = options.with_default_content_type(default_content_type); 284 | } 285 | if let Some(proxy_url) = &self.proxy_url { 286 | options = options.with_proxy_url(proxy_url); 287 | } 288 | if let Some(timeout) = self.timeout { 289 | options = options.with_timeout(Duration::from_secs(timeout)); 290 | } 291 | if let Some(connect_timeout) = self.connect_timeout { 292 | options = options.with_connect_timeout(Duration::from_secs(connect_timeout)); 293 | } 294 | if let Some(pool_idle_timeout) = self.pool_idle_timeout { 295 | options = options.with_pool_idle_timeout(Duration::from_secs(pool_idle_timeout)); 296 | } 297 | if let Some(pool_max_idle_per_host) = self.pool_max_idle_per_host { 298 | options = options.with_pool_max_idle_per_host(pool_max_idle_per_host); 299 | } 300 | if let Some(http2_keep_alive_interval) = self.http2_keep_alive_interval { 301 | options = options 302 | .with_http2_keep_alive_interval(Duration::from_secs(http2_keep_alive_interval)); 303 | } 304 | if let Some(http2_keep_alive_timeout) = self.http2_keep_alive_timeout { 305 | options = options 306 | .with_http2_keep_alive_timeout(Duration::from_secs(http2_keep_alive_timeout)); 307 | } 308 | if self.http2_keep_alive_while_idle { 309 | options = options.with_http2_keep_alive_while_idle(); 310 | } 311 | if self.http1_only { 312 | options = options.with_http1_only(); 313 | } 314 | if self.http2_only { 315 | options = options.with_http2_only(); 316 | } 317 | Ok(options) 318 | } 319 | 320 | fn retry_config(&self) -> Result { 321 | let mut backoff = BackoffConfig::default(); 322 | if let Some(init_backoff) = self.retry_init_backoff { 323 | backoff.init_backoff = Duration::from_secs(init_backoff); 324 | } 325 | if let Some(max_backoff) = self.retry_max_backoff { 326 | backoff.max_backoff = Duration::from_secs(max_backoff); 327 | } 328 | if let Some(backoff_base) = self.retry_backoff_base { 329 | backoff.base = backoff_base; 330 | } 331 | let mut config = RetryConfig { 332 | backoff, 333 | ..Default::default() 334 | }; 335 | if let Some(max_retries) = self.retry_max_retries { 336 | config.max_retries = max_retries; 337 | } 338 | if let Some(timeout) = self.retry_timeout { 339 | config.retry_timeout = Duration::from_secs(timeout); 340 | } 341 | Ok(config) 342 | } 343 | } 344 | 345 | impl TryFrom for ClientOptions { 346 | type Error = ObjectStoreError; 347 | 348 | fn try_from(value: PyClientOptions) -> Result { 349 | let mut options = ClientOptions::new() 350 | .with_allow_http(value.allow_http) 351 | .with_allow_invalid_certificates(value.allow_insecure); 352 | if let Some(user_agent) = value.user_agent { 353 | options = options.with_user_agent( 354 | user_agent 355 | .clone() 356 | .try_into() 357 | .map_err(|_| ObjectStoreError::InputValue(user_agent))?, 358 | ); 359 | } 360 | if let Some(default_content_type) = value.default_content_type { 361 | options = options.with_default_content_type(default_content_type); 362 | } 363 | if let Some(proxy_url) = value.proxy_url { 364 | options = options.with_proxy_url(proxy_url); 365 | } 366 | if let Some(timeout) = value.timeout { 367 | options = options.with_timeout(Duration::from_secs(timeout)); 368 | } 369 | if let Some(connect_timeout) = value.connect_timeout { 370 | options = options.with_connect_timeout(Duration::from_secs(connect_timeout)); 371 | } 372 | if let Some(pool_idle_timeout) = value.pool_idle_timeout { 373 | options = options.with_pool_idle_timeout(Duration::from_secs(pool_idle_timeout)); 374 | } 375 | if let Some(pool_max_idle_per_host) = value.pool_max_idle_per_host { 376 | options = options.with_pool_max_idle_per_host(pool_max_idle_per_host); 377 | } 378 | if let Some(http2_keep_alive_interval) = value.http2_keep_alive_interval { 379 | options = options 380 | .with_http2_keep_alive_interval(Duration::from_secs(http2_keep_alive_interval)); 381 | } 382 | if let Some(http2_keep_alive_timeout) = value.http2_keep_alive_timeout { 383 | options = options 384 | .with_http2_keep_alive_timeout(Duration::from_secs(http2_keep_alive_timeout)); 385 | } 386 | if value.http2_keep_alive_while_idle { 387 | options = options.with_http2_keep_alive_while_idle(); 388 | } 389 | if value.http1_only { 390 | options = options.with_http1_only(); 391 | } 392 | if value.http2_only { 393 | options = options.with_http2_only(); 394 | } 395 | Ok(options) 396 | } 397 | } 398 | 399 | #[pymethods] 400 | impl PyClientOptions { 401 | #[new] 402 | #[pyo3(signature = ( 403 | user_agent = None, 404 | content_type_map = None, 405 | default_content_type = None, 406 | proxy_url = None, 407 | allow_http = false, 408 | allow_insecure = false, 409 | timeout = None, 410 | connect_timeout = None, 411 | pool_idle_timeout = None, 412 | pool_max_idle_per_host = None, 413 | http2_keep_alive_interval = None, 414 | http2_keep_alive_timeout = None, 415 | http2_keep_alive_while_idle = false, 416 | http1_only = false, 417 | http2_only = false, 418 | retry_init_backoff = None, 419 | retry_max_backoff = None, 420 | retry_backoff_base = None, 421 | retry_max_retries = None, 422 | retry_timeout = None, 423 | ))] 424 | /// Create a new ObjectStore instance 425 | #[allow(clippy::too_many_arguments)] 426 | fn new( 427 | user_agent: Option, 428 | content_type_map: Option>, 429 | default_content_type: Option, 430 | proxy_url: Option, 431 | allow_http: bool, 432 | allow_insecure: bool, 433 | timeout: Option, 434 | connect_timeout: Option, 435 | pool_idle_timeout: Option, 436 | pool_max_idle_per_host: Option, 437 | http2_keep_alive_interval: Option, 438 | http2_keep_alive_timeout: Option, 439 | http2_keep_alive_while_idle: bool, 440 | http1_only: bool, 441 | http2_only: bool, 442 | retry_init_backoff: Option, 443 | retry_max_backoff: Option, 444 | retry_backoff_base: Option, 445 | retry_max_retries: Option, 446 | retry_timeout: Option, 447 | ) -> Self { 448 | Self { 449 | user_agent, 450 | content_type_map: content_type_map.unwrap_or_default(), 451 | default_content_type, 452 | proxy_url, 453 | allow_http, 454 | allow_insecure, 455 | timeout, 456 | connect_timeout, 457 | pool_idle_timeout, 458 | pool_max_idle_per_host, 459 | http2_keep_alive_interval, 460 | http2_keep_alive_timeout, 461 | http2_keep_alive_while_idle, 462 | http1_only, 463 | http2_only, 464 | retry_init_backoff, 465 | retry_max_backoff, 466 | retry_backoff_base, 467 | retry_max_retries, 468 | retry_timeout, 469 | } 470 | } 471 | } 472 | 473 | #[pyclass(name = "ObjectStore", subclass)] 474 | #[derive(Debug, Clone)] 475 | /// A generic object store interface for uniformly interacting with AWS S3, Google Cloud Storage, 476 | /// Azure Blob Storage and local files. 477 | pub struct PyObjectStore { 478 | pub inner: Arc, 479 | rt: Arc, 480 | #[pyo3(get)] 481 | root_url: String, 482 | options: Option>, 483 | } 484 | 485 | #[pymethods] 486 | impl PyObjectStore { 487 | #[new] 488 | #[pyo3(signature = (root, options = None, client_options = None))] 489 | /// Create a new ObjectStore instance 490 | fn new( 491 | root: String, 492 | options: Option>, 493 | client_options: Option, 494 | ) -> PyResult { 495 | let client_options = client_options.unwrap_or_default(); 496 | let inner = ObjectStoreBuilder::new(root.clone()) 497 | .with_path_as_prefix(true) 498 | .with_options(options.clone().unwrap_or_default()) 499 | .with_client_options(client_options.client_options()?) 500 | .with_retry_config(client_options.retry_config()?) 501 | .build() 502 | .map_err(ObjectStoreError::from)?; 503 | Ok(Self { 504 | root_url: root, 505 | inner, 506 | rt: Arc::new(Runtime::new()?), 507 | options, 508 | }) 509 | } 510 | 511 | /// Save the provided bytes to the specified location. 512 | #[pyo3(text_signature = "($self, location, bytes)")] 513 | fn put(&self, py: Python, location: PyPath, bytes: Vec) -> PyResult<()> { 514 | py.allow_threads(|| { 515 | self.rt 516 | .block_on(self.inner.put(&location.into(), bytes.into())) 517 | .map_err(ObjectStoreError::from)?; 518 | Ok(()) 519 | }) 520 | } 521 | 522 | /// Save the provided bytes to the specified location. 523 | #[pyo3(text_signature = "($self, location, bytes)")] 524 | fn put_async<'a>( 525 | &'a self, 526 | py: Python<'a>, 527 | location: PyPath, 528 | bytes: Vec, 529 | ) -> PyResult> { 530 | let inner = self.inner.clone(); 531 | pyo3_asyncio::tokio::future_into_py(py, async move { 532 | inner 533 | .put(&location.into(), bytes.into()) 534 | .await 535 | .map_err(ObjectStoreError::from)?; 536 | Ok(()) 537 | }) 538 | } 539 | 540 | /// Return the bytes that are stored at the specified location. 541 | #[pyo3(text_signature = "($self, location)")] 542 | fn get(&self, py: Python, location: PyPath) -> PyResult> { 543 | py.allow_threads(|| { 544 | let obj = self 545 | .rt 546 | .block_on(get_bytes(self.inner.as_ref(), &location.into())) 547 | .map_err(ObjectStoreError::from)?; 548 | Ok(Cow::Owned(obj.to_vec())) 549 | }) 550 | } 551 | 552 | /// Return the bytes that are stored at the specified location. 553 | #[pyo3(text_signature = "($self, location)")] 554 | fn get_async<'a>(&'a self, py: Python<'a>, location: PyPath) -> PyResult> { 555 | let inner = self.inner.clone(); 556 | pyo3_asyncio::tokio::future_into_py(py, async move { 557 | let obj = get_bytes(inner.as_ref(), &location.into()) 558 | .await 559 | .map_err(ObjectStoreError::from)?; 560 | Ok(Cow::<[u8]>::Owned(obj.to_vec())) 561 | }) 562 | } 563 | 564 | /// Return the bytes that are stored at the specified location in the given byte range 565 | #[pyo3(text_signature = "($self, location, start, length)")] 566 | fn get_range( 567 | &self, 568 | py: Python, 569 | location: PyPath, 570 | start: usize, 571 | length: usize, 572 | ) -> PyResult> { 573 | py.allow_threads(|| { 574 | let range = std::ops::Range { 575 | start, 576 | end: start + length, 577 | }; 578 | let obj = self 579 | .rt 580 | .block_on(self.inner.get_range(&location.into(), range)) 581 | .map_err(ObjectStoreError::from)? 582 | .to_vec(); 583 | Ok(Cow::Owned(obj.to_vec())) 584 | }) 585 | } 586 | 587 | /// Return the bytes that are stored at the specified location in the given byte range 588 | #[pyo3(text_signature = "($self, location, start, length)")] 589 | fn get_range_async<'a>( 590 | &'a self, 591 | py: Python<'a>, 592 | location: PyPath, 593 | start: usize, 594 | length: usize, 595 | ) -> PyResult> { 596 | let inner = self.inner.clone(); 597 | let range = std::ops::Range { 598 | start, 599 | end: start + length, 600 | }; 601 | 602 | pyo3_asyncio::tokio::future_into_py(py, async move { 603 | let obj = inner 604 | .get_range(&location.into(), range) 605 | .await 606 | .map_err(ObjectStoreError::from)?; 607 | Ok(Cow::<[u8]>::Owned(obj.to_vec())) 608 | }) 609 | } 610 | 611 | /// Return the metadata for the specified location 612 | #[pyo3(text_signature = "($self, location)")] 613 | fn head(&self, py: Python, location: PyPath) -> PyResult { 614 | py.allow_threads(|| { 615 | let meta = self 616 | .rt 617 | .block_on(self.inner.head(&location.into())) 618 | .map_err(ObjectStoreError::from)?; 619 | Ok(meta.into()) 620 | }) 621 | } 622 | 623 | /// Return the metadata for the specified location 624 | #[pyo3(text_signature = "($self, location)")] 625 | fn head_async<'a>(&'a self, py: Python<'a>, location: PyPath) -> PyResult> { 626 | let inner = self.inner.clone(); 627 | pyo3_asyncio::tokio::future_into_py(py, async move { 628 | let meta = inner 629 | .head(&location.into()) 630 | .await 631 | .map_err(ObjectStoreError::from)?; 632 | Ok(PyObjectMeta::from(meta)) 633 | }) 634 | } 635 | 636 | /// Delete the object at the specified location. 637 | #[pyo3(text_signature = "($self, location)")] 638 | fn delete(&self, py: Python, location: PyPath) -> PyResult<()> { 639 | py.allow_threads(|| { 640 | self.rt 641 | .block_on(self.inner.delete(&location.into())) 642 | .map_err(ObjectStoreError::from)?; 643 | Ok(()) 644 | }) 645 | } 646 | 647 | /// Delete the object at the specified location. 648 | #[pyo3(text_signature = "($self, location)")] 649 | fn delete_async<'a>(&'a self, py: Python<'a>, location: PyPath) -> PyResult> { 650 | let inner = self.inner.clone(); 651 | pyo3_asyncio::tokio::future_into_py(py, async move { 652 | inner 653 | .delete(&location.into()) 654 | .await 655 | .map_err(ObjectStoreError::from)?; 656 | Ok(()) 657 | }) 658 | } 659 | 660 | /// List all the objects with the given prefix. 661 | /// 662 | /// Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix 663 | /// of `foo/bar/x` but not of `foo/bar_baz/x`. 664 | #[pyo3(text_signature = "($self, prefix)")] 665 | fn list(&self, py: Python, prefix: Option) -> PyResult> { 666 | py.allow_threads(|| { 667 | Ok(self 668 | .rt 669 | .block_on(flatten_list_stream( 670 | self.inner.as_ref(), 671 | prefix.map(Path::from).as_ref(), 672 | )) 673 | .map_err(ObjectStoreError::from)? 674 | .into_iter() 675 | .map(PyObjectMeta::from) 676 | .collect()) 677 | }) 678 | } 679 | 680 | /// List all the objects with the given prefix. 681 | /// 682 | /// Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix 683 | /// of `foo/bar/x` but not of `foo/bar_baz/x`. 684 | #[pyo3(text_signature = "($self, prefix)")] 685 | fn list_async<'a>(&'a self, py: Python<'a>, prefix: Option) -> PyResult> { 686 | let inner = self.inner.clone(); 687 | pyo3_asyncio::tokio::future_into_py(py, async move { 688 | let object_metas = flatten_list_stream(inner.as_ref(), prefix.map(Path::from).as_ref()) 689 | .await 690 | .map_err(ObjectStoreError::from)?; 691 | let py_object_metas = object_metas 692 | .into_iter() 693 | .map(PyObjectMeta::from) 694 | .collect::>(); 695 | Ok(py_object_metas) 696 | }) 697 | } 698 | 699 | /// List objects with the given prefix and an implementation specific 700 | /// delimiter. Returns common prefixes (directories) in addition to object 701 | /// metadata. 702 | /// 703 | /// Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix 704 | /// of `foo/bar/x` but not of `foo/bar_baz/x`. 705 | #[pyo3(text_signature = "($self, prefix)")] 706 | fn list_with_delimiter(&self, py: Python, prefix: Option) -> PyResult { 707 | py.allow_threads(|| { 708 | let list = self 709 | .rt 710 | .block_on( 711 | self.inner 712 | .list_with_delimiter(prefix.map(Path::from).as_ref()), 713 | ) 714 | .map_err(ObjectStoreError::from)?; 715 | Ok(list.into()) 716 | }) 717 | } 718 | 719 | /// List objects with the given prefix and an implementation specific 720 | /// delimiter. Returns common prefixes (directories) in addition to object 721 | /// metadata. 722 | /// 723 | /// Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix 724 | /// of `foo/bar/x` but not of `foo/bar_baz/x`. 725 | #[pyo3(text_signature = "($self, prefix)")] 726 | fn list_with_delimiter_async<'a>( 727 | &'a self, 728 | py: Python<'a>, 729 | prefix: Option, 730 | ) -> PyResult> { 731 | let inner = self.inner.clone(); 732 | pyo3_asyncio::tokio::future_into_py(py, async move { 733 | let list_result = inner 734 | .list_with_delimiter(prefix.map(Path::from).as_ref()) 735 | .await 736 | .map_err(ObjectStoreError::from)?; 737 | Ok(PyListResult::from(list_result)) 738 | }) 739 | } 740 | 741 | /// Copy an object from one path to another in the same object store. 742 | /// 743 | /// If there exists an object at the destination, it will be overwritten. 744 | #[pyo3(text_signature = "($self, from, to)")] 745 | fn copy(&self, py: Python, from: PyPath, to: PyPath) -> PyResult<()> { 746 | py.allow_threads(|| { 747 | self.rt 748 | .block_on(self.inner.copy(&from.into(), &to.into())) 749 | .map_err(ObjectStoreError::from)?; 750 | Ok(()) 751 | }) 752 | } 753 | 754 | /// Copy an object from one path to another in the same object store. 755 | /// 756 | /// If there exists an object at the destination, it will be overwritten. 757 | #[pyo3(text_signature = "($self, from, to)")] 758 | fn copy_async<'a>( 759 | &'a self, 760 | py: Python<'a>, 761 | from: PyPath, 762 | to: PyPath, 763 | ) -> PyResult> { 764 | let inner = self.inner.clone(); 765 | pyo3_asyncio::tokio::future_into_py(py, async move { 766 | inner 767 | .copy(&from.into(), &to.into()) 768 | .await 769 | .map_err(ObjectStoreError::from)?; 770 | Ok(()) 771 | }) 772 | } 773 | 774 | /// Copy an object from one path to another, only if destination is empty. 775 | /// 776 | /// Will return an error if the destination already has an object. 777 | #[pyo3(text_signature = "($self, from, to)")] 778 | fn copy_if_not_exists(&self, py: Python, from: PyPath, to: PyPath) -> PyResult<()> { 779 | py.allow_threads(|| { 780 | self.rt 781 | .block_on(self.inner.copy_if_not_exists(&from.into(), &to.into())) 782 | .map_err(ObjectStoreError::from)?; 783 | Ok(()) 784 | }) 785 | } 786 | 787 | /// Copy an object from one path to another, only if destination is empty. 788 | /// 789 | /// Will return an error if the destination already has an object. 790 | #[pyo3(text_signature = "($self, from, to)")] 791 | fn copy_if_not_exists_async<'a>( 792 | &'a self, 793 | py: Python<'a>, 794 | from: PyPath, 795 | to: PyPath, 796 | ) -> PyResult> { 797 | let inner = self.inner.clone(); 798 | pyo3_asyncio::tokio::future_into_py(py, async move { 799 | inner 800 | .copy_if_not_exists(&from.into(), &to.into()) 801 | .await 802 | .map_err(ObjectStoreError::from)?; 803 | Ok(()) 804 | }) 805 | } 806 | 807 | /// Move an object from one path to another in the same object store. 808 | /// 809 | /// By default, this is implemented as a copy and then delete source. It may not 810 | /// check when deleting source that it was the same object that was originally copied. 811 | /// 812 | /// If there exists an object at the destination, it will be overwritten. 813 | #[pyo3(text_signature = "($self, from, to)")] 814 | fn rename(&self, py: Python, from: PyPath, to: PyPath) -> PyResult<()> { 815 | py.allow_threads(|| { 816 | self.rt 817 | .block_on(self.inner.rename(&from.into(), &to.into())) 818 | .map_err(ObjectStoreError::from)?; 819 | Ok(()) 820 | }) 821 | } 822 | 823 | /// Move an object from one path to another in the same object store. 824 | /// 825 | /// By default, this is implemented as a copy and then delete source. It may not 826 | /// check when deleting source that it was the same object that was originally copied. 827 | /// 828 | /// If there exists an object at the destination, it will be overwritten. 829 | #[pyo3(text_signature = "($self, from, to)")] 830 | fn rename_async<'a>( 831 | &'a self, 832 | py: Python<'a>, 833 | from: PyPath, 834 | to: PyPath, 835 | ) -> PyResult> { 836 | let inner = self.inner.clone(); 837 | pyo3_asyncio::tokio::future_into_py(py, async move { 838 | inner 839 | .rename(&from.into(), &to.into()) 840 | .await 841 | .map_err(ObjectStoreError::from)?; 842 | Ok(()) 843 | }) 844 | } 845 | 846 | /// Move an object from one path to another in the same object store. 847 | /// 848 | /// Will return an error if the destination already has an object. 849 | #[pyo3(text_signature = "($self, from, to)")] 850 | fn rename_if_not_exists(&self, py: Python, from: PyPath, to: PyPath) -> PyResult<()> { 851 | py.allow_threads(|| { 852 | self.rt 853 | .block_on(self.inner.rename_if_not_exists(&from.into(), &to.into())) 854 | .map_err(ObjectStoreError::from)?; 855 | Ok(()) 856 | }) 857 | } 858 | 859 | /// Move an object from one path to another in the same object store. 860 | /// 861 | /// Will return an error if the destination already has an object. 862 | #[pyo3(text_signature = "($self, from, to)")] 863 | fn rename_if_not_exists_async<'a>( 864 | &'a self, 865 | py: Python<'a>, 866 | from: PyPath, 867 | to: PyPath, 868 | ) -> PyResult> { 869 | let inner = self.inner.clone(); 870 | pyo3_asyncio::tokio::future_into_py(py, async move { 871 | inner 872 | .rename_if_not_exists(&from.into(), &to.into()) 873 | .await 874 | .map_err(ObjectStoreError::from)?; 875 | Ok(()) 876 | }) 877 | } 878 | 879 | pub fn __getnewargs__(&self) -> PyResult<(String, Option>)> { 880 | Ok((self.root_url.clone(), self.options.clone())) 881 | } 882 | } 883 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "addr2line" 7 | version = "0.21.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" 10 | dependencies = [ 11 | "gimli", 12 | ] 13 | 14 | [[package]] 15 | name = "adler" 16 | version = "1.0.2" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" 19 | 20 | [[package]] 21 | name = "android-tzdata" 22 | version = "0.1.1" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" 25 | 26 | [[package]] 27 | name = "android_system_properties" 28 | version = "0.1.5" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" 31 | dependencies = [ 32 | "libc", 33 | ] 34 | 35 | [[package]] 36 | name = "async-trait" 37 | version = "0.1.77" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" 40 | dependencies = [ 41 | "proc-macro2", 42 | "quote", 43 | "syn 2.0.49", 44 | ] 45 | 46 | [[package]] 47 | name = "atomic-waker" 48 | version = "1.1.2" 49 | source = "registry+https://github.com/rust-lang/crates.io-index" 50 | checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" 51 | 52 | [[package]] 53 | name = "autocfg" 54 | version = "1.1.0" 55 | source = "registry+https://github.com/rust-lang/crates.io-index" 56 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 57 | 58 | [[package]] 59 | name = "backtrace" 60 | version = "0.3.69" 61 | source = "registry+https://github.com/rust-lang/crates.io-index" 62 | checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" 63 | dependencies = [ 64 | "addr2line", 65 | "cc", 66 | "cfg-if", 67 | "libc", 68 | "miniz_oxide", 69 | "object", 70 | "rustc-demangle", 71 | ] 72 | 73 | [[package]] 74 | name = "base64" 75 | version = "0.21.7" 76 | source = "registry+https://github.com/rust-lang/crates.io-index" 77 | checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" 78 | 79 | [[package]] 80 | name = "base64" 81 | version = "0.22.1" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" 84 | 85 | [[package]] 86 | name = "bitflags" 87 | version = "1.3.2" 88 | source = "registry+https://github.com/rust-lang/crates.io-index" 89 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 90 | 91 | [[package]] 92 | name = "bitflags" 93 | version = "2.4.2" 94 | source = "registry+https://github.com/rust-lang/crates.io-index" 95 | checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" 96 | 97 | [[package]] 98 | name = "block-buffer" 99 | version = "0.10.4" 100 | source = "registry+https://github.com/rust-lang/crates.io-index" 101 | checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" 102 | dependencies = [ 103 | "generic-array", 104 | ] 105 | 106 | [[package]] 107 | name = "bumpalo" 108 | version = "3.15.0" 109 | source = "registry+https://github.com/rust-lang/crates.io-index" 110 | checksum = "d32a994c2b3ca201d9b263612a374263f05e7adde37c4707f693dcd375076d1f" 111 | 112 | [[package]] 113 | name = "bytes" 114 | version = "1.5.0" 115 | source = "registry+https://github.com/rust-lang/crates.io-index" 116 | checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" 117 | 118 | [[package]] 119 | name = "cc" 120 | version = "1.0.83" 121 | source = "registry+https://github.com/rust-lang/crates.io-index" 122 | checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" 123 | dependencies = [ 124 | "libc", 125 | ] 126 | 127 | [[package]] 128 | name = "cfg-if" 129 | version = "1.0.0" 130 | source = "registry+https://github.com/rust-lang/crates.io-index" 131 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 132 | 133 | [[package]] 134 | name = "chrono" 135 | version = "0.4.34" 136 | source = "registry+https://github.com/rust-lang/crates.io-index" 137 | checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" 138 | dependencies = [ 139 | "android-tzdata", 140 | "iana-time-zone", 141 | "num-traits", 142 | "serde", 143 | "windows-targets 0.52.0", 144 | ] 145 | 146 | [[package]] 147 | name = "core-foundation" 148 | version = "0.9.4" 149 | source = "registry+https://github.com/rust-lang/crates.io-index" 150 | checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" 151 | dependencies = [ 152 | "core-foundation-sys", 153 | "libc", 154 | ] 155 | 156 | [[package]] 157 | name = "core-foundation-sys" 158 | version = "0.8.6" 159 | source = "registry+https://github.com/rust-lang/crates.io-index" 160 | checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" 161 | 162 | [[package]] 163 | name = "crypto-common" 164 | version = "0.1.6" 165 | source = "registry+https://github.com/rust-lang/crates.io-index" 166 | checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" 167 | dependencies = [ 168 | "generic-array", 169 | "typenum", 170 | ] 171 | 172 | [[package]] 173 | name = "digest" 174 | version = "0.10.7" 175 | source = "registry+https://github.com/rust-lang/crates.io-index" 176 | checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" 177 | dependencies = [ 178 | "block-buffer", 179 | "crypto-common", 180 | ] 181 | 182 | [[package]] 183 | name = "doc-comment" 184 | version = "0.3.3" 185 | source = "registry+https://github.com/rust-lang/crates.io-index" 186 | checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" 187 | 188 | [[package]] 189 | name = "either" 190 | version = "1.10.0" 191 | source = "registry+https://github.com/rust-lang/crates.io-index" 192 | checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" 193 | 194 | [[package]] 195 | name = "encoding_rs" 196 | version = "0.8.33" 197 | source = "registry+https://github.com/rust-lang/crates.io-index" 198 | checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" 199 | dependencies = [ 200 | "cfg-if", 201 | ] 202 | 203 | [[package]] 204 | name = "equivalent" 205 | version = "1.0.1" 206 | source = "registry+https://github.com/rust-lang/crates.io-index" 207 | checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" 208 | 209 | [[package]] 210 | name = "errno" 211 | version = "0.3.8" 212 | source = "registry+https://github.com/rust-lang/crates.io-index" 213 | checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" 214 | dependencies = [ 215 | "libc", 216 | "windows-sys 0.52.0", 217 | ] 218 | 219 | [[package]] 220 | name = "fastrand" 221 | version = "2.0.1" 222 | source = "registry+https://github.com/rust-lang/crates.io-index" 223 | checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" 224 | 225 | [[package]] 226 | name = "fnv" 227 | version = "1.0.7" 228 | source = "registry+https://github.com/rust-lang/crates.io-index" 229 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 230 | 231 | [[package]] 232 | name = "foreign-types" 233 | version = "0.3.2" 234 | source = "registry+https://github.com/rust-lang/crates.io-index" 235 | checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" 236 | dependencies = [ 237 | "foreign-types-shared", 238 | ] 239 | 240 | [[package]] 241 | name = "foreign-types-shared" 242 | version = "0.1.1" 243 | source = "registry+https://github.com/rust-lang/crates.io-index" 244 | checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" 245 | 246 | [[package]] 247 | name = "form_urlencoded" 248 | version = "1.2.1" 249 | source = "registry+https://github.com/rust-lang/crates.io-index" 250 | checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" 251 | dependencies = [ 252 | "percent-encoding", 253 | ] 254 | 255 | [[package]] 256 | name = "futures" 257 | version = "0.3.30" 258 | source = "registry+https://github.com/rust-lang/crates.io-index" 259 | checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" 260 | dependencies = [ 261 | "futures-channel", 262 | "futures-core", 263 | "futures-executor", 264 | "futures-io", 265 | "futures-sink", 266 | "futures-task", 267 | "futures-util", 268 | ] 269 | 270 | [[package]] 271 | name = "futures-channel" 272 | version = "0.3.30" 273 | source = "registry+https://github.com/rust-lang/crates.io-index" 274 | checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" 275 | dependencies = [ 276 | "futures-core", 277 | "futures-sink", 278 | ] 279 | 280 | [[package]] 281 | name = "futures-core" 282 | version = "0.3.30" 283 | source = "registry+https://github.com/rust-lang/crates.io-index" 284 | checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" 285 | 286 | [[package]] 287 | name = "futures-executor" 288 | version = "0.3.30" 289 | source = "registry+https://github.com/rust-lang/crates.io-index" 290 | checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" 291 | dependencies = [ 292 | "futures-core", 293 | "futures-task", 294 | "futures-util", 295 | ] 296 | 297 | [[package]] 298 | name = "futures-io" 299 | version = "0.3.30" 300 | source = "registry+https://github.com/rust-lang/crates.io-index" 301 | checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" 302 | 303 | [[package]] 304 | name = "futures-macro" 305 | version = "0.3.30" 306 | source = "registry+https://github.com/rust-lang/crates.io-index" 307 | checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" 308 | dependencies = [ 309 | "proc-macro2", 310 | "quote", 311 | "syn 2.0.49", 312 | ] 313 | 314 | [[package]] 315 | name = "futures-sink" 316 | version = "0.3.30" 317 | source = "registry+https://github.com/rust-lang/crates.io-index" 318 | checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" 319 | 320 | [[package]] 321 | name = "futures-task" 322 | version = "0.3.30" 323 | source = "registry+https://github.com/rust-lang/crates.io-index" 324 | checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" 325 | 326 | [[package]] 327 | name = "futures-util" 328 | version = "0.3.30" 329 | source = "registry+https://github.com/rust-lang/crates.io-index" 330 | checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" 331 | dependencies = [ 332 | "futures-channel", 333 | "futures-core", 334 | "futures-io", 335 | "futures-macro", 336 | "futures-sink", 337 | "futures-task", 338 | "memchr", 339 | "pin-project-lite", 340 | "pin-utils", 341 | "slab", 342 | ] 343 | 344 | [[package]] 345 | name = "generic-array" 346 | version = "0.14.7" 347 | source = "registry+https://github.com/rust-lang/crates.io-index" 348 | checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" 349 | dependencies = [ 350 | "typenum", 351 | "version_check", 352 | ] 353 | 354 | [[package]] 355 | name = "getrandom" 356 | version = "0.2.12" 357 | source = "registry+https://github.com/rust-lang/crates.io-index" 358 | checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" 359 | dependencies = [ 360 | "cfg-if", 361 | "libc", 362 | "wasi", 363 | ] 364 | 365 | [[package]] 366 | name = "gimli" 367 | version = "0.28.1" 368 | source = "registry+https://github.com/rust-lang/crates.io-index" 369 | checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" 370 | 371 | [[package]] 372 | name = "h2" 373 | version = "0.4.5" 374 | source = "registry+https://github.com/rust-lang/crates.io-index" 375 | checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" 376 | dependencies = [ 377 | "atomic-waker", 378 | "bytes", 379 | "fnv", 380 | "futures-core", 381 | "futures-sink", 382 | "http", 383 | "indexmap", 384 | "slab", 385 | "tokio", 386 | "tokio-util", 387 | "tracing", 388 | ] 389 | 390 | [[package]] 391 | name = "hashbrown" 392 | version = "0.14.3" 393 | source = "registry+https://github.com/rust-lang/crates.io-index" 394 | checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" 395 | 396 | [[package]] 397 | name = "heck" 398 | version = "0.4.1" 399 | source = "registry+https://github.com/rust-lang/crates.io-index" 400 | checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" 401 | 402 | [[package]] 403 | name = "hermit-abi" 404 | version = "0.3.6" 405 | source = "registry+https://github.com/rust-lang/crates.io-index" 406 | checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" 407 | 408 | [[package]] 409 | name = "http" 410 | version = "1.1.0" 411 | source = "registry+https://github.com/rust-lang/crates.io-index" 412 | checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" 413 | dependencies = [ 414 | "bytes", 415 | "fnv", 416 | "itoa", 417 | ] 418 | 419 | [[package]] 420 | name = "http-body" 421 | version = "1.0.0" 422 | source = "registry+https://github.com/rust-lang/crates.io-index" 423 | checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" 424 | dependencies = [ 425 | "bytes", 426 | "http", 427 | ] 428 | 429 | [[package]] 430 | name = "http-body-util" 431 | version = "0.1.1" 432 | source = "registry+https://github.com/rust-lang/crates.io-index" 433 | checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" 434 | dependencies = [ 435 | "bytes", 436 | "futures-core", 437 | "http", 438 | "http-body", 439 | "pin-project-lite", 440 | ] 441 | 442 | [[package]] 443 | name = "httparse" 444 | version = "1.8.0" 445 | source = "registry+https://github.com/rust-lang/crates.io-index" 446 | checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" 447 | 448 | [[package]] 449 | name = "humantime" 450 | version = "2.1.0" 451 | source = "registry+https://github.com/rust-lang/crates.io-index" 452 | checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" 453 | 454 | [[package]] 455 | name = "hyper" 456 | version = "1.3.1" 457 | source = "registry+https://github.com/rust-lang/crates.io-index" 458 | checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" 459 | dependencies = [ 460 | "bytes", 461 | "futures-channel", 462 | "futures-util", 463 | "h2", 464 | "http", 465 | "http-body", 466 | "httparse", 467 | "itoa", 468 | "pin-project-lite", 469 | "smallvec", 470 | "tokio", 471 | "want", 472 | ] 473 | 474 | [[package]] 475 | name = "hyper-rustls" 476 | version = "0.26.0" 477 | source = "registry+https://github.com/rust-lang/crates.io-index" 478 | checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" 479 | dependencies = [ 480 | "futures-util", 481 | "http", 482 | "hyper", 483 | "hyper-util", 484 | "rustls", 485 | "rustls-pki-types", 486 | "tokio", 487 | "tokio-rustls", 488 | "tower-service", 489 | ] 490 | 491 | [[package]] 492 | name = "hyper-tls" 493 | version = "0.6.0" 494 | source = "registry+https://github.com/rust-lang/crates.io-index" 495 | checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" 496 | dependencies = [ 497 | "bytes", 498 | "http-body-util", 499 | "hyper", 500 | "hyper-util", 501 | "native-tls", 502 | "tokio", 503 | "tokio-native-tls", 504 | "tower-service", 505 | ] 506 | 507 | [[package]] 508 | name = "hyper-util" 509 | version = "0.1.5" 510 | source = "registry+https://github.com/rust-lang/crates.io-index" 511 | checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" 512 | dependencies = [ 513 | "bytes", 514 | "futures-channel", 515 | "futures-util", 516 | "http", 517 | "http-body", 518 | "hyper", 519 | "pin-project-lite", 520 | "socket2", 521 | "tokio", 522 | "tower", 523 | "tower-service", 524 | "tracing", 525 | ] 526 | 527 | [[package]] 528 | name = "iana-time-zone" 529 | version = "0.1.60" 530 | source = "registry+https://github.com/rust-lang/crates.io-index" 531 | checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" 532 | dependencies = [ 533 | "android_system_properties", 534 | "core-foundation-sys", 535 | "iana-time-zone-haiku", 536 | "js-sys", 537 | "wasm-bindgen", 538 | "windows-core", 539 | ] 540 | 541 | [[package]] 542 | name = "iana-time-zone-haiku" 543 | version = "0.1.2" 544 | source = "registry+https://github.com/rust-lang/crates.io-index" 545 | checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" 546 | dependencies = [ 547 | "cc", 548 | ] 549 | 550 | [[package]] 551 | name = "idna" 552 | version = "0.5.0" 553 | source = "registry+https://github.com/rust-lang/crates.io-index" 554 | checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" 555 | dependencies = [ 556 | "unicode-bidi", 557 | "unicode-normalization", 558 | ] 559 | 560 | [[package]] 561 | name = "indexmap" 562 | version = "2.2.3" 563 | source = "registry+https://github.com/rust-lang/crates.io-index" 564 | checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" 565 | dependencies = [ 566 | "equivalent", 567 | "hashbrown", 568 | ] 569 | 570 | [[package]] 571 | name = "indoc" 572 | version = "2.0.4" 573 | source = "registry+https://github.com/rust-lang/crates.io-index" 574 | checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" 575 | 576 | [[package]] 577 | name = "ipnet" 578 | version = "2.9.0" 579 | source = "registry+https://github.com/rust-lang/crates.io-index" 580 | checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" 581 | 582 | [[package]] 583 | name = "itertools" 584 | version = "0.13.0" 585 | source = "registry+https://github.com/rust-lang/crates.io-index" 586 | checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" 587 | dependencies = [ 588 | "either", 589 | ] 590 | 591 | [[package]] 592 | name = "itoa" 593 | version = "1.0.10" 594 | source = "registry+https://github.com/rust-lang/crates.io-index" 595 | checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" 596 | 597 | [[package]] 598 | name = "js-sys" 599 | version = "0.3.68" 600 | source = "registry+https://github.com/rust-lang/crates.io-index" 601 | checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" 602 | dependencies = [ 603 | "wasm-bindgen", 604 | ] 605 | 606 | [[package]] 607 | name = "lazy_static" 608 | version = "1.4.0" 609 | source = "registry+https://github.com/rust-lang/crates.io-index" 610 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 611 | 612 | [[package]] 613 | name = "libc" 614 | version = "0.2.153" 615 | source = "registry+https://github.com/rust-lang/crates.io-index" 616 | checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" 617 | 618 | [[package]] 619 | name = "linux-raw-sys" 620 | version = "0.4.13" 621 | source = "registry+https://github.com/rust-lang/crates.io-index" 622 | checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" 623 | 624 | [[package]] 625 | name = "lock_api" 626 | version = "0.4.11" 627 | source = "registry+https://github.com/rust-lang/crates.io-index" 628 | checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" 629 | dependencies = [ 630 | "autocfg", 631 | "scopeguard", 632 | ] 633 | 634 | [[package]] 635 | name = "log" 636 | version = "0.4.20" 637 | source = "registry+https://github.com/rust-lang/crates.io-index" 638 | checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" 639 | 640 | [[package]] 641 | name = "md-5" 642 | version = "0.10.6" 643 | source = "registry+https://github.com/rust-lang/crates.io-index" 644 | checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" 645 | dependencies = [ 646 | "cfg-if", 647 | "digest", 648 | ] 649 | 650 | [[package]] 651 | name = "memchr" 652 | version = "2.7.1" 653 | source = "registry+https://github.com/rust-lang/crates.io-index" 654 | checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" 655 | 656 | [[package]] 657 | name = "memoffset" 658 | version = "0.9.0" 659 | source = "registry+https://github.com/rust-lang/crates.io-index" 660 | checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" 661 | dependencies = [ 662 | "autocfg", 663 | ] 664 | 665 | [[package]] 666 | name = "mime" 667 | version = "0.3.17" 668 | source = "registry+https://github.com/rust-lang/crates.io-index" 669 | checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" 670 | 671 | [[package]] 672 | name = "miniz_oxide" 673 | version = "0.7.2" 674 | source = "registry+https://github.com/rust-lang/crates.io-index" 675 | checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" 676 | dependencies = [ 677 | "adler", 678 | ] 679 | 680 | [[package]] 681 | name = "mio" 682 | version = "0.8.10" 683 | source = "registry+https://github.com/rust-lang/crates.io-index" 684 | checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" 685 | dependencies = [ 686 | "libc", 687 | "wasi", 688 | "windows-sys 0.48.0", 689 | ] 690 | 691 | [[package]] 692 | name = "native-tls" 693 | version = "0.2.11" 694 | source = "registry+https://github.com/rust-lang/crates.io-index" 695 | checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" 696 | dependencies = [ 697 | "lazy_static", 698 | "libc", 699 | "log", 700 | "openssl", 701 | "openssl-probe", 702 | "openssl-sys", 703 | "schannel", 704 | "security-framework", 705 | "security-framework-sys", 706 | "tempfile", 707 | ] 708 | 709 | [[package]] 710 | name = "num-traits" 711 | version = "0.2.18" 712 | source = "registry+https://github.com/rust-lang/crates.io-index" 713 | checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" 714 | dependencies = [ 715 | "autocfg", 716 | ] 717 | 718 | [[package]] 719 | name = "num_cpus" 720 | version = "1.16.0" 721 | source = "registry+https://github.com/rust-lang/crates.io-index" 722 | checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" 723 | dependencies = [ 724 | "hermit-abi", 725 | "libc", 726 | ] 727 | 728 | [[package]] 729 | name = "object" 730 | version = "0.32.2" 731 | source = "registry+https://github.com/rust-lang/crates.io-index" 732 | checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" 733 | dependencies = [ 734 | "memchr", 735 | ] 736 | 737 | [[package]] 738 | name = "object-store-internal" 739 | version = "0.2.0" 740 | dependencies = [ 741 | "async-trait", 742 | "bytes", 743 | "futures", 744 | "object_store", 745 | "once_cell", 746 | "percent-encoding", 747 | "pyo3", 748 | "pyo3-asyncio-0-21", 749 | "reqwest", 750 | "thiserror", 751 | "tokio", 752 | "url", 753 | ] 754 | 755 | [[package]] 756 | name = "object-store-python" 757 | version = "0.2.0" 758 | dependencies = [ 759 | "object-store-internal", 760 | "pyo3", 761 | ] 762 | 763 | [[package]] 764 | name = "object_store" 765 | version = "0.10.2" 766 | source = "registry+https://github.com/rust-lang/crates.io-index" 767 | checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" 768 | dependencies = [ 769 | "async-trait", 770 | "base64 0.22.1", 771 | "bytes", 772 | "chrono", 773 | "futures", 774 | "humantime", 775 | "hyper", 776 | "itertools", 777 | "md-5", 778 | "parking_lot", 779 | "percent-encoding", 780 | "quick-xml", 781 | "rand", 782 | "reqwest", 783 | "ring", 784 | "rustls-pemfile", 785 | "serde", 786 | "serde_json", 787 | "snafu", 788 | "tokio", 789 | "tracing", 790 | "url", 791 | "walkdir", 792 | ] 793 | 794 | [[package]] 795 | name = "once_cell" 796 | version = "1.19.0" 797 | source = "registry+https://github.com/rust-lang/crates.io-index" 798 | checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" 799 | 800 | [[package]] 801 | name = "openssl" 802 | version = "0.10.63" 803 | source = "registry+https://github.com/rust-lang/crates.io-index" 804 | checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" 805 | dependencies = [ 806 | "bitflags 2.4.2", 807 | "cfg-if", 808 | "foreign-types", 809 | "libc", 810 | "once_cell", 811 | "openssl-macros", 812 | "openssl-sys", 813 | ] 814 | 815 | [[package]] 816 | name = "openssl-macros" 817 | version = "0.1.1" 818 | source = "registry+https://github.com/rust-lang/crates.io-index" 819 | checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" 820 | dependencies = [ 821 | "proc-macro2", 822 | "quote", 823 | "syn 2.0.49", 824 | ] 825 | 826 | [[package]] 827 | name = "openssl-probe" 828 | version = "0.1.5" 829 | source = "registry+https://github.com/rust-lang/crates.io-index" 830 | checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" 831 | 832 | [[package]] 833 | name = "openssl-src" 834 | version = "300.2.3+3.2.1" 835 | source = "registry+https://github.com/rust-lang/crates.io-index" 836 | checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" 837 | dependencies = [ 838 | "cc", 839 | ] 840 | 841 | [[package]] 842 | name = "openssl-sys" 843 | version = "0.9.99" 844 | source = "registry+https://github.com/rust-lang/crates.io-index" 845 | checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" 846 | dependencies = [ 847 | "cc", 848 | "libc", 849 | "openssl-src", 850 | "pkg-config", 851 | "vcpkg", 852 | ] 853 | 854 | [[package]] 855 | name = "parking_lot" 856 | version = "0.12.1" 857 | source = "registry+https://github.com/rust-lang/crates.io-index" 858 | checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" 859 | dependencies = [ 860 | "lock_api", 861 | "parking_lot_core", 862 | ] 863 | 864 | [[package]] 865 | name = "parking_lot_core" 866 | version = "0.9.9" 867 | source = "registry+https://github.com/rust-lang/crates.io-index" 868 | checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" 869 | dependencies = [ 870 | "cfg-if", 871 | "libc", 872 | "redox_syscall", 873 | "smallvec", 874 | "windows-targets 0.48.5", 875 | ] 876 | 877 | [[package]] 878 | name = "percent-encoding" 879 | version = "2.3.1" 880 | source = "registry+https://github.com/rust-lang/crates.io-index" 881 | checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" 882 | 883 | [[package]] 884 | name = "pin-project" 885 | version = "1.1.5" 886 | source = "registry+https://github.com/rust-lang/crates.io-index" 887 | checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" 888 | dependencies = [ 889 | "pin-project-internal", 890 | ] 891 | 892 | [[package]] 893 | name = "pin-project-internal" 894 | version = "1.1.5" 895 | source = "registry+https://github.com/rust-lang/crates.io-index" 896 | checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" 897 | dependencies = [ 898 | "proc-macro2", 899 | "quote", 900 | "syn 2.0.49", 901 | ] 902 | 903 | [[package]] 904 | name = "pin-project-lite" 905 | version = "0.2.13" 906 | source = "registry+https://github.com/rust-lang/crates.io-index" 907 | checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" 908 | 909 | [[package]] 910 | name = "pin-utils" 911 | version = "0.1.0" 912 | source = "registry+https://github.com/rust-lang/crates.io-index" 913 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 914 | 915 | [[package]] 916 | name = "pkg-config" 917 | version = "0.3.30" 918 | source = "registry+https://github.com/rust-lang/crates.io-index" 919 | checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" 920 | 921 | [[package]] 922 | name = "portable-atomic" 923 | version = "1.7.0" 924 | source = "registry+https://github.com/rust-lang/crates.io-index" 925 | checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" 926 | 927 | [[package]] 928 | name = "ppv-lite86" 929 | version = "0.2.17" 930 | source = "registry+https://github.com/rust-lang/crates.io-index" 931 | checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" 932 | 933 | [[package]] 934 | name = "proc-macro2" 935 | version = "1.0.78" 936 | source = "registry+https://github.com/rust-lang/crates.io-index" 937 | checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" 938 | dependencies = [ 939 | "unicode-ident", 940 | ] 941 | 942 | [[package]] 943 | name = "pyo3" 944 | version = "0.21.2" 945 | source = "registry+https://github.com/rust-lang/crates.io-index" 946 | checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" 947 | dependencies = [ 948 | "cfg-if", 949 | "indoc", 950 | "libc", 951 | "memoffset", 952 | "parking_lot", 953 | "portable-atomic", 954 | "pyo3-build-config", 955 | "pyo3-ffi", 956 | "pyo3-macros", 957 | "unindent", 958 | ] 959 | 960 | [[package]] 961 | name = "pyo3-asyncio-0-21" 962 | version = "0.21.0" 963 | source = "registry+https://github.com/rust-lang/crates.io-index" 964 | checksum = "8fde289486f7d5cee0ac7c20b2637a0657654681079cc5eedc90d9a2a79af1e5" 965 | dependencies = [ 966 | "futures", 967 | "once_cell", 968 | "pin-project-lite", 969 | "pyo3", 970 | "tokio", 971 | ] 972 | 973 | [[package]] 974 | name = "pyo3-build-config" 975 | version = "0.21.2" 976 | source = "registry+https://github.com/rust-lang/crates.io-index" 977 | checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" 978 | dependencies = [ 979 | "once_cell", 980 | "target-lexicon", 981 | ] 982 | 983 | [[package]] 984 | name = "pyo3-ffi" 985 | version = "0.21.2" 986 | source = "registry+https://github.com/rust-lang/crates.io-index" 987 | checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" 988 | dependencies = [ 989 | "libc", 990 | "pyo3-build-config", 991 | ] 992 | 993 | [[package]] 994 | name = "pyo3-macros" 995 | version = "0.21.2" 996 | source = "registry+https://github.com/rust-lang/crates.io-index" 997 | checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" 998 | dependencies = [ 999 | "proc-macro2", 1000 | "pyo3-macros-backend", 1001 | "quote", 1002 | "syn 2.0.49", 1003 | ] 1004 | 1005 | [[package]] 1006 | name = "pyo3-macros-backend" 1007 | version = "0.21.2" 1008 | source = "registry+https://github.com/rust-lang/crates.io-index" 1009 | checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" 1010 | dependencies = [ 1011 | "heck", 1012 | "proc-macro2", 1013 | "pyo3-build-config", 1014 | "quote", 1015 | "syn 2.0.49", 1016 | ] 1017 | 1018 | [[package]] 1019 | name = "quick-xml" 1020 | version = "0.36.1" 1021 | source = "registry+https://github.com/rust-lang/crates.io-index" 1022 | checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" 1023 | dependencies = [ 1024 | "memchr", 1025 | "serde", 1026 | ] 1027 | 1028 | [[package]] 1029 | name = "quote" 1030 | version = "1.0.35" 1031 | source = "registry+https://github.com/rust-lang/crates.io-index" 1032 | checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" 1033 | dependencies = [ 1034 | "proc-macro2", 1035 | ] 1036 | 1037 | [[package]] 1038 | name = "rand" 1039 | version = "0.8.5" 1040 | source = "registry+https://github.com/rust-lang/crates.io-index" 1041 | checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" 1042 | dependencies = [ 1043 | "libc", 1044 | "rand_chacha", 1045 | "rand_core", 1046 | ] 1047 | 1048 | [[package]] 1049 | name = "rand_chacha" 1050 | version = "0.3.1" 1051 | source = "registry+https://github.com/rust-lang/crates.io-index" 1052 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 1053 | dependencies = [ 1054 | "ppv-lite86", 1055 | "rand_core", 1056 | ] 1057 | 1058 | [[package]] 1059 | name = "rand_core" 1060 | version = "0.6.4" 1061 | source = "registry+https://github.com/rust-lang/crates.io-index" 1062 | checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" 1063 | dependencies = [ 1064 | "getrandom", 1065 | ] 1066 | 1067 | [[package]] 1068 | name = "redox_syscall" 1069 | version = "0.4.1" 1070 | source = "registry+https://github.com/rust-lang/crates.io-index" 1071 | checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" 1072 | dependencies = [ 1073 | "bitflags 1.3.2", 1074 | ] 1075 | 1076 | [[package]] 1077 | name = "reqwest" 1078 | version = "0.12.4" 1079 | source = "registry+https://github.com/rust-lang/crates.io-index" 1080 | checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" 1081 | dependencies = [ 1082 | "base64 0.22.1", 1083 | "bytes", 1084 | "encoding_rs", 1085 | "futures-core", 1086 | "futures-util", 1087 | "h2", 1088 | "http", 1089 | "http-body", 1090 | "http-body-util", 1091 | "hyper", 1092 | "hyper-rustls", 1093 | "hyper-tls", 1094 | "hyper-util", 1095 | "ipnet", 1096 | "js-sys", 1097 | "log", 1098 | "mime", 1099 | "native-tls", 1100 | "once_cell", 1101 | "percent-encoding", 1102 | "pin-project-lite", 1103 | "rustls", 1104 | "rustls-native-certs", 1105 | "rustls-pemfile", 1106 | "rustls-pki-types", 1107 | "serde", 1108 | "serde_json", 1109 | "serde_urlencoded", 1110 | "sync_wrapper", 1111 | "system-configuration", 1112 | "tokio", 1113 | "tokio-native-tls", 1114 | "tokio-rustls", 1115 | "tokio-util", 1116 | "tower-service", 1117 | "url", 1118 | "wasm-bindgen", 1119 | "wasm-bindgen-futures", 1120 | "wasm-streams", 1121 | "web-sys", 1122 | "winreg", 1123 | ] 1124 | 1125 | [[package]] 1126 | name = "ring" 1127 | version = "0.17.7" 1128 | source = "registry+https://github.com/rust-lang/crates.io-index" 1129 | checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" 1130 | dependencies = [ 1131 | "cc", 1132 | "getrandom", 1133 | "libc", 1134 | "spin", 1135 | "untrusted", 1136 | "windows-sys 0.48.0", 1137 | ] 1138 | 1139 | [[package]] 1140 | name = "rustc-demangle" 1141 | version = "0.1.23" 1142 | source = "registry+https://github.com/rust-lang/crates.io-index" 1143 | checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" 1144 | 1145 | [[package]] 1146 | name = "rustix" 1147 | version = "0.38.31" 1148 | source = "registry+https://github.com/rust-lang/crates.io-index" 1149 | checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" 1150 | dependencies = [ 1151 | "bitflags 2.4.2", 1152 | "errno", 1153 | "libc", 1154 | "linux-raw-sys", 1155 | "windows-sys 0.52.0", 1156 | ] 1157 | 1158 | [[package]] 1159 | name = "rustls" 1160 | version = "0.22.4" 1161 | source = "registry+https://github.com/rust-lang/crates.io-index" 1162 | checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" 1163 | dependencies = [ 1164 | "log", 1165 | "ring", 1166 | "rustls-pki-types", 1167 | "rustls-webpki", 1168 | "subtle", 1169 | "zeroize", 1170 | ] 1171 | 1172 | [[package]] 1173 | name = "rustls-native-certs" 1174 | version = "0.7.0" 1175 | source = "registry+https://github.com/rust-lang/crates.io-index" 1176 | checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" 1177 | dependencies = [ 1178 | "openssl-probe", 1179 | "rustls-pemfile", 1180 | "rustls-pki-types", 1181 | "schannel", 1182 | "security-framework", 1183 | ] 1184 | 1185 | [[package]] 1186 | name = "rustls-pemfile" 1187 | version = "2.1.0" 1188 | source = "registry+https://github.com/rust-lang/crates.io-index" 1189 | checksum = "3c333bb734fcdedcea57de1602543590f545f127dc8b533324318fd492c5c70b" 1190 | dependencies = [ 1191 | "base64 0.21.7", 1192 | "rustls-pki-types", 1193 | ] 1194 | 1195 | [[package]] 1196 | name = "rustls-pki-types" 1197 | version = "1.7.0" 1198 | source = "registry+https://github.com/rust-lang/crates.io-index" 1199 | checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" 1200 | 1201 | [[package]] 1202 | name = "rustls-webpki" 1203 | version = "0.102.4" 1204 | source = "registry+https://github.com/rust-lang/crates.io-index" 1205 | checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" 1206 | dependencies = [ 1207 | "ring", 1208 | "rustls-pki-types", 1209 | "untrusted", 1210 | ] 1211 | 1212 | [[package]] 1213 | name = "ryu" 1214 | version = "1.0.16" 1215 | source = "registry+https://github.com/rust-lang/crates.io-index" 1216 | checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" 1217 | 1218 | [[package]] 1219 | name = "same-file" 1220 | version = "1.0.6" 1221 | source = "registry+https://github.com/rust-lang/crates.io-index" 1222 | checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" 1223 | dependencies = [ 1224 | "winapi-util", 1225 | ] 1226 | 1227 | [[package]] 1228 | name = "schannel" 1229 | version = "0.1.23" 1230 | source = "registry+https://github.com/rust-lang/crates.io-index" 1231 | checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" 1232 | dependencies = [ 1233 | "windows-sys 0.52.0", 1234 | ] 1235 | 1236 | [[package]] 1237 | name = "scopeguard" 1238 | version = "1.2.0" 1239 | source = "registry+https://github.com/rust-lang/crates.io-index" 1240 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 1241 | 1242 | [[package]] 1243 | name = "security-framework" 1244 | version = "2.9.2" 1245 | source = "registry+https://github.com/rust-lang/crates.io-index" 1246 | checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" 1247 | dependencies = [ 1248 | "bitflags 1.3.2", 1249 | "core-foundation", 1250 | "core-foundation-sys", 1251 | "libc", 1252 | "security-framework-sys", 1253 | ] 1254 | 1255 | [[package]] 1256 | name = "security-framework-sys" 1257 | version = "2.9.1" 1258 | source = "registry+https://github.com/rust-lang/crates.io-index" 1259 | checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" 1260 | dependencies = [ 1261 | "core-foundation-sys", 1262 | "libc", 1263 | ] 1264 | 1265 | [[package]] 1266 | name = "serde" 1267 | version = "1.0.196" 1268 | source = "registry+https://github.com/rust-lang/crates.io-index" 1269 | checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" 1270 | dependencies = [ 1271 | "serde_derive", 1272 | ] 1273 | 1274 | [[package]] 1275 | name = "serde_derive" 1276 | version = "1.0.196" 1277 | source = "registry+https://github.com/rust-lang/crates.io-index" 1278 | checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" 1279 | dependencies = [ 1280 | "proc-macro2", 1281 | "quote", 1282 | "syn 2.0.49", 1283 | ] 1284 | 1285 | [[package]] 1286 | name = "serde_json" 1287 | version = "1.0.113" 1288 | source = "registry+https://github.com/rust-lang/crates.io-index" 1289 | checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" 1290 | dependencies = [ 1291 | "itoa", 1292 | "ryu", 1293 | "serde", 1294 | ] 1295 | 1296 | [[package]] 1297 | name = "serde_urlencoded" 1298 | version = "0.7.1" 1299 | source = "registry+https://github.com/rust-lang/crates.io-index" 1300 | checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" 1301 | dependencies = [ 1302 | "form_urlencoded", 1303 | "itoa", 1304 | "ryu", 1305 | "serde", 1306 | ] 1307 | 1308 | [[package]] 1309 | name = "slab" 1310 | version = "0.4.9" 1311 | source = "registry+https://github.com/rust-lang/crates.io-index" 1312 | checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" 1313 | dependencies = [ 1314 | "autocfg", 1315 | ] 1316 | 1317 | [[package]] 1318 | name = "smallvec" 1319 | version = "1.13.1" 1320 | source = "registry+https://github.com/rust-lang/crates.io-index" 1321 | checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" 1322 | 1323 | [[package]] 1324 | name = "snafu" 1325 | version = "0.7.5" 1326 | source = "registry+https://github.com/rust-lang/crates.io-index" 1327 | checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" 1328 | dependencies = [ 1329 | "doc-comment", 1330 | "snafu-derive", 1331 | ] 1332 | 1333 | [[package]] 1334 | name = "snafu-derive" 1335 | version = "0.7.5" 1336 | source = "registry+https://github.com/rust-lang/crates.io-index" 1337 | checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" 1338 | dependencies = [ 1339 | "heck", 1340 | "proc-macro2", 1341 | "quote", 1342 | "syn 1.0.109", 1343 | ] 1344 | 1345 | [[package]] 1346 | name = "socket2" 1347 | version = "0.5.5" 1348 | source = "registry+https://github.com/rust-lang/crates.io-index" 1349 | checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" 1350 | dependencies = [ 1351 | "libc", 1352 | "windows-sys 0.48.0", 1353 | ] 1354 | 1355 | [[package]] 1356 | name = "spin" 1357 | version = "0.9.8" 1358 | source = "registry+https://github.com/rust-lang/crates.io-index" 1359 | checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" 1360 | 1361 | [[package]] 1362 | name = "subtle" 1363 | version = "2.5.0" 1364 | source = "registry+https://github.com/rust-lang/crates.io-index" 1365 | checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" 1366 | 1367 | [[package]] 1368 | name = "syn" 1369 | version = "1.0.109" 1370 | source = "registry+https://github.com/rust-lang/crates.io-index" 1371 | checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" 1372 | dependencies = [ 1373 | "proc-macro2", 1374 | "quote", 1375 | "unicode-ident", 1376 | ] 1377 | 1378 | [[package]] 1379 | name = "syn" 1380 | version = "2.0.49" 1381 | source = "registry+https://github.com/rust-lang/crates.io-index" 1382 | checksum = "915aea9e586f80826ee59f8453c1101f9d1c4b3964cd2460185ee8e299ada496" 1383 | dependencies = [ 1384 | "proc-macro2", 1385 | "quote", 1386 | "unicode-ident", 1387 | ] 1388 | 1389 | [[package]] 1390 | name = "sync_wrapper" 1391 | version = "0.1.2" 1392 | source = "registry+https://github.com/rust-lang/crates.io-index" 1393 | checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" 1394 | 1395 | [[package]] 1396 | name = "system-configuration" 1397 | version = "0.5.1" 1398 | source = "registry+https://github.com/rust-lang/crates.io-index" 1399 | checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" 1400 | dependencies = [ 1401 | "bitflags 1.3.2", 1402 | "core-foundation", 1403 | "system-configuration-sys", 1404 | ] 1405 | 1406 | [[package]] 1407 | name = "system-configuration-sys" 1408 | version = "0.5.0" 1409 | source = "registry+https://github.com/rust-lang/crates.io-index" 1410 | checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" 1411 | dependencies = [ 1412 | "core-foundation-sys", 1413 | "libc", 1414 | ] 1415 | 1416 | [[package]] 1417 | name = "target-lexicon" 1418 | version = "0.12.13" 1419 | source = "registry+https://github.com/rust-lang/crates.io-index" 1420 | checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" 1421 | 1422 | [[package]] 1423 | name = "tempfile" 1424 | version = "3.10.0" 1425 | source = "registry+https://github.com/rust-lang/crates.io-index" 1426 | checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" 1427 | dependencies = [ 1428 | "cfg-if", 1429 | "fastrand", 1430 | "rustix", 1431 | "windows-sys 0.52.0", 1432 | ] 1433 | 1434 | [[package]] 1435 | name = "thiserror" 1436 | version = "1.0.57" 1437 | source = "registry+https://github.com/rust-lang/crates.io-index" 1438 | checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" 1439 | dependencies = [ 1440 | "thiserror-impl", 1441 | ] 1442 | 1443 | [[package]] 1444 | name = "thiserror-impl" 1445 | version = "1.0.57" 1446 | source = "registry+https://github.com/rust-lang/crates.io-index" 1447 | checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" 1448 | dependencies = [ 1449 | "proc-macro2", 1450 | "quote", 1451 | "syn 2.0.49", 1452 | ] 1453 | 1454 | [[package]] 1455 | name = "tinyvec" 1456 | version = "1.6.0" 1457 | source = "registry+https://github.com/rust-lang/crates.io-index" 1458 | checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" 1459 | dependencies = [ 1460 | "tinyvec_macros", 1461 | ] 1462 | 1463 | [[package]] 1464 | name = "tinyvec_macros" 1465 | version = "0.1.1" 1466 | source = "registry+https://github.com/rust-lang/crates.io-index" 1467 | checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" 1468 | 1469 | [[package]] 1470 | name = "tokio" 1471 | version = "1.36.0" 1472 | source = "registry+https://github.com/rust-lang/crates.io-index" 1473 | checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" 1474 | dependencies = [ 1475 | "backtrace", 1476 | "bytes", 1477 | "libc", 1478 | "mio", 1479 | "num_cpus", 1480 | "pin-project-lite", 1481 | "socket2", 1482 | "tokio-macros", 1483 | "windows-sys 0.48.0", 1484 | ] 1485 | 1486 | [[package]] 1487 | name = "tokio-macros" 1488 | version = "2.2.0" 1489 | source = "registry+https://github.com/rust-lang/crates.io-index" 1490 | checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" 1491 | dependencies = [ 1492 | "proc-macro2", 1493 | "quote", 1494 | "syn 2.0.49", 1495 | ] 1496 | 1497 | [[package]] 1498 | name = "tokio-native-tls" 1499 | version = "0.3.1" 1500 | source = "registry+https://github.com/rust-lang/crates.io-index" 1501 | checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" 1502 | dependencies = [ 1503 | "native-tls", 1504 | "tokio", 1505 | ] 1506 | 1507 | [[package]] 1508 | name = "tokio-rustls" 1509 | version = "0.25.0" 1510 | source = "registry+https://github.com/rust-lang/crates.io-index" 1511 | checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" 1512 | dependencies = [ 1513 | "rustls", 1514 | "rustls-pki-types", 1515 | "tokio", 1516 | ] 1517 | 1518 | [[package]] 1519 | name = "tokio-util" 1520 | version = "0.7.10" 1521 | source = "registry+https://github.com/rust-lang/crates.io-index" 1522 | checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" 1523 | dependencies = [ 1524 | "bytes", 1525 | "futures-core", 1526 | "futures-sink", 1527 | "pin-project-lite", 1528 | "tokio", 1529 | "tracing", 1530 | ] 1531 | 1532 | [[package]] 1533 | name = "tower" 1534 | version = "0.4.13" 1535 | source = "registry+https://github.com/rust-lang/crates.io-index" 1536 | checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" 1537 | dependencies = [ 1538 | "futures-core", 1539 | "futures-util", 1540 | "pin-project", 1541 | "pin-project-lite", 1542 | "tokio", 1543 | "tower-layer", 1544 | "tower-service", 1545 | ] 1546 | 1547 | [[package]] 1548 | name = "tower-layer" 1549 | version = "0.3.2" 1550 | source = "registry+https://github.com/rust-lang/crates.io-index" 1551 | checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" 1552 | 1553 | [[package]] 1554 | name = "tower-service" 1555 | version = "0.3.2" 1556 | source = "registry+https://github.com/rust-lang/crates.io-index" 1557 | checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" 1558 | 1559 | [[package]] 1560 | name = "tracing" 1561 | version = "0.1.40" 1562 | source = "registry+https://github.com/rust-lang/crates.io-index" 1563 | checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" 1564 | dependencies = [ 1565 | "pin-project-lite", 1566 | "tracing-attributes", 1567 | "tracing-core", 1568 | ] 1569 | 1570 | [[package]] 1571 | name = "tracing-attributes" 1572 | version = "0.1.27" 1573 | source = "registry+https://github.com/rust-lang/crates.io-index" 1574 | checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" 1575 | dependencies = [ 1576 | "proc-macro2", 1577 | "quote", 1578 | "syn 2.0.49", 1579 | ] 1580 | 1581 | [[package]] 1582 | name = "tracing-core" 1583 | version = "0.1.32" 1584 | source = "registry+https://github.com/rust-lang/crates.io-index" 1585 | checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" 1586 | dependencies = [ 1587 | "once_cell", 1588 | ] 1589 | 1590 | [[package]] 1591 | name = "try-lock" 1592 | version = "0.2.5" 1593 | source = "registry+https://github.com/rust-lang/crates.io-index" 1594 | checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" 1595 | 1596 | [[package]] 1597 | name = "typenum" 1598 | version = "1.17.0" 1599 | source = "registry+https://github.com/rust-lang/crates.io-index" 1600 | checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" 1601 | 1602 | [[package]] 1603 | name = "unicode-bidi" 1604 | version = "0.3.15" 1605 | source = "registry+https://github.com/rust-lang/crates.io-index" 1606 | checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" 1607 | 1608 | [[package]] 1609 | name = "unicode-ident" 1610 | version = "1.0.12" 1611 | source = "registry+https://github.com/rust-lang/crates.io-index" 1612 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 1613 | 1614 | [[package]] 1615 | name = "unicode-normalization" 1616 | version = "0.1.22" 1617 | source = "registry+https://github.com/rust-lang/crates.io-index" 1618 | checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" 1619 | dependencies = [ 1620 | "tinyvec", 1621 | ] 1622 | 1623 | [[package]] 1624 | name = "unindent" 1625 | version = "0.2.3" 1626 | source = "registry+https://github.com/rust-lang/crates.io-index" 1627 | checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" 1628 | 1629 | [[package]] 1630 | name = "untrusted" 1631 | version = "0.9.0" 1632 | source = "registry+https://github.com/rust-lang/crates.io-index" 1633 | checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" 1634 | 1635 | [[package]] 1636 | name = "url" 1637 | version = "2.5.0" 1638 | source = "registry+https://github.com/rust-lang/crates.io-index" 1639 | checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" 1640 | dependencies = [ 1641 | "form_urlencoded", 1642 | "idna", 1643 | "percent-encoding", 1644 | ] 1645 | 1646 | [[package]] 1647 | name = "vcpkg" 1648 | version = "0.2.15" 1649 | source = "registry+https://github.com/rust-lang/crates.io-index" 1650 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 1651 | 1652 | [[package]] 1653 | name = "version_check" 1654 | version = "0.9.4" 1655 | source = "registry+https://github.com/rust-lang/crates.io-index" 1656 | checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" 1657 | 1658 | [[package]] 1659 | name = "walkdir" 1660 | version = "2.4.0" 1661 | source = "registry+https://github.com/rust-lang/crates.io-index" 1662 | checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" 1663 | dependencies = [ 1664 | "same-file", 1665 | "winapi-util", 1666 | ] 1667 | 1668 | [[package]] 1669 | name = "want" 1670 | version = "0.3.1" 1671 | source = "registry+https://github.com/rust-lang/crates.io-index" 1672 | checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" 1673 | dependencies = [ 1674 | "try-lock", 1675 | ] 1676 | 1677 | [[package]] 1678 | name = "wasi" 1679 | version = "0.11.0+wasi-snapshot-preview1" 1680 | source = "registry+https://github.com/rust-lang/crates.io-index" 1681 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 1682 | 1683 | [[package]] 1684 | name = "wasm-bindgen" 1685 | version = "0.2.91" 1686 | source = "registry+https://github.com/rust-lang/crates.io-index" 1687 | checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" 1688 | dependencies = [ 1689 | "cfg-if", 1690 | "wasm-bindgen-macro", 1691 | ] 1692 | 1693 | [[package]] 1694 | name = "wasm-bindgen-backend" 1695 | version = "0.2.91" 1696 | source = "registry+https://github.com/rust-lang/crates.io-index" 1697 | checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" 1698 | dependencies = [ 1699 | "bumpalo", 1700 | "log", 1701 | "once_cell", 1702 | "proc-macro2", 1703 | "quote", 1704 | "syn 2.0.49", 1705 | "wasm-bindgen-shared", 1706 | ] 1707 | 1708 | [[package]] 1709 | name = "wasm-bindgen-futures" 1710 | version = "0.4.41" 1711 | source = "registry+https://github.com/rust-lang/crates.io-index" 1712 | checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" 1713 | dependencies = [ 1714 | "cfg-if", 1715 | "js-sys", 1716 | "wasm-bindgen", 1717 | "web-sys", 1718 | ] 1719 | 1720 | [[package]] 1721 | name = "wasm-bindgen-macro" 1722 | version = "0.2.91" 1723 | source = "registry+https://github.com/rust-lang/crates.io-index" 1724 | checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" 1725 | dependencies = [ 1726 | "quote", 1727 | "wasm-bindgen-macro-support", 1728 | ] 1729 | 1730 | [[package]] 1731 | name = "wasm-bindgen-macro-support" 1732 | version = "0.2.91" 1733 | source = "registry+https://github.com/rust-lang/crates.io-index" 1734 | checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" 1735 | dependencies = [ 1736 | "proc-macro2", 1737 | "quote", 1738 | "syn 2.0.49", 1739 | "wasm-bindgen-backend", 1740 | "wasm-bindgen-shared", 1741 | ] 1742 | 1743 | [[package]] 1744 | name = "wasm-bindgen-shared" 1745 | version = "0.2.91" 1746 | source = "registry+https://github.com/rust-lang/crates.io-index" 1747 | checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" 1748 | 1749 | [[package]] 1750 | name = "wasm-streams" 1751 | version = "0.4.0" 1752 | source = "registry+https://github.com/rust-lang/crates.io-index" 1753 | checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" 1754 | dependencies = [ 1755 | "futures-util", 1756 | "js-sys", 1757 | "wasm-bindgen", 1758 | "wasm-bindgen-futures", 1759 | "web-sys", 1760 | ] 1761 | 1762 | [[package]] 1763 | name = "web-sys" 1764 | version = "0.3.68" 1765 | source = "registry+https://github.com/rust-lang/crates.io-index" 1766 | checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" 1767 | dependencies = [ 1768 | "js-sys", 1769 | "wasm-bindgen", 1770 | ] 1771 | 1772 | [[package]] 1773 | name = "winapi" 1774 | version = "0.3.9" 1775 | source = "registry+https://github.com/rust-lang/crates.io-index" 1776 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 1777 | dependencies = [ 1778 | "winapi-i686-pc-windows-gnu", 1779 | "winapi-x86_64-pc-windows-gnu", 1780 | ] 1781 | 1782 | [[package]] 1783 | name = "winapi-i686-pc-windows-gnu" 1784 | version = "0.4.0" 1785 | source = "registry+https://github.com/rust-lang/crates.io-index" 1786 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 1787 | 1788 | [[package]] 1789 | name = "winapi-util" 1790 | version = "0.1.6" 1791 | source = "registry+https://github.com/rust-lang/crates.io-index" 1792 | checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" 1793 | dependencies = [ 1794 | "winapi", 1795 | ] 1796 | 1797 | [[package]] 1798 | name = "winapi-x86_64-pc-windows-gnu" 1799 | version = "0.4.0" 1800 | source = "registry+https://github.com/rust-lang/crates.io-index" 1801 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 1802 | 1803 | [[package]] 1804 | name = "windows-core" 1805 | version = "0.52.0" 1806 | source = "registry+https://github.com/rust-lang/crates.io-index" 1807 | checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" 1808 | dependencies = [ 1809 | "windows-targets 0.52.0", 1810 | ] 1811 | 1812 | [[package]] 1813 | name = "windows-sys" 1814 | version = "0.48.0" 1815 | source = "registry+https://github.com/rust-lang/crates.io-index" 1816 | checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" 1817 | dependencies = [ 1818 | "windows-targets 0.48.5", 1819 | ] 1820 | 1821 | [[package]] 1822 | name = "windows-sys" 1823 | version = "0.52.0" 1824 | source = "registry+https://github.com/rust-lang/crates.io-index" 1825 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" 1826 | dependencies = [ 1827 | "windows-targets 0.52.0", 1828 | ] 1829 | 1830 | [[package]] 1831 | name = "windows-targets" 1832 | version = "0.48.5" 1833 | source = "registry+https://github.com/rust-lang/crates.io-index" 1834 | checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" 1835 | dependencies = [ 1836 | "windows_aarch64_gnullvm 0.48.5", 1837 | "windows_aarch64_msvc 0.48.5", 1838 | "windows_i686_gnu 0.48.5", 1839 | "windows_i686_msvc 0.48.5", 1840 | "windows_x86_64_gnu 0.48.5", 1841 | "windows_x86_64_gnullvm 0.48.5", 1842 | "windows_x86_64_msvc 0.48.5", 1843 | ] 1844 | 1845 | [[package]] 1846 | name = "windows-targets" 1847 | version = "0.52.0" 1848 | source = "registry+https://github.com/rust-lang/crates.io-index" 1849 | checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" 1850 | dependencies = [ 1851 | "windows_aarch64_gnullvm 0.52.0", 1852 | "windows_aarch64_msvc 0.52.0", 1853 | "windows_i686_gnu 0.52.0", 1854 | "windows_i686_msvc 0.52.0", 1855 | "windows_x86_64_gnu 0.52.0", 1856 | "windows_x86_64_gnullvm 0.52.0", 1857 | "windows_x86_64_msvc 0.52.0", 1858 | ] 1859 | 1860 | [[package]] 1861 | name = "windows_aarch64_gnullvm" 1862 | version = "0.48.5" 1863 | source = "registry+https://github.com/rust-lang/crates.io-index" 1864 | checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" 1865 | 1866 | [[package]] 1867 | name = "windows_aarch64_gnullvm" 1868 | version = "0.52.0" 1869 | source = "registry+https://github.com/rust-lang/crates.io-index" 1870 | checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" 1871 | 1872 | [[package]] 1873 | name = "windows_aarch64_msvc" 1874 | version = "0.48.5" 1875 | source = "registry+https://github.com/rust-lang/crates.io-index" 1876 | checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" 1877 | 1878 | [[package]] 1879 | name = "windows_aarch64_msvc" 1880 | version = "0.52.0" 1881 | source = "registry+https://github.com/rust-lang/crates.io-index" 1882 | checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" 1883 | 1884 | [[package]] 1885 | name = "windows_i686_gnu" 1886 | version = "0.48.5" 1887 | source = "registry+https://github.com/rust-lang/crates.io-index" 1888 | checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" 1889 | 1890 | [[package]] 1891 | name = "windows_i686_gnu" 1892 | version = "0.52.0" 1893 | source = "registry+https://github.com/rust-lang/crates.io-index" 1894 | checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" 1895 | 1896 | [[package]] 1897 | name = "windows_i686_msvc" 1898 | version = "0.48.5" 1899 | source = "registry+https://github.com/rust-lang/crates.io-index" 1900 | checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" 1901 | 1902 | [[package]] 1903 | name = "windows_i686_msvc" 1904 | version = "0.52.0" 1905 | source = "registry+https://github.com/rust-lang/crates.io-index" 1906 | checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" 1907 | 1908 | [[package]] 1909 | name = "windows_x86_64_gnu" 1910 | version = "0.48.5" 1911 | source = "registry+https://github.com/rust-lang/crates.io-index" 1912 | checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" 1913 | 1914 | [[package]] 1915 | name = "windows_x86_64_gnu" 1916 | version = "0.52.0" 1917 | source = "registry+https://github.com/rust-lang/crates.io-index" 1918 | checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" 1919 | 1920 | [[package]] 1921 | name = "windows_x86_64_gnullvm" 1922 | version = "0.48.5" 1923 | source = "registry+https://github.com/rust-lang/crates.io-index" 1924 | checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" 1925 | 1926 | [[package]] 1927 | name = "windows_x86_64_gnullvm" 1928 | version = "0.52.0" 1929 | source = "registry+https://github.com/rust-lang/crates.io-index" 1930 | checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" 1931 | 1932 | [[package]] 1933 | name = "windows_x86_64_msvc" 1934 | version = "0.48.5" 1935 | source = "registry+https://github.com/rust-lang/crates.io-index" 1936 | checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" 1937 | 1938 | [[package]] 1939 | name = "windows_x86_64_msvc" 1940 | version = "0.52.0" 1941 | source = "registry+https://github.com/rust-lang/crates.io-index" 1942 | checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" 1943 | 1944 | [[package]] 1945 | name = "winreg" 1946 | version = "0.52.0" 1947 | source = "registry+https://github.com/rust-lang/crates.io-index" 1948 | checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" 1949 | dependencies = [ 1950 | "cfg-if", 1951 | "windows-sys 0.48.0", 1952 | ] 1953 | 1954 | [[package]] 1955 | name = "zeroize" 1956 | version = "1.8.1" 1957 | source = "registry+https://github.com/rust-lang/crates.io-index" 1958 | checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" 1959 | --------------------------------------------------------------------------------