├── .github └── workflows │ ├── publish-to-pypi.yml │ └── publish-to-readthedocs.yml ├── .gitignore ├── .gitleaks.toml ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CONTRIBUTING.md ├── LICENSE.txt ├── MANIFEST.in ├── Makefile ├── README-development.rst ├── README.md ├── SECURITY.md ├── THIRD_PARTY_LICENSES.txt ├── docs ├── Makefile ├── make.bat ├── requirements.txt └── source │ ├── conf.py │ ├── faqs.rst │ ├── faqs_title.rst │ ├── getting-connected.ipynb │ ├── getting-started.ipynb │ ├── index.rst │ ├── modules.rst │ ├── ocifs.rst │ ├── release_notes.rst │ ├── telemetry.rst │ └── unix-operations.ipynb ├── ocifs ├── __init__.py ├── core.py ├── data_lake │ ├── __init__.py │ ├── lake_mount.py │ ├── lake_sharing_client.py │ ├── lake_sharing_object_storage_client.py │ ├── lakehouse.py │ ├── lakehouse_client.py │ ├── managed_prefix_collection.py │ ├── managed_prefix_summary.py │ ├── mount_specification.py │ ├── par_response.py │ └── rename_object_details.py ├── errors.py ├── tests │ ├── __init__.py │ ├── test-requirements.txt │ ├── test_integration.py │ ├── test_integration_lake.py │ ├── test_spec.py │ └── test_spec_lake.py └── utils.py ├── pyproject.toml ├── ruff.toml ├── sbom_generation.yaml ├── setup.cfg └── setup.py /.github/workflows/publish-to-pypi.yml: -------------------------------------------------------------------------------- 1 | name: "[DO NOT TRIGGER] Publish to PyPI" 2 | 3 | # To run this workflow manually from the Actions tab 4 | on: workflow_dispatch 5 | 6 | jobs: 7 | build-n-publish: 8 | name: Build and publish Python 🐍 distribution 📦 to PyPI 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v4 13 | - name: Set up Python 14 | uses: actions/setup-python@v5 15 | with: 16 | python-version: "3.x" 17 | - name: Build distribution 📦 18 | run: | 19 | pip install build 20 | make dist 21 | - name: Validate 22 | run: | 23 | pip install dist/*.whl 24 | python -c "from ocifs import OCIFileSystem;" 25 | ## To run publish to test PyPI a secret with token needs to be added, 26 | ## this one GH_OCIFS_TESTPYPI_TOKEN - removed after initial test. 27 | ## regular name is occupied by former developer and can't be used for testing 28 | # - name: Publish distribution 📦 to Test PyPI 29 | # env: 30 | # TWINE_USERNAME: __token__ 31 | # TWINE_PASSWORD: ${{ secrets.GH_OCIFS_TESTPYPI_TOKEN }} 32 | # run: | 33 | # pip install twine 34 | # twine upload -r testpypi dist/* -u $TWINE_USERNAME -p $TWINE_PASSWORD 35 | - name: Publish distribution 📦 to PyPI 36 | env: 37 | TWINE_USERNAME: __token__ 38 | TWINE_PASSWORD: ${{ secrets.GH_OCIFS_PYPI_TOKEN}} 39 | run: | 40 | pip install twine 41 | twine upload dist/* -u $TWINE_USERNAME -p $TWINE_PASSWORD 42 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-readthedocs.yml: -------------------------------------------------------------------------------- 1 | name: "Publish Docs" 2 | 3 | # To run this workflow manually from the Actions tab 4 | on: 5 | # Auto-trigger this workflow on tag creation 6 | workflow_dispatch: 7 | push: 8 | tags: 9 | - 'v*.*.*' 10 | 11 | env: 12 | RTDS_OCIFS_PROJECT: https://readthedocs.org/api/v3/projects/ocifs 13 | RTDS_OCIFS_TOKEN: ${{ secrets.RTDS_OCIFS_TOKEN }} 14 | 15 | jobs: 16 | build-n-publish: 17 | name: Build and publish Docs 📖 to Readthedocs 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - name: When tag 🏷️ pushed - Trigger Readthedocs build 22 | if: github.event_name == 'push' && startsWith(github.ref_name, 'v') 23 | run: | 24 | # trigger build/publish of latest version 25 | curl \ 26 | -X POST \ 27 | -H "Authorization: Token $RTDS_OCIFS_TOKEN" $RTDS_OCIFS_PROJECT/versions/latest/builds/ 28 | # add 15 minutes wait time for readthedocs see freshly created tag 29 | sleep 15m 30 | # trigger build/publish of v*.*.* version 31 | curl \ 32 | -X POST \ 33 | -H "Authorization: Token $RTDS_OCIFS_TOKEN" $RTDS_OCIFS_PROJECT/versions/${{ github.ref_name }}/builds/ 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # DS Store 10 | .DS_Store 11 | 12 | # OCI Build Service 13 | input_ocibuild/ 14 | output_ocibuild_packagewheel/ 15 | output_ocibuild_testwheel/ 16 | 17 | # Distribution / packaging 18 | .Python 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | wheels/ 31 | pip-wheel-metadata/ 32 | share/python-wheels/ 33 | *.egg-info/ 34 | .installed.cfg 35 | *.egg 36 | MANIFEST 37 | 38 | # PyInstaller 39 | # Usually these files are written by a python script from a template 40 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 41 | *.manifest 42 | *.spec 43 | 44 | # Installer logs 45 | pip-log.txt 46 | pip-delete-this-directory.txt 47 | 48 | # Unit test / coverage reports 49 | htmlcov/ 50 | .tox/ 51 | .nox/ 52 | .coverage 53 | .coverage.* 54 | .cache 55 | nosetests.xml 56 | coverage.xml 57 | *.cover 58 | *.py,cover 59 | .hypothesis/ 60 | .pytest_cache/ 61 | 62 | # Translations 63 | *.mo 64 | *.pot 65 | 66 | # Django stuff: 67 | *.log 68 | local_settings.py 69 | db.sqlite3 70 | db.sqlite3-journal 71 | 72 | # Flask stuff: 73 | instance/ 74 | .webassets-cache 75 | 76 | # Scrapy stuff: 77 | .scrapy 78 | 79 | # Sphinx documentation 80 | docs/_build/ 81 | 82 | # PyBuilder 83 | target/ 84 | 85 | # Jupyter Notebook 86 | .ipynb_checkpoints 87 | 88 | # IPython 89 | profile_default/ 90 | ipython_config.py 91 | 92 | # pyenv 93 | .python-version 94 | 95 | # pipenv 96 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 97 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 98 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 99 | # install all needed dependencies. 100 | #Pipfile.lock 101 | 102 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 103 | __pypackages__/ 104 | 105 | # Celery stuff 106 | celerybeat-schedule 107 | celerybeat.pid 108 | 109 | # SageMath parsed files 110 | *.sage.py 111 | 112 | # Environments 113 | .env 114 | .venv 115 | env/ 116 | venv/ 117 | ENV/ 118 | env.bak/ 119 | venv.bak/ 120 | 121 | # Spyder project settings 122 | .spyderproject 123 | .spyproject 124 | 125 | # Rope project settings 126 | .ropeproject 127 | 128 | # mkdocs documentation 129 | /site 130 | 131 | # mypy 132 | .mypy_cache/ 133 | .dmypy.json 134 | dmypy.json 135 | 136 | # Pyre type checker 137 | .pyre/ 138 | 139 | # IntelliJ XML 140 | .idea/** 141 | 142 | # VSCode 143 | .vscode/* 144 | -------------------------------------------------------------------------------- /.gitleaks.toml: -------------------------------------------------------------------------------- 1 | title = "Gitleaks Config" 2 | 3 | # Gitleaks feature, extending the existing base config from: 4 | # https://github.com/zricethezav/gitleaks/blob/master/config/gitleaks.toml 5 | [extend] 6 | useDefault = true 7 | 8 | # Allowlist's 'stopwords' and 'regexes' excludes any secrets or mathching patterns from the current repository. 9 | # Paths listed in allowlist will not be scanned. 10 | [allowlist] 11 | description = "Global allow list" 12 | stopwords = ["test_password", "sample_key"] 13 | regexes = [ 14 | '''example-password''', 15 | '''this-is-not-the-secret''', 16 | '''''', 17 | '''security_token''', 18 | ] 19 | 20 | # Describe rule to search real ocids 21 | [[rules]] 22 | description = "Real ocids" 23 | id = "ocid" 24 | regex = '''ocid[123]\.[a-z1-9A-Z]*\.oc\d\.[a-z1-9A-Z]*\.[a-z1-9A-Z]+''' 25 | keywords = [ 26 | "ocid" 27 | ] 28 | 29 | # Describe rule to search generic secrets 30 | [[rules]] 31 | description = "Generic secret" 32 | id = "generic-secret" 33 | regex = '''(?i)((key|api|token|secret|passwd|password|psw|pass|pswd)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z!@#$%^&*<>\\\-_.=]{3,100})['\"]''' 34 | entropy = 0 35 | secretGroup = 4 36 | keywords = [ 37 | "key","api","token","secret","passwd","password", "psw", "pass", "pswd" 38 | ] 39 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.4.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-docstring-first 8 | - id: check-yaml 9 | - id: check-added-large-files 10 | # ruff 11 | - repo: https://github.com/astral-sh/ruff-pre-commit 12 | rev: v0.4.9 13 | hooks: 14 | - id: ruff 15 | args: [ --fix ] 16 | files: ^ads 17 | exclude: ^docs/ 18 | - id: ruff-format 19 | exclude: ^docs/ 20 | # Hardcoded secrets and ocids detector 21 | - repo: https://github.com/gitleaks/gitleaks 22 | rev: v8.17.0 23 | hooks: 24 | - id: gitleaks 25 | # Oracle copyright checker 26 | - repo: https://github.com/oracle-samples/oci-data-science-ai-samples/ 27 | rev: 1bc5270a443b791c62f634233c0f4966dfcc0dd6 28 | hooks: 29 | - id: check-copyright 30 | name: check-copyright 31 | entry: .pre-commit-scripts/check-copyright.py 32 | language: script 33 | types_or: ['python', 'shell', 'bash'] 34 | exclude: ^docs/ 35 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version and other tools you might need 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.8" 12 | 13 | # Build documentation in the "docs/" directory with Sphinx 14 | sphinx: 15 | configuration: docs/source/conf.py 16 | # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs 17 | # builder: "dirhtml" 18 | # Fail on all warnings to avoid broken references 19 | # fail_on_warning: true 20 | 21 | # Optionally build your docs in additional formats such as PDF and ePub 22 | # formats: 23 | # - pdf 24 | # - epub 25 | 26 | # Optional but recommended, declare the Python requirements required 27 | # to build your documentation 28 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 29 | python: 30 | install: 31 | - requirements: docs/requirements.txt 32 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to this repository 2 | 3 | We welcome your contributions! There are multiple ways to contribute. 4 | 5 | ## Opening issues 6 | 7 | For bugs or enhancement requests, please file a GitHub issue unless it's 8 | security related. When filing a bug remember that the better written the bug is, 9 | the more likely it is to be fixed. If you think you've found a security 10 | vulnerability, do not raise a GitHub issue and follow the instructions in our 11 | [security policy](./SECURITY.md). 12 | 13 | ## Contributing code 14 | 15 | We welcome your code contributions. Before submitting code via a pull request, 16 | you will need to have signed the [Oracle Contributor Agreement][OCA] (OCA) and 17 | your commits need to include the following line using the name and e-mail 18 | address you used to sign the OCA: 19 | 20 | ```text 21 | Signed-off-by: Your Name 22 | ``` 23 | 24 | This can be automatically added to pull requests by committing with `--sign-off` 25 | or `-s`, e.g. 26 | 27 | ```text 28 | git commit --signoff 29 | ``` 30 | 31 | Only pull requests from committers that can be verified as having signed the OCA 32 | can be accepted. 33 | 34 | ## Pull request process 35 | 36 | 1. Ensure there is an issue created to track and discuss the fix or enhancement 37 | you intend to submit. 38 | 1. Fork this repository 39 | 1. Create a branch in your fork to implement the changes. We recommend using 40 | the issue number as part of your branch name, e.g. `1234-fixes` 41 | 1. Ensure that any documentation is updated with the changes that are required 42 | by your change. 43 | 1. Ensure that any samples are updated if the base image has been changed. 44 | 1. Submit the pull request. *Do not leave the pull request blank*. Explain exactly 45 | what your changes are meant to do and provide simple steps on how to validate 46 | your changes. Ensure that you reference the issue you created as well. 47 | 1. We will assign the pull request to 2-3 people for review before it is merged. 48 | 49 | ## Code of conduct 50 | 51 | Follow the [Golden Rule](https://en.wikipedia.org/wiki/Golden_Rule). If you'd 52 | like more specific guidelines, see the [Contributor Covenant Code of Conduct][COC]. 53 | 54 | [OCA]: https://oca.opensource.oracle.com 55 | [COC]: https://www.contributor-covenant.org/version/1/4/code-of-conduct/ 56 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021, 2022 Oracle and/or its affiliates. 2 | 3 | The Universal Permissive License (UPL), Version 1.0 4 | 5 | Subject to the condition set forth below, permission is hereby granted to any 6 | person obtaining a copy of this software, associated documentation and/or data 7 | (collectively the "Software"), free of charge and under any and all copyright 8 | rights in the Software, and any and all patent rights owned or freely 9 | licensable by each licensor hereunder covering either (i) the unmodified 10 | Software as contributed to or provided by such licensor, or (ii) the Larger 11 | Works (as defined below), to deal in both 12 | 13 | (a) the Software, and 14 | (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if 15 | one is included with the Software (each a "Larger Work" to which the Software 16 | is contributed by such licensors), 17 | 18 | without restriction, including without limitation the rights to copy, create 19 | derivative works of, display, perform, and distribute the Software and make, 20 | use, sell, offer for sale, import, export, have made, and have sold the 21 | Software and the Larger Work(s), and to sublicense the foregoing rights on 22 | either these or other terms. 23 | 24 | This license is subject to the following condition: 25 | The above copyright notice and either this complete permission notice or at 26 | a minimum a reference to the UPL must be included in all copies or 27 | substantial portions of the Software. 28 | 29 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 35 | SOFTWARE. 36 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.txt 2 | include THIRD_PARTY_LICENSES.txt 3 | exclude docs/** 4 | exclude Makefile 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, 2022 Oracle and/or its affiliates. 2 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 3 | 4 | dist: clean 5 | @python3 -m build 6 | 7 | publish: dist 8 | @twine upload dist/* 9 | 10 | clean: 11 | @rm -rf dist build ocifs.egg-info 12 | @find ./ -name '*.pyc' -exec rm -f {} \; 13 | @find ./ -name 'Thumbs.db' -exec rm -f {} \; 14 | @find ./ -name '*~' -exec rm -f {} \; 15 | -------------------------------------------------------------------------------- /README-development.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Development 3 | ============ 4 | 5 | The target audience for this README is developers wanting to contribute to ocifs, Oracle 6 | Cloud Infrastructure (OCI) Object Storage implementation of fsspec's filesystem. 7 | If you want to use the SDK with your own programs, see README.md. 8 | 9 | Getting Started 10 | =============== 11 | Assuming that you have Python and `conda` installed, set up your environment and install the required dependencies like this: 12 | 13 | .. code-block:: sh 14 | 15 | git clone https://github.com/oracle/ocifs.git 16 | cd ocifs 17 | conda create python=3.8 --name ocifs -y 18 | conda activate ocifs 19 | # Install the current package in your environment in an editable mode: 20 | python3 -m pip install -e . 21 | 22 | You should also set up your configuration files, see the `SDK and CLI Configuration File`__. 23 | 24 | __ https://docs.cloud.oracle.com/Content/API/Concepts/sdkconfig.htm 25 | 26 | Running Tests 27 | ============= 28 | The SDK uses `pytest` as its test framework. If you want to run an individual test, then run: 29 | 30 | .. code-block:: sh 31 | 32 | python -m pytest ocifs/tests/test_spec.py::test_simple 33 | 34 | 35 | Specifying environment variables 36 | -------------------------------- 37 | In addition to a valid config file for your tenancy, the tests also require the following environment 38 | variables to be set: 39 | 40 | * ``OCIFS_TEST_NAMESPACE``: The namespace of a bucket in Object Storage to use for testing. 41 | * ``OCIFS_TEST_BUCKET``: The bucket in Object Storage to use for testing. 42 | 43 | 44 | Checking Style 45 | ============== 46 | The ocifs SDK adheres to PEP8 style guilds, and uses Flake8 to validate style. There are some exceptions and they can 47 | be viewed in the ``setup.cfg`` file. 48 | 49 | There is a pre-commit hook setup for this repo. To use this pre-commit hook, run the following: 50 | 51 | .. code-block:: sh 52 | python3 -m pip install pre-commit 53 | pre-commit install 54 | 55 | 56 | Signing Commits 57 | ================ 58 | Please ensure that all commits are signed following the process outlined here: 59 | https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits 60 | 61 | 62 | Generating Documentation 63 | ======================== 64 | Sphinx is used for documentation. You can generate HTML locally with the following: 65 | 66 | .. code-block:: sh 67 | 68 | python3 -m pip install -r docs/requirements.txt 69 | cd docs 70 | make html 71 | 72 | Generating the wheel 73 | ==================== 74 | The SDK using [build](https://pypa-build.readthedocs.io/en/stable/index.html) as build frontend. To generate sdist and wheel, you can run: 75 | 76 | .. code-block:: sh 77 | 78 | pip install build 79 | python -m build 80 | 81 | This wheel can then be installed using `pip`. 82 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Oracle Cloud Infrastructure Object Storage fsspec Implementation 3 | 4 | 5 | [![PyPI](https://img.shields.io/pypi/v/ocifs.svg?style=for-the-badge&logo=pypi&logoColor=white)](https://pypi.org/project/ocifs/) [![Python](https://img.shields.io/pypi/pyversions/ocifs.svg?style=for-the-badge&logo=pypi&logoColor=white)](https://pypi.org/project/ocifs/) 6 | 7 | 8 | ​ 9 | The [Oracle Cloud Infrastructure Object Storage](https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/objectstorageoverview.htm) service is an internet-scale, high-performance storage platform that offers reliable and cost-efficient data durability. With Object Storage, you can safely and securely store or retrieve data directly from the internet or from within the cloud platform. 10 | ​ 11 | `ocifs` is part of the `fsspec` [intake/filesystem_spec ecosystem](https://github.com/intake/filesystem_spec) 12 | ​ 13 | > a template or specification for a file-system interface, that specific implementations should follow, so that applications making use of them can rely on a common interface and not have to worry about the specific internal implementation decisions with any given backend. 14 | ​ 15 | `ocifs` joins the list of file systems supported with this package. 16 | ​ 17 | The `intake/filesystem_spec` project is used by [Pandas](https://pandas.pydata.org/), [Dask](https://dask.org/) and other data libraries in python, this package adds Oracle OCI Object Storage capabilties to these libraries. 18 | ​ 19 | ## OCIFS file system style operations Example: 20 | ```python 21 | from ocifs import OCIFileSystem 22 | 23 | fs = OCIFilesystem("~/.oci/config") 24 | # 1.Create empty file or truncate in OCI objectstorage bucket 25 | fs.touch("oci://@//hello.txt", truncate=True, data=b"Writing to Object Storage!") 26 | # 2.Fetch(potentially multiple paths' contents 27 | fs.cat("oci://@//hello.txt") 28 | # 3.Get metadata about a file from a head or list call 29 | fs.info("oci://@//hello.txt") 30 | # 4.Get directory listing page 31 | fs.ls("oci://@//", detail=True) 32 | # 5.Is this entry directory-like? 33 | fs.isdir("oci://@") 34 | # 6.Is this entry file-like? 35 | fs.isfile("oci://@//hello.txt") 36 | # 7.If there is a file at the given path (including broken links) 37 | fs.lexists("oci://@//hello.txt") 38 | # 8.List of files for the given path 39 | fs.listdir("oci://@/", detail=True) 40 | # 9.Get the first ``size`` bytes from file 41 | fs.head("oci://@//hello.txt", size=1024) 42 | # 10.Get the last ``size`` bytes from file 43 | fs.tail("oci://@//hello.txt", size=1024) 44 | # 11.Hash of file properties, to tell if it has changed 45 | fs.ukey("oci://@//hello.txt") 46 | # 12.Size in bytes of file 47 | fs.size("oci://@//hello.txt") 48 | # 13.Size in bytes of each file in a list of paths 49 | paths = ["oci://@//hello.txt"] 50 | fs.sizes(paths) 51 | # 14.Normalise OCI path string into bucket and key. 52 | fs.split_path("oci://@//hello.txt") 53 | # 15.Delete a file from the bucket 54 | fs.rm("oci://@//hello.txt") 55 | # 16.Get the contents of the file as a byte 56 | fs.read_bytes("oci://@//hello.txt", start=0, end=13) 57 | # 17.Get the contents of the file as a string 58 | fs.read_text("oci://@//hello.txt", encoding=None, errors=None, newline=None) 59 | # 18.Get the contents of the file as a byte 60 | fs.read_block("oci://@//hello.txt", 0, 13) 61 | # 19.Open a file for writing/flushing into file in OCI objectstorage bucket 62 | # Ocifs sets the best-guessed content-type for hello.txt i.e "text/plain" 63 | with fs.open("oci://@//hello.txt", 'w', autocommit=True) as f: 64 | f.write("Writing data to buffer, before manually flushing and closing.") # data is flushed and file closed 65 | f.flush() 66 | # Ocifs uses the specified content-type passed in the open while writing to OCI objectstorage bucket 67 | with fs.open("oci://@//hello.txt", 'w',content_type='text/plain') as f: 68 | f.write("Writing data to buffer, before manually flushing and closing.") # data is flushed and file closed 69 | f.flush() 70 | # 20.Open a file for reading a file from OCI objectstorage bucket 71 | with fs.open("oci://@//hello.txt") as f: 72 | print(f.read()) 73 | # 21.Space used by files and optionally directories within a path 74 | fs.du("oci://@//hello10.csv") 75 | # 22.Find files by glob-matching. 76 | fs.glob("oci://@//*.txt") 77 | # 23.Renames an object in a particular bucket in tenancy namespace on OCI 78 | fs.rename("oci://@//hello.txt", "oci://@//hello2.txt") 79 | # 24.Delete multiple files from the same bucket 80 | pathlist = ["oci://@//hello2.txt"] 81 | fs.bulk_delete(pathlist) 82 | 83 | ``` 84 | 85 | 86 | 87 | ### Or Use With Pandas 88 | ​ 89 | ```python 90 | import pandas as pd 91 | import ocifs 92 | ​ 93 | df = pd.read_csv( 94 | "oci://my_bucket@my_namespace/my_object.csv", 95 | storage_options={"config": "~/.oci/config"}, 96 | ) 97 | ``` 98 | 99 | ### Or Use With PyArrow 100 | ​ 101 | ```python 102 | import pandas as pd 103 | import ocifs 104 | ​ 105 | df = pd.read_csv( 106 | "oci://my_bucket@my_namespace/my_object.csv",storage_options={"config": "~/.oci/config"}) 107 | ``` 108 | 109 | ### Or Use With ADSDataset 110 | ​ 111 | ```python 112 | import ads 113 | import pandas as pd 114 | from ads.common.auth import default_signer 115 | from ads.dataset.dataset import ADSDataset 116 | 117 | ​ 118 | ads.set_auth(auth="api_key", oci_config_location="~/.oci/config", profile="") 119 | ds = ADSDataset( 120 | df=pd.read_csv(f"oci://my_bucket@my_namespace/my_object.csv", storage_options=default_signer()), 121 | type_discovery=False 122 | ) 123 | print(ds.df) 124 | ``` 125 | 126 | ​ 127 | ## Getting Started 128 | ```bash 129 | python3 -m pip install ocifs 130 | ``` 131 | 132 | ## Software Prerequisites 133 | Python >= 3.6 134 | 135 | ## Environment Variables for Authentication: 136 | ```bash 137 | export OCIFS_IAM_TYPE=api_key 138 | export OCIFS_CONFIG_LOCATION=~/.oci/config 139 | export OCIFS_CONFIG_PROFILE=DEFAULT 140 | ``` 141 | 142 | Note, if you are operating on OCI with an alternative valid signer, such as resource principal, instead set the following: 143 | ```bash 144 | export OCIFS_IAM_TYPE=resource_principal 145 | ``` 146 | 147 | ## Environment Variables for enabling Logging: 148 | To quickly see all messages, you can set the environment variable OCIFS_LOGGING_LEVEL=DEBUG. 149 | ```bash 150 | export OCIFS_LOGGING_LEVEL=DEBUG 151 | ``` 152 | 153 | ## Documentation 154 | * [![PyPI](https://img.shields.io/pypi/v/ocifs.svg?style=for-the-badge&logo=pypi&logoColor=white)](https://pypi.org/project/ocifs/) [![Python](https://img.shields.io/pypi/pyversions/ocifs.svg?style=for-the-badge&logo=pypi&logoColor=white)](https://pypi.org/project/ocifs/) 155 | * [ocifs Documentation](https://ocifs.readthedocs.io/en/latest/index.html) 156 | * [ocifs GitHub](https://github.com/oracle/ocifs) 157 | 158 | ## Support 159 | [The built-in filesystems in `fsspec`](https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations) are maintained by the `intake` project team, where as `ocifs` is an external implementation (similar to `s3fs`, `gcsfs`, `adl/abfs`, and so on), which is maintained by Oracle. 160 | 161 | ## Contributing 162 | This project welcomes contributions from the community. Before submitting a pull request, please [review our contribution guide](./CONTRIBUTING.md) 163 | 164 | ## Security 165 | Please consult the [security guide](./SECURITY.md) for our responsible security vulnerability disclosure process 166 | 167 | ## License 168 | Copyright (c) 2021, 2023 Oracle and/or its affiliates. 169 | 170 | Released under the Universal Permissive License v1.0 as shown at 171 | . 172 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting security vulnerabilities 2 | 3 | Oracle values the independent security research community and believes that 4 | responsible disclosure of security vulnerabilities helps us ensure the security 5 | and privacy of all our users. 6 | 7 | Please do NOT raise a GitHub Issue to report a security vulnerability. If you 8 | believe you have found a security vulnerability, please submit a report to 9 | [secalert_us@oracle.com][1] preferably with a proof of concept. Please review 10 | some additional information on [how to report security vulnerabilities to Oracle][2]. 11 | We encourage people who contact Oracle Security to use email encryption using 12 | [our encryption key][3]. 13 | 14 | We ask that you do not use other channels or contact the project maintainers 15 | directly. 16 | 17 | Non-vulnerability related security issues including ideas for new or improved 18 | security features are welcome on GitHub Issues. 19 | 20 | ## Security updates, alerts and bulletins 21 | 22 | Security updates will be released on a regular cadence. Many of our projects 23 | will typically release security fixes in conjunction with the 24 | Oracle Critical Patch Update program. Additional 25 | information, including past advisories, is available on our [security alerts][4] 26 | page. 27 | 28 | ## Security-related information 29 | 30 | We will provide security related information such as a threat model, considerations 31 | for secure use, or any known security issues in our documentation. Please note 32 | that labs and sample code are intended to demonstrate a concept and may not be 33 | sufficiently hardened for production use. 34 | 35 | [1]: mailto:secalert_us@oracle.com 36 | [2]: https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html 37 | [3]: https://www.oracle.com/security-alerts/encryptionkey.html 38 | [4]: https://www.oracle.com/security-alerts/ 39 | -------------------------------------------------------------------------------- /THIRD_PARTY_LICENSES.txt: -------------------------------------------------------------------------------- 1 | OCI Object Storage Filesystem Wrapper Third Party License File 2 | 3 | ------------------------ Third Party Components ------------------------ 4 | ------------------------------- Licenses ------------------------------- 5 | - Universal Permissive License (UPL), Version 1.0 6 | - BSD 3-Clause "New" or "Revised" License 7 | - Apache License 2.0 8 | 9 | ======================== Third Party Components ======================== 10 | fsspec 11 | * Copyright (c) 2018, Martin Durant 12 | * License: BSD 3-Clause License 13 | * Source code: https://github.com/intake/filesystem_spec 14 | * Project home: https://filesystem-spec.readthedocs.io/en/latest/ 15 | 16 | oci python sdk 17 | * Copyright (c) 2016, 2018, Oracle and/or its affiliates. 18 | * License: Universal Permissive License (UPL) 1.0 or Apache License 2.0. 19 | * Source code: https://github.com/oracle/oci-python-sdk 20 | * Project home: https://oracle-cloud-infrastructure-python-sdk.readthedocs.io/en/latest/ 21 | 22 | 23 | 24 | =============================== Licenses =============================== 25 | 26 | ------------------------ BSD-3-Clause New License ---------------------- 27 | BSD 3-Clause License 28 | 29 | Copyright (c) 2018, Martin Durant 30 | All rights reserved. 31 | 32 | Redistribution and use in source and binary forms, with or without 33 | modification, are permitted provided that the following conditions are met: 34 | 35 | * Redistributions of source code must retain the above copyright notice, this 36 | list of conditions and the following disclaimer. 37 | 38 | * Redistributions in binary form must reproduce the above copyright notice, 39 | this list of conditions and the following disclaimer in the documentation 40 | and/or other materials provided with the distribution. 41 | 42 | * Neither the name of the copyright holder nor the names of its 43 | contributors may be used to endorse or promote products derived from 44 | this software without specific prior written permission. 45 | 46 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 47 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 48 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 49 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 50 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 51 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 52 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 53 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 54 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 55 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 56 | ------------------------------------------------------------------------ 57 | 58 | --------------------- Universal Permissive License ---------------------- 59 | The Universal Permissive License (UPL), Version 1.0 60 | Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. 61 | 62 | Subject to the condition set forth below, permission is hereby granted to any person obtaining a copy of this software, associated documentation and/or data (collectively the "Software"), free of charge and under any and all copyright rights in the Software, and any and all patent rights owned or freely licensable by each licensor hereunder covering either (i) the unmodified Software as contributed to or provided by such licensor, or (ii) the Larger Works (as defined below), to deal in both 63 | 64 | (a) the Software, and 65 | (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if one is included with the Software (each a "Larger Work" to which the Software is contributed by such licensors), 66 | 67 | without restriction, including without limitation the rights to copy, create derivative works of, display, perform, and distribute the Software and make, use, sell, offer for sale, import, export, have made, and have sold the Software and the Larger Work(s), and to sublicense the foregoing rights on either these or other terms. 68 | 69 | This license is subject to the following condition: 70 | 71 | The above copyright notice and either this complete permission notice or at a minimum a reference to the UPL must be included in all copies or substantial portions of the Software. 72 | 73 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 74 | ------------------------------------------------------------------------ 75 | 76 | -------------------------- Apache License 2.0 -------------------------- 77 | The Apache Software License, Version 2.0 78 | Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. 79 | 80 | Licensed under the Apache License, Version 2.0 (the "License"); You may not use this product except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0. A copy of the license is also reproduced below. Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 81 | 82 | Apache License 83 | 84 | Version 2.0, January 2004 85 | 86 | http://www.apache.org/licenses/ 87 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 88 | 1. Definitions. 89 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 90 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 91 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 92 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 93 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 94 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 95 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 96 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 97 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 98 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 99 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 100 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 101 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 102 | You must give any other recipients of the Work or Derivative Works a copy of this License; and 103 | You must cause any modified files to carry prominent notices stating that You changed the files; and 104 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 105 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 106 | 107 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 108 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 109 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 110 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 111 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 112 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 113 | END OF TERMS AND CONDITIONS 114 | 115 | APPENDIX: How to apply the Apache License to your work. 116 | 117 | To apply the Apache License to your work, attach the following 118 | boilerplate notice, with the fields enclosed by brackets "[]" 119 | replaced with your own identifying information. (Don't include 120 | the brackets!) The text should be enclosed in the appropriate 121 | comment syntax for the file format. We also recommend that a 122 | file or class name and description of purpose be included on the 123 | same "printed page" as the copyright notice for easier 124 | identification within third-party archives. 125 | 126 | Copyright [yyyy] [name of copyright owner] 127 | 128 | Licensed under the Apache License, Version 2.0 (the "License"); 129 | you may not use this file except in compliance with the License. 130 | You may obtain a copy of the License at 131 | 132 | http://www.apache.org/licenses/LICENSE-2.0 133 | 134 | Unless required by applicable law or agreed to in writing, software 135 | distributed under the License is distributed on an "AS IS" BASIS, 136 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 137 | See the License for the specific language governing permissions and 138 | limitations under the License. 139 | ------------------------------------------------------------------------ 140 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, 2022 Oracle and/or its affiliates. 2 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | VERSION = $(shell python3 -c 'import ocifs; print(ocifs.__version__)') 11 | ZIP_TARGET = ocifs-"$(VERSION)".zip 12 | 13 | # Put it first so that "make" without argument is like "make help". 14 | help: 15 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 16 | 17 | .PHONY: help Makefile 18 | 19 | zip: html 20 | @cd build/html && zip -r ../../"$(ZIP_TARGET)" . 21 | @echo "built: $(ZIP_TARGET)" 22 | 23 | # Catch-all target: route all unknown targets to Sphinx using the new 24 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 25 | %: Makefile 26 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 27 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @REM Copyright (c) 2021, 2022 Oracle and/or its affiliates. 2 | @REM Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 3 | 4 | @ECHO OFF 5 | 6 | pushd %~dp0 7 | 8 | REM Command file for Sphinx documentation 9 | 10 | if "%SPHINXBUILD%" == "" ( 11 | set SPHINXBUILD=sphinx-build 12 | ) 13 | set SOURCEDIR=. 14 | set BUILDDIR=_build 15 | 16 | if "%1" == "" goto help 17 | 18 | %SPHINXBUILD% >NUL 2>NUL 19 | if errorlevel 9009 ( 20 | echo. 21 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 22 | echo.installed, then set the SPHINXBUILD environment variable to point 23 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 24 | echo.may add the Sphinx directory to PATH. 25 | echo. 26 | echo.If you don't have Sphinx installed, grab it from 27 | echo.http://sphinx-doc.org/ 28 | exit /b 1 29 | ) 30 | 31 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 32 | goto end 33 | 34 | :help 35 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 36 | 37 | :end 38 | popd 39 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | nbsphinx 3 | pandoc 4 | ipython 5 | sphinx_rtd_theme 6 | autodoc 7 | sphinxcontrib-napoleon 8 | -e . 9 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 2 | # Licensed under the Universal Permissive License v 1.0 as shown at 3 | # https://oss.oracle.com/licenses/upl/ 4 | 5 | # -- Path setup -------------------------------------------------------------- 6 | 7 | # If extensions (or modules to document with autodoc) are in another directory, 8 | # add these directories to sys.path here. If the directory is relative to the 9 | # documentation root, use os.path.abspath to make it absolute, like shown here. 10 | # 11 | import os 12 | import sys 13 | 14 | sys.path.insert(0, os.path.abspath("..")) 15 | 16 | 17 | # -- Project information ----------------------------------------------------- 18 | 19 | project = "ocifs" 20 | copyright = "Oracle and/or its affiliates 2021, 2023" 21 | author = "Allen Hosler" 22 | 23 | # -- General configuration --------------------------------------------------- 24 | 25 | # Add any Sphinx extension module names here, as strings. They can be 26 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 27 | # ones. 28 | extensions = [ 29 | "sphinx.ext.autodoc", 30 | "sphinx.ext.napoleon", 31 | "nbsphinx", 32 | "sphinx_rtd_theme", 33 | ] 34 | 35 | # Add any paths that contain templates here, relative to this directory. 36 | templates_path = ["_templates"] 37 | 38 | # Get version 39 | import ocifs 40 | 41 | version = ocifs.__version__ 42 | release = version 43 | 44 | # Unless we want to expose real buckets and namespaces 45 | nbsphinx_allow_errors = True 46 | 47 | # List of patterns, relative to source directory, that match files and 48 | # directories to ignore when looking for source files. 49 | # This pattern also affects html_static_path and html_extra_path. 50 | exclude_patterns = ["_build", "**.ipynb_checkpoints", "Thumbs.db", ".DS_Store"] 51 | 52 | 53 | # -- Options for HTML output ------------------------------------------------- 54 | 55 | # The theme to use for HTML and HTML Help pages. See the documentation for 56 | # a list of builtin themes. 57 | # 58 | html_theme = "sphinx_rtd_theme" 59 | 60 | html_theme_options = { 61 | "logo_only": True, 62 | # Toc options 63 | "sticky_navigation": True, 64 | "navigation_depth": 4, 65 | "includehidden": True, 66 | "titles_only": False, 67 | "display_version": True, 68 | } 69 | 70 | # Add any paths that contain custom static files (such as style sheets) here, 71 | # relative to this directory. They are copied after the builtin static files, 72 | # so a file named "default.css" will overwrite the builtin "default.css". 73 | html_static_path = [] 74 | -------------------------------------------------------------------------------- /docs/source/faqs.rst: -------------------------------------------------------------------------------- 1 | Frequently Asked Questions 2 | --------------------------- 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | **Is ocifs asynchronous?** 8 | 9 | No. Ocifs currently inherits the AbstractFile and AbstractFileSystem classes, rather than the Async versions of these. This development may happen in the future if there's enough interest. 10 | 11 | 12 | **Does ocifs use multipart uploads?** 13 | 14 | Yes. Ocifs uses multipart uploads to upload files to Object Storage when the file is larger than 5 GB. 15 | 16 | 17 | **Can I bring my own signer?** 18 | 19 | Yes. Whether you want to use Resource Principal, Instance Principal, or some other type of OCI signer, simply pass that signer to the OCIFileSystem init method. 20 | 21 | 22 | **Can I use ocifs with a different region?** 23 | 24 | Yes, pass this region into the OCIFileSystem init method, and ensure your auth credentials work cross-region. 25 | 26 | 27 | **Can I use ocifs with a different tenancy?** 28 | 29 | Yes, pass this tenancy into the OCIFileSystem init method, and ensure your auth credentials work cross-region. 30 | 31 | 32 | **Can I set auth once and forget?** 33 | 34 | Yes, you can use environment variables: `OCIFS_IAM_TYPE`, `OCIFS_CONFIG_LOCATION`, `OCIFS_CONFIG_PROFILE`. Read more in the Getting Connected section. 35 | 36 | 37 | **Can I refresh the cache from pandas?** 38 | 39 | Yes, you can. Pass `storage_options = {"refresh": "True"}` into any pandas read method. 40 | -------------------------------------------------------------------------------- /docs/source/faqs_title.rst: -------------------------------------------------------------------------------- 1 | FAQS 2 | ===== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | faqs 8 | -------------------------------------------------------------------------------- /docs/source/getting-connected.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "serial-equivalent", 6 | "metadata": {}, 7 | "source": [ 8 | "## Getting Connected" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "knowing-biotechnology", 14 | "metadata": {}, 15 | "source": [ 16 | "### Configuring Your Connection" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "signal-genealogy", 22 | "metadata": {}, 23 | "source": [ 24 | "There are two IAM policy validation options in `ocifs`. The first option is using an identity policy, which is a configuration file. This is the most commonly used policy, and is the default in this documentation. " 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 1, 30 | "id": "silver-passport", 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "from ocifs import OCIFileSystem\n", 35 | "\n", 36 | "fs = OCIFileSystem(config=\"~/.oci/config\", profile=\"DEFAULT\")" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "objective-string", 42 | "metadata": {}, 43 | "source": [ 44 | "Using Pandas and Dask are just as easy:" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "id": "blocked-dover", 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "import pandas as pd\n", 55 | "\n", 56 | "pd.read_csv(\"oci://bucket@namespace/path/file.csv\", \n", 57 | " storage_options={\"config\": \"~/.oci/config\", \"profile\": \"DEFAULT\"})" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "id": "choice-chocolate", 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "from dask import dataframe as dd\n", 68 | "\n", 69 | "dd.read_csv(\"oci://bucket@namespace/path/file.csv\", \n", 70 | " storage_options={\"config\": \"~/.oci/config\", \"profile\": \"DEFAULT\"})" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "id": "5f4f34d1", 76 | "metadata": {}, 77 | "source": [ 78 | "### Using Environment Variables\n", 79 | "Users can provide default authentication configuration details as enviornment variables.\n", 80 | "The following environment variables are inspected:\n", 81 | "\n", 82 | "* `OCIFS_IAM_TYPE`: which can be any of: [\"api_key\", \"resource_principal\", \"instance_principal\", \"unknown_signer\"]\n", 83 | "* `OCIFS_CONFIG_LOCATION`: (optional) will be referenced in the case of \"api_key\", but defaults to the default config location provided by the oci sdk\n", 84 | "* `OCIFS_CONFIG_PROFILE`: (optional) will be referenced in the case of \"api_key\", but defaults to the default config profile provided by the oci sdk\n" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "id": "8a54e4b3", 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "import os\n", 95 | "\n", 96 | "os.environ['OCIFS_IAM_TYPE'] = \"api_key\"\n", 97 | "fs = OCIFileSystem()" 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "id": "1df56db6", 103 | "metadata": {}, 104 | "source": [ 105 | "Note, the order of precedence for authentication is: signer arg, config arg, environment variables, then ocifs will attempt to set up Resource Principal, as exemplified below." 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "id": "sonic-response", 111 | "metadata": {}, 112 | "source": [ 113 | "### Resource Principal" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "id": "embedded-confidence", 119 | "metadata": {}, 120 | "source": [ 121 | "The second policy option is using a resource principal. This policy only works if you're operating within a valid OCI resource, such as an [OCI Data Science notebook session](https://www.oracle.com/data-science/cloud-infrastructure-data-science.html). With this option, your resource token path is set by [global OCI signing variables](https://oracle-cloud-infrastructure-python-sdk.readthedocs.io/en/latest/api/signing.html)." 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "id": "median-lying", 128 | "metadata": {}, 129 | "outputs": [], 130 | "source": [ 131 | "fs = OCIFileSystem()" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "id": "warming-warehouse", 137 | "metadata": {}, 138 | "source": [ 139 | "And with pandas or dask:" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "id": "hollow-confidence", 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "pd.read_csv(\"oci://bucket@namespace/path/file.csv\")" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": null, 155 | "id": "psychological-affair", 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "dd.read_csv(\"oci://bucket@namespace/path/file.csv\")" 160 | ] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "id": "conditional-contribution", 165 | "metadata": {}, 166 | "source": [ 167 | "### Connecting Using a Signer\n", 168 | "Any signer can be passed in using the signer argument." 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": null, 174 | "id": "distant-chassis", 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [ 178 | "resource_principal_signer = oci.auth.signers.get_resource_principals_signer()\n", 179 | "fs_rp = OCIFileSystem(signer=resource_principal_signer)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": null, 185 | "id": "manufactured-poison", 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "instance_principal_signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()\n", 190 | "fs_ip = OCIFileSystem(signer=instance_principal_signer)" 191 | ] 192 | }, 193 | { 194 | "cell_type": "markdown", 195 | "id": "terminal-christmas", 196 | "metadata": {}, 197 | "source": [ 198 | "And with pandas or dask:" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "id": "detected-powder", 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "pd.read_csv(\"oci://bucket@namespace/path/file.csv\", \n", 209 | " storage_options={\"signer\": resource_principal_signer})\n", 210 | "\n", 211 | "dd.read_csv(\"oci://bucket@namespace/path/file.csv\",\n", 212 | " storage_options={\"signer\": instance_principal_signer})" 213 | ] 214 | }, 215 | { 216 | "cell_type": "markdown", 217 | "id": "peripheral-muslim", 218 | "metadata": {}, 219 | "source": [ 220 | "### Connecting to a Different Region" 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "id": "owned-pavilion", 226 | "metadata": {}, 227 | "source": [ 228 | "Each filesystem instance has a home region and won't operate outside of that region. The home region defaults to the region of the IAM policy. With a configuration policy, it is `region`. With a resource principal, the region is derived from the `OCI_REGION_METADATA` environment variable. " 229 | ] 230 | }, 231 | { 232 | "cell_type": "markdown", 233 | "id": "greenhouse-richardson", 234 | "metadata": {}, 235 | "source": [ 236 | "The `OCIFileSystem` delegates this region set up to the [Object Storage Client __init__ method](https://oracle-cloud-infrastructure-python-sdk.readthedocs.io/en/latest/api/object_storage/client/oci.object_storage.ObjectStorageClient.html#oci.object_storage.ObjectStorageClient.__init__) in the OCI Python SDK. The `region` argument accepts any [valid region identifier](https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm) and constructs the corresponding service endpoint for the Object Storage Client. The following cell is an example of connecting to the sydney region." 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "execution_count": null, 242 | "id": "designing-difference", 243 | "metadata": {}, 244 | "outputs": [], 245 | "source": [ 246 | "fs_sydney = OCIFileSystem(config=\"~/.oci/config\", region=\"ap-sydney-1\")" 247 | ] 248 | }, 249 | { 250 | "cell_type": "markdown", 251 | "id": "monetary-standard", 252 | "metadata": {}, 253 | "source": [ 254 | "Using Pandas or Dask:" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 2, 260 | "id": "anticipated-solomon", 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [ 264 | "df.to_csv(\"oci://bucket@namespace/path/file.csv\", \n", 265 | " storage_options = {\"config\": \"~/.oci/config\", \"region\": \"ap-sydney-1\"})" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": null, 271 | "id": "adolescent-aggregate", 272 | "metadata": {}, 273 | "outputs": [], 274 | "source": [ 275 | "ddf.to_csv(\"oci://bucket@namespace/path/file.csv\", \n", 276 | " storage_options = {\"config\": \"~/.oci/config\", \"region\": \"ap-sydney-1\"})" 277 | ] 278 | }, 279 | { 280 | "cell_type": "markdown", 281 | "id": "robust-douglas", 282 | "metadata": {}, 283 | "source": [ 284 | "Note: You must ensure that you have valid cross-region permissions before attempting to instantiate a file system in a non-home region, see the list of [valid OCI Region Identifiers](https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm)." 285 | ] 286 | } 287 | ], 288 | "metadata": { 289 | "kernelspec": { 290 | "display_name": "ds", 291 | "language": "python", 292 | "name": "ds" 293 | }, 294 | "language_info": { 295 | "codemirror_mode": { 296 | "name": "ipython", 297 | "version": 3 298 | }, 299 | "file_extension": ".py", 300 | "mimetype": "text/x-python", 301 | "name": "python", 302 | "nbconvert_exporter": "python", 303 | "pygments_lexer": "ipython3", 304 | "version": "3.7.9" 305 | } 306 | }, 307 | "nbformat": 4, 308 | "nbformat_minor": 5 309 | } 310 | -------------------------------------------------------------------------------- /docs/source/getting-started.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Getting Started\n", 8 | "\n", 9 | "The Oracle Cloud Infrastructure (OCI) Object Storage filesystem (ocifs) is an fsspec implementation for use with Object Storage." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "### Quickstart with Pandas" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "Begin by importing `ocifs` and `pandas`. When importing `ocifs`, you are registering the `oci` protocol with `pandas`:" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 1, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "import ocifs\n", 33 | "import pandas as pd" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "Now that the `oci` protocol is registered with `pandas`, you can read and write from and to Object Storage as easily as you can locally. For example, you could read an Excel file, `path/file.xls`, from your bucket in a namespace easily using:" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "df = pd.read_excel(\"oci://bucket@namespace/path/file.xls\",\n", 50 | " storage_options={\"config\": \"~/.oci/config\"})" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "df.to_parquet(\"oci://bucket@namespace/path/file.parquet\",\n", 60 | " storage_options={\"config\": \"~/.oci/config\"})" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "metadata": {}, 66 | "source": [ 67 | "You could also use [Dask](https://docs.dask.org/en/latest/index.html):" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "from dask import dataframe as dd\n", 77 | "\n", 78 | "ddf = dd.read_csv(\"oci://bucket@namespace/path/file*.csv\",\n", 79 | " storage_options={\"config\": \"~/.oci/config\"})" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "The `storage_options` parameter contains a dictionary of arguments that are passed to the underlying `OCIFileSystem` method. The following `docstring` lists the valid arguments to storage options:" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "### Quickstart to UNIX Operations" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "You can interact with the filesytem directly using most UNIX commands like `ls`, `cp`, `exists`, `mkdir`, `rm`, `walk`, `find`, and so on." 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "Instantiate a filesystem from your configuration, see Getting Connected. Every filesystem instance operates within the home region of the configuration. The `cp` command is the only command that has cross-region support. You must create a unique filesystem instance for each region." 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 3, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "fs = ocifs.OCIFileSystem(config=\"~/.oci/config\", profile=\"DEFAULT\", default_block_size=5*2**20)" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [ 125 | "fs.ls(\"oci://bucket@namespace/path\")\n", 126 | "# []" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "fs.touch(\"oci://bucket@namespace/path/file.txt\")" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "fs.exists(\"oci://bucket@namespace/path/file.txt\")\n", 145 | "# True" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "fs.cat(\"oci://bucket@namespace/path/file.txt\")\n", 155 | "# \"\"" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "fs.rm(\"oci://bucket@namespace\", recursive=True)" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [ 173 | "fs.exists(\"oci://bucket@namespace/path/file.txt\")\n", 174 | "# False" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "Following are examples of how you can use the `OCIFileSystem` and `OCIFile` objects." 182 | ] 183 | } 184 | ], 185 | "metadata": { 186 | "kernelspec": { 187 | "display_name": "ds", 188 | "language": "python", 189 | "name": "ds" 190 | }, 191 | "language_info": { 192 | "codemirror_mode": { 193 | "name": "ipython", 194 | "version": 3 195 | }, 196 | "file_extension": ".py", 197 | "mimetype": "text/x-python", 198 | "name": "python", 199 | "nbconvert_exporter": "python", 200 | "pygments_lexer": "ipython3", 201 | "version": "3.7.9" 202 | } 203 | }, 204 | "nbformat": 4, 205 | "nbformat_minor": 5 206 | } 207 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Copyright (c) 2021, 2022 Oracle and/or its affiliates. 2 | .. Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 3 | 4 | OCIFS 5 | ===== 6 | 7 | Installation 8 | ------------ 9 | Install using pip: 10 | 11 | .. code-block:: bash 12 | 13 | python3 -m pip install ocifs 14 | 15 | Overview 16 | --------- 17 | 18 | `ocifs`_ is a Pythonic filesystem interface to Oracle Cloud Infrastructure (OCI) Object Storage. It builds on top of `oci`_. 19 | 20 | The top-level class ``OCIFileSystem`` holds connection information and allows 21 | typical file-system style operations like ``cp``, ``mv``, ``ls``, ``du``, 22 | ``glob``, as well as put/get of local files to/from OCI. 23 | 24 | The connection gets validated via a configuration file (usually ~/.oci/config) or `Resource Principal`_. 25 | 26 | Calling ``open()`` on a ``OCIFileSystem`` (typically using a context manager) 27 | provides an ``OCIFile`` for read or write access to a particular key. The object 28 | emulates the standard `file object`_ (``read``, ``write``, ``tell``, 29 | ``seek``), such that functions can interact with OCI Object Storage. Only 30 | binary read and write modes are implemented, with blocked caching. 31 | 32 | `ocifs`_ uses and is based upon `fsspec`_. 33 | 34 | 35 | .. _fsspec: https://filesystem-spec.readthedocs.io/en/latest/ 36 | .. _Resource Principal: https://oracle-cloud-infrastructure-python-sdk.readthedocs.io/en/latest/api/signing.html?highlight=resource%20principal#resource-principals-signer 37 | .. _file object: https://docs.python.org/3/glossary.html#term-file-object 38 | .. _ocifs: https://pypi.org/project/ocifs/ 39 | 40 | Examples 41 | -------- 42 | 43 | Simple locate and read a file: 44 | 45 | .. code-block:: python 46 | 47 | >>> import ocifs 48 | >>> fs = ocifs.OCIFileSystem() 49 | >>> fs.ls('my-bucket@my-namespace') 50 | ['my-bucket@my-namespace/my-file.txt'] 51 | >>> with fs.open('my-bucket@my-namespace/my-file.txt', 'rb') as f: 52 | ... print(f.read()) 53 | b'Hello, world' 54 | 55 | (see also ``walk`` and ``glob`` in the Unix Operations section) 56 | 57 | Reading with delimited blocks: 58 | 59 | .. code-block:: python 60 | 61 | >>> fs.read_block(path, offset=1000, length=10, delimiter=b'\n') 62 | b'A whole line of text\n' 63 | 64 | Learn more about `read_block`_. 65 | 66 | Writing with blocked caching: 67 | 68 | .. code-block:: python 69 | 70 | >>> fs = ocifs.OCIFileSystem(config=".oci/config") # uses default credentials 71 | >>> with fs.open('mybucket@mynamespace/new-file', 'wb') as f: 72 | ... f.write(2*2**20 * b'a') 73 | ... f.write(2*2**20 * b'a') # data is flushed and file closed 74 | >>> fs.du('mybucket@mynamespace/new-file') 75 | {'mybucket@mynamespace/new-file': 4194304} 76 | 77 | Learn more about ``fsspec``'s `caching system`_. 78 | 79 | Because ``ocifs`` copies the Python file interface it can be used with 80 | other projects that consume the file interface like ``gzip`` or ``pandas``. 81 | 82 | .. code-block:: python 83 | 84 | >>> with fs.open('mybucket@mynamespace/my-file.csv.gz', 'rb') as f: 85 | ... g = gzip.GzipFile(fileobj=f) # Decompress data with gzip 86 | ... df = pd.read_csv(g) # Read CSV file with Pandas 87 | 88 | 89 | .. _read_block: https://filesystem-spec.readthedocs.io/en/latest/api.html?highlight=read_block#fsspec.spec.AbstractFileSystem.read_block 90 | .. _caching system: https://filesystem-spec.readthedocs.io/en/latest/features.html?highlight=cache#instance-caching 91 | 92 | Adding content type while writing: 93 | 94 | .. code-block:: python 95 | 96 | >>> fs = ocifs.OCIFileSystem(config=".oci/config") # uses default credentials 97 | # Example: passing content_type as the parameter value, OCIFS will use this value to set the content_type. 98 | >>> with fs.open('oci://mybucket@mynamespace/path/new-file.txt', 'wb', content_type = 'text/plain' ) as f: 99 | ... f.write(2*2**20 * b'a') # data is flushed and file closed 100 | ... f.flush() # data is flushed and file closed 101 | 102 | #Example - ocifs will determine the best-suited content-type for 'new-file.txt' 103 | >>> with fs.open('oci://mybucket@mynamespace/path/new-file.txt', 'wb') as f: 104 | ... f.write(2*2**20 * b'a') # data is flushed and file closed 105 | ... f.flush() # data is flushed and file closed 106 | 107 | 108 | When uploading a file, you can also specify ``content_type`` (commonly referred to as MIME type). 109 | ``ocifs`` automatically infers the content type from the file extension, but if you specify ``content_type`` 110 | it will override the auto-detected type. If no content type is specified and the file doesn't have a file 111 | extension, ``ocifs`` defaults to the type ``application/json``. 112 | 113 | Integration 114 | ----------- 115 | 116 | The libraries ``intake``, ``pandas`` and ``dask`` accept URLs with the prefix 117 | "oci://", and will use ``ocifs`` to complete the IO operation in question. The 118 | IO functions take an argument ``storage_options``, which will be passed verbatim 119 | to ``OCIFileSystem``, for example: 120 | 121 | .. code-block:: python 122 | 123 | df = pd.read_excel("oci://bucket@namespace/path/file.xls", 124 | storage_options={"config": "~/.oci/config"}) 125 | 126 | Use the ``storage_options`` parameter to pass any additional arguments to ``ocifs``, 127 | for example security credentials. 128 | 129 | 130 | Authentication and Credentials 131 | ------------------------------ 132 | 133 | An OCI config file (API Keys) may be provided as a filepath or a config instance returned 134 | from ``oci.config.from_file`` when creating an ``OCIFileSystem`` using the ``config`` argument. 135 | For example, two valid inputs to the ``config`` argument are: ``oci.config.from_file("~/.oci/config")`` 136 | and ``~/.oci/config``. Specify the profile using the ``profile`` argument: ``OCIFileSystem(config="~/.oci/config", profile='PROFILE')``. 137 | 138 | Alternatively a signer may be used to create an ``OCIFileSystem`` using the ``signer`` argument. 139 | A Resource Principal Signer can be created using ``oci.auth.signers.get_resource_principals_signer()``. 140 | An Instance Principal Signer can be created using ``oci.auth.signers.InstancePrincipalsSecurityTokenSigner()`` 141 | If neither config nor signer is provided, ``OCIFileSystem`` will attempt to create a Resource Principal, then an Instance Principal. 142 | However, passing a signer directly is always preferred. 143 | 144 | Learn more about using signers with ``ocifs`` in the Getting Connected tab, or learn more about `Resource Principal here`_. 145 | 146 | .. _oci: https://oracle-cloud-infrastructure-python-sdk.readthedocs.io/en/latest/index.html 147 | .. _Resource Principal here: https://oracle-cloud-infrastructure-python-sdk.readthedocs.io/en/latest/api/signing.html 148 | 149 | 150 | Logging 151 | ------- 152 | 153 | The logger named ``ocifs`` provides information about the operations of the file 154 | system. To see all messages, set the logging level to ``INFO``: 155 | 156 | .. code-block:: python 157 | 158 | import logging 159 | logging.getLogger("ocifs").setLevel(logging.INFO) 160 | 161 | Info mode will print messages to stderr. More advanced logging configuration is possible using 162 | Python's standard `logging framework`_. 163 | 164 | .. _logging framework: https://docs.python.org/3/library/logging.html 165 | 166 | 167 | Limitations 168 | ----------- 169 | 170 | This project is meant for convenience, rather than feature completeness. 171 | The following are known current omissions: 172 | 173 | - file access is always binary (although reading files line by line as strings is possible) 174 | 175 | - no permissions/access-control (no ``chmod``/``chown`` methods) 176 | 177 | - ``ocifs`` only reads the latest version of a file. Versioned support is on the roadmap for a future release. 178 | 179 | 180 | A Note on Caching 181 | ----------------- 182 | To save time and bandwidth, ocifs caches the results from listing buckets and objects. To refresh the cache, 183 | set the ``refresh`` argument to ``True`` in the ``ls()`` method. However, ``info``, and as a result ``exists``, 184 | ignores the cache and makes a call to Object Storage directly. If the underlying bucket is modified after a call 185 | to list, the change will be reflected in calls to info, but not list (unless ``refresh=True``). 186 | 187 | 188 | Contents 189 | ======== 190 | 191 | .. toctree:: 192 | :hidden: 193 | :maxdepth: 4 194 | 195 | getting-started.ipynb 196 | getting-connected.ipynb 197 | unix-operations.ipynb 198 | modules 199 | faqs_title 200 | telemetry 201 | release_notes 202 | 203 | 204 | Indices and tables 205 | ================== 206 | 207 | * :ref:`genindex` 208 | * :ref:`modindex` 209 | * :ref:`search` 210 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | ocifs 8 | -------------------------------------------------------------------------------- /docs/source/ocifs.rst: -------------------------------------------------------------------------------- 1 | ocifs classes 2 | ============= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | OCIFileSystem 8 | -------------- 9 | 10 | .. autoclass:: ocifs.core.OCIFileSystem 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | :inherited-members: 15 | 16 | OCIFile 17 | -------- 18 | 19 | .. autoclass:: ocifs.core.OCIFile 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | :inherited-members: 24 | -------------------------------------------------------------------------------- /docs/source/release_notes.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Release Notes 3 | ============= 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | 8 | 1.3.2 9 | ------ 10 | Release date: February 18, 2025 11 | 12 | * Added support for EU Sovereign Cloud 13 | * Minor bug fixes 14 | 15 | 1.3.1 16 | ------ 17 | Release date: December 8, 2023 18 | 19 | * Introducing support for setting the content type implicitly or explicitly when writing files 20 | 21 | 1.3.0 22 | ------ 23 | Release date: November 16, 2023 24 | 25 | * Introducing DataLake support in ocifs 26 | 27 | 28 | 1.2.2 29 | ------ 30 | Release date: November 16, 2023 31 | 32 | * Fix for Windows os.path bug 33 | 34 | 35 | 1.2.1 36 | ----- 37 | Release date: March 29, 2023 38 | 39 | * Patch issue with region parameter being required for unknown signers 40 | 41 | 42 | 1.2.0 43 | ----- 44 | Release date: March 24, 2023 45 | 46 | * Filter ``ls`` to not show empty files with names ending in "/". 47 | * Add ``sync`` method to allow copying folders between local and object storage. 48 | * Update ``copy`` to forward requests to ``sync`` when one of the paths is local and the other is ``oci`` prefixed. 49 | * Add ``oci-cli`` as an optional requirement. 50 | * Create a default region upon instantiation of the filesystem instance. 51 | -------------------------------------------------------------------------------- /docs/source/telemetry.rst: -------------------------------------------------------------------------------- 1 | Telemetry 2 | --------- 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | The `ocifs` library utilizes user-agent to record telemetry on usage: `ocifs` version used, operating system, python version, etc. This allows the ocifs team to prioritize future development plans. 8 | -------------------------------------------------------------------------------- /docs/source/unix-operations.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "aefd23b8", 6 | "metadata": {}, 7 | "source": [ 8 | "## Unix Operations" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "858282b9", 14 | "metadata": {}, 15 | "source": [ 16 | "_Important: The ocifs SDK isn't a one-to-one adaptor of OCI Object Storage and UNIX filesystem operations. It's a set of convenient wrappings to assist Pandas in natively reading from Object Storage. It supports many of the common UNIX functions, and many of the Object Storage API though not all._\n", 17 | "\n", 18 | "Following are examples of some of the most popular filesystem and file methods. First, you must instantiate your region-specific filesystem instance:" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 1, 24 | "id": "9d450d82", 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "from ocifs import OCIFileSystem\n", 29 | "\n", 30 | "fs = OCIFileSystem(config=\"~/.oci/config\")" 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "id": "4c7a2cd9", 36 | "metadata": {}, 37 | "source": [ 38 | "### Filesystem Operations\n", 39 | "\n", 40 | "#### list\n", 41 | "List the files in a bucket or subdirectory using `ls`:" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "id": "2dee4999", 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "fs.ls(\"bucket@namespace/\")\n", 52 | "# ['bucket@namespace/file.txt', \n", 53 | "# 'bucket@namespace/data.csv', \n", 54 | "# 'bucket@namespace/folder1/', \n", 55 | "# 'bucket@namespace/folder2/']" 56 | ] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "id": "02c3a1bc", 61 | "metadata": {}, 62 | "source": [ 63 | "`list` has the following args: 1) `compartment_id`: a specific compartment from which to list. 2)`detail`: If true, return a list of dictionaries with various details about each object. 3)`refresh`: If true, ignore the cache and pull fresh." 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "id": "8abc4ef0", 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "fs.ls(\"bucket@namespace/\", detail=True)\n", 74 | "# [{'name': 'bucket@namespace/file.txt', \n", 75 | "# 'etag': 'abcdefghijklmnop',\n", 76 | "# 'type': 'file',\n", 77 | "# 'timeCreated': ,\n", 78 | "# ... },\n", 79 | "# ...\n", 80 | "# ]" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "id": "fa2650b9", 86 | "metadata": {}, 87 | "source": [ 88 | "#### touch\n", 89 | "The UNIX `touch` command creates empty files in Object Storage. The `data` parameter accepts a bytestream and writes it to the new file." 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": null, 95 | "id": "408d380c", 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "fs.touch(\"bucket@namespace/newfile\", data=b\"Hello World!\")" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "id": "592180ce", 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "fs.cat(\"bucket@namespace/newfile\")\n", 110 | "# \"Hello World!\"" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "id": "128b4656", 116 | "metadata": {}, 117 | "source": [ 118 | "#### copy\n", 119 | "The `copy` method is a popular UNIX method, and it has a special role in ocifs as the only method capable of cross-tenancy calls. Your IAM Policy must permit you to read and write cross-region to use the `copy` method cross-region. Note: Another benefit of `copy` is that it can move large data between locations in Object Storage without needing to store anything locally." 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "id": "f6c81ce8", 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "fs.copy(\"bucket@namespace/newfile\", \"bucket@namespace/newfile-sydney\",\n", 130 | " destination_region=\"ap-sydney-1\")" 131 | ] 132 | }, 133 | { 134 | "cell_type": "markdown", 135 | "id": "50c6a28b", 136 | "metadata": {}, 137 | "source": [ 138 | "#### rm\n", 139 | "The `rm` method is another essential UNIX filesystem method. It accepts one additional argument (beyond the path), `recursive`. When `recursive=True`, it is equivalent to an `rm -rf` command. It deletes all files underneath the prefix." 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "id": "aa74be1d", 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "fs.exists(\"oci://bucket@namespace/folder/file\")\n", 150 | "# True" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "id": "fe753c02", 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "fs.rm(\"oci://bucket@namespace/folder\", recursive=True)" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": null, 166 | "id": "79ea99e6", 167 | "metadata": {}, 168 | "outputs": [], 169 | "source": [ 170 | "fs.exists(\"oci://bucket@namespace/folder/file\")\n", 171 | "# False" 172 | ] 173 | }, 174 | { 175 | "cell_type": "markdown", 176 | "id": "4074ec28", 177 | "metadata": {}, 178 | "source": [ 179 | "#### glob\n", 180 | "Fsspec implementations, including ocifs, support UNIX glob patterns, see [Globbing](https://man7.org/linux/man-pages/man7/glob.7.html)." 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": null, 186 | "id": "0f9271b7", 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "fs.glob(\"oci://bucket@namespace/folder/*.csv\")\n", 191 | "# [\"bucket@namespace/folder/part1.csv\", \"bucket@namespace/folder/part2.csv\"]" 192 | ] 193 | }, 194 | { 195 | "cell_type": "markdown", 196 | "id": "cde56557", 197 | "metadata": {}, 198 | "source": [ 199 | "Dask has special support for reading from and writing to a set of files using glob expressions (Pandas doesn't support glob), see [Dask's Glob support](https://docs.dask.org/en/latest/remote-data-services.html)." 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "id": "18e0ccab", 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [ 209 | "from dask import dataframe as dd\n", 210 | "\n", 211 | "ddf = dd.read_csv(\"oci://bucket@namespace/folder/*.csv\")\n", 212 | "ddf.to_csv(\"oci://bucket@namespace/folder_copy/*.csv\")" 213 | ] 214 | }, 215 | { 216 | "cell_type": "markdown", 217 | "id": "9ab01f8d", 218 | "metadata": {}, 219 | "source": [ 220 | "#### walk\n", 221 | "\n", 222 | "Use the UNIX `walk` method for iterating through the subdirectories of a given path. This is a valuable method for determining every file within a bucket or folder." 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": null, 228 | "id": "6b352841", 229 | "metadata": {}, 230 | "outputs": [], 231 | "source": [ 232 | "fs.walk(\"oci://bucket@namespace/folder\")\n", 233 | "# [\"bucket@namespace/folder/part1.csv\", \"bucket@namespace/folder/part2.csv\",\n", 234 | "# \"bucket@namespace/folder/subdir/file1.csv\", \"bucket@namespace/folder/subdir/file2.csv\"]" 235 | ] 236 | }, 237 | { 238 | "cell_type": "markdown", 239 | "id": "f4f44be1", 240 | "metadata": {}, 241 | "source": [ 242 | "#### open\n", 243 | "This method opens a file and returns an `OCIFile` object. There are examples of what you can do with an `OCIFile` in the next section." 244 | ] 245 | }, 246 | { 247 | "cell_type": "markdown", 248 | "id": "8c3bd2ca", 249 | "metadata": {}, 250 | "source": [ 251 | "### File Operations\n", 252 | "After calling open, you get an `OCIFile` object, which is subclassed from fsspec's `AbstractBufferedFile`. This file object can do almost everything a UNIX file can. Following are a few examples, see [a full list of methods](https://filesystem-spec.readthedocs.io/en/latest/api.html?highlight=AbstractFileSystem#fsspec.spec.AbstractBufferedFile)." 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "id": "043178d2", 258 | "metadata": {}, 259 | "source": [ 260 | "#### read\n", 261 | "The `read` method works exactly as you would expect with a UNIX file:" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": null, 267 | "id": "47c8c713", 268 | "metadata": {}, 269 | "outputs": [], 270 | "source": [ 271 | "import fsspec\n", 272 | "\n", 273 | "with fsspec.open(\"oci://bucket@namespace/folder/file\", 'rb') as f:\n", 274 | " buffer = f.read()" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": null, 280 | "id": "9921ecfd", 281 | "metadata": {}, 282 | "outputs": [], 283 | "source": [ 284 | "from ocifs import OCIFileSystem\n", 285 | "\n", 286 | "fs = OCIFileSystem()\n", 287 | "with fs.open(\"oci://bucket@namespace/folder/file\", 'rb') as f:\n", 288 | " buffer = f.read()" 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": null, 294 | "id": "f6947d24", 295 | "metadata": {}, 296 | "outputs": [], 297 | "source": [ 298 | "file = fs.open(\"oci://bucket@namespace/folder/file\")\n", 299 | "buffer = file.read()\n", 300 | "file.close()" 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "id": "83da3320", 306 | "metadata": {}, 307 | "source": [ 308 | "#### seek\n", 309 | "The `seek` method is also valuable in navigating files:" 310 | ] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": null, 315 | "id": "d4aac2be", 316 | "metadata": {}, 317 | "outputs": [], 318 | "source": [ 319 | "fs.touch(\"bucket@namespace/newfile\", data=b\"Hello World!\")\n", 320 | "with fs.open(\"bucket@namespace/newfile\") as f:\n", 321 | " f.seek(3)\n", 322 | " print(f.read(1))\n", 323 | " f.seek(0)\n", 324 | " print(f.read(1))\n", 325 | "\n", 326 | "# l\n", 327 | "# H" 328 | ] 329 | }, 330 | { 331 | "cell_type": "markdown", 332 | "id": "b5c279a9", 333 | "metadata": {}, 334 | "source": [ 335 | "#### write\n", 336 | "You can use the `write` operation:" 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": null, 342 | "id": "306767fa", 343 | "metadata": {}, 344 | "outputs": [], 345 | "source": [ 346 | "with fsspec.open(\"oci://bucket@namespace/newfile\", 'wb') as f:\n", 347 | " buffer = f.write(b\"new text\")\n", 348 | " \n", 349 | "with fsspec.open(\"oci://bucket@namespace/newfile\", 'rb') as f:\n", 350 | " assert f.read() == b\"new text\"" 351 | ] 352 | }, 353 | { 354 | "cell_type": "markdown", 355 | "id": "9a51c4bb", 356 | "metadata": {}, 357 | "source": [ 358 | "### Learn More\n", 359 | "There are many more operations that you can use with `ocifs`, see the [AbstractBufferedFile spec](https://filesystem-spec.readthedocs.io/en/latest/api.html?highlight=AbstractFileSystem#fsspec.spec.AbstractBufferedFile) and the [AbstractFileSystem spec](https://filesystem-spec.readthedocs.io/en/latest/api.html?highlight=AbstractFileSystem#fsspec.spec.AbstractFileSystem)." 360 | ] 361 | }, 362 | { 363 | "cell_type": "code", 364 | "execution_count": null, 365 | "id": "d14f907a", 366 | "metadata": {}, 367 | "outputs": [], 368 | "source": [] 369 | } 370 | ], 371 | "metadata": { 372 | "kernelspec": { 373 | "display_name": "ds", 374 | "language": "python", 375 | "name": "ds" 376 | }, 377 | "language_info": { 378 | "codemirror_mode": { 379 | "name": "ipython", 380 | "version": 3 381 | }, 382 | "file_extension": ".py", 383 | "mimetype": "text/x-python", 384 | "name": "python", 385 | "nbconvert_exporter": "python", 386 | "pygments_lexer": "ipython3", 387 | "version": "3.7.9" 388 | } 389 | }, 390 | "nbformat": 4, 391 | "nbformat_minor": 5 392 | } 393 | -------------------------------------------------------------------------------- /ocifs/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | 5 | from .core import OCIFileSystem 6 | from fsspec import register_implementation 7 | import sys 8 | from .utils import __version__ 9 | 10 | 11 | if sys.version_info.major < 3: 12 | raise ImportError("Python < 3 is unsupported.") 13 | 14 | register_implementation("oci", OCIFileSystem) 15 | -------------------------------------------------------------------------------- /ocifs/data_lake/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | -------------------------------------------------------------------------------- /ocifs/data_lake/lake_mount.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | from oci.util import ( 5 | formatted_flat_dict, 6 | NONE_SENTINEL, 7 | value_allowed_none_or_none_sentinel, 8 | ) # noqa: F401 9 | from oci.decorators import init_model_state_from_kwargs 10 | 11 | 12 | @init_model_state_from_kwargs 13 | class LakeMount(object): 14 | """ 15 | A Mount under the given lake. 16 | """ 17 | 18 | #: A constant which can be used with the lifecycle_state property of a LakeMount. 19 | #: This constant has a value of "ACTIVE" 20 | LIFECYCLE_STATE_ACTIVE = "ACTIVE" 21 | 22 | def __init__(self, **kwargs): 23 | """ 24 | Initializes a new LakeMount object with values from keyword arguments. 25 | The following keyword arguments are supported (corresponding to the getters/setters of this class): 26 | 27 | :param key: 28 | The value to assign to the key property of this LakeMount. 29 | :type key: str 30 | 31 | :param display_name: 32 | The value to assign to the display_name property of this LakeMount. 33 | :type display_name: str 34 | 35 | :param is_operation_pending: 36 | The value to assign to the is_operation_pending property of this LakeMount. 37 | :type is_operation_pending: bool 38 | 39 | :param mount_type: 40 | The value to assign to the mount_type property of this LakeMount. 41 | :type mount_type: str 42 | 43 | :param storage_type: 44 | The value to assign to the storage_type property of this LakeMount. 45 | :type storage_type: str 46 | 47 | :param access_type: 48 | The value to assign to the access_type property of this LakeMount. 49 | :type access_type: str 50 | 51 | :param mount_spec: 52 | The value to assign to the mount_spec property of this LakeMount. 53 | :type mount_spec: oci.lakehouse.models.MountSpecification 54 | 55 | :param credentials: 56 | The value to assign to the credentials property of this LakeMount. 57 | :type credentials: str 58 | 59 | :param encryption: 60 | The value to assign to the encryption property of this LakeMount. 61 | :type encryption: str 62 | 63 | :param mount_options: 64 | The value to assign to the mount_options property of this LakeMount. 65 | :type mount_options: str 66 | 67 | :param permissions: 68 | The value to assign to the permissions property of this LakeMount. 69 | :type permissions: list[oci.lakehouse.models.PermissionLine] 70 | 71 | :param time_created: 72 | The value to assign to the time_created property of this LakeMount. 73 | :type time_created: datetime 74 | 75 | :param time_updated: 76 | The value to assign to the time_updated property of this LakeMount. 77 | :type time_updated: datetime 78 | 79 | :param created_by: 80 | The value to assign to the created_by property of this LakeMount. 81 | :type created_by: str 82 | 83 | :param is_assigned: 84 | The value to assign to the is_assigned property of this LakeMount. 85 | :type is_assigned: bool 86 | 87 | :param lifecycle_state: 88 | The value to assign to the lifecycle_state property of this LakeMount. 89 | Allowed values for this property are: "ACTIVE", 'UNKNOWN_ENUM_VALUE'. 90 | Any unrecognized values returned by a service will be mapped to 'UNKNOWN_ENUM_VALUE'. 91 | :type lifecycle_state: str 92 | 93 | :param lifecycle_details: 94 | The value to assign to the lifecycle_details property of this LakeMount. 95 | :type lifecycle_details: str 96 | 97 | :param description: 98 | The value to assign to the description property of this LakeMount. 99 | :type description: str 100 | 101 | :param lake_id: 102 | The value to assign to the lake_id property of this LakeMount. 103 | :type lake_id: str 104 | 105 | :param compartment_id: 106 | The value to assign to the compartment_id property of this LakeMount. 107 | :type compartment_id: str 108 | 109 | :param freeform_tags: 110 | The value to assign to the freeform_tags property of this LakeMount. 111 | :type freeform_tags: dict(str, str) 112 | 113 | :param defined_tags: 114 | The value to assign to the defined_tags property of this LakeMount. 115 | :type defined_tags: dict(str, dict(str, object)) 116 | 117 | :param system_tags: 118 | The value to assign to the system_tags property of this LakeMount. 119 | :type system_tags: dict(str, dict(str, object)) 120 | 121 | """ 122 | self.swagger_types = { 123 | "key": "str", 124 | "display_name": "str", 125 | "is_operation_pending": "bool", 126 | "mount_type": "str", 127 | "storage_type": "str", 128 | "access_type": "str", 129 | "mount_spec": "MountSpecification", 130 | "credentials": "str", 131 | "encryption": "str", 132 | "mount_options": "str", 133 | "permissions": "list[PermissionLine]", 134 | "time_created": "datetime", 135 | "time_updated": "datetime", 136 | "created_by": "str", 137 | "is_assigned": "bool", 138 | "lifecycle_state": "str", 139 | "lifecycle_details": "str", 140 | "description": "str", 141 | "lake_id": "str", 142 | "compartment_id": "str", 143 | "freeform_tags": "dict(str, str)", 144 | "defined_tags": "dict(str, dict(str, object))", 145 | "system_tags": "dict(str, dict(str, object))", 146 | } 147 | 148 | self.attribute_map = { 149 | "key": "key", 150 | "display_name": "displayName", 151 | "is_operation_pending": "isOperationPending", 152 | "mount_type": "mountType", 153 | "storage_type": "storageType", 154 | "access_type": "accessType", 155 | "mount_spec": "mountSpec", 156 | "credentials": "credentials", 157 | "encryption": "encryption", 158 | "mount_options": "mountOptions", 159 | "permissions": "permissions", 160 | "time_created": "timeCreated", 161 | "time_updated": "timeUpdated", 162 | "created_by": "createdBy", 163 | "is_assigned": "isAssigned", 164 | "lifecycle_state": "lifecycleState", 165 | "lifecycle_details": "lifecycleDetails", 166 | "description": "description", 167 | "lake_id": "lakeId", 168 | "compartment_id": "compartmentId", 169 | "freeform_tags": "freeformTags", 170 | "defined_tags": "definedTags", 171 | "system_tags": "systemTags", 172 | } 173 | 174 | self._key = None 175 | self._display_name = None 176 | self._is_operation_pending = None 177 | self._mount_type = None 178 | self._storage_type = None 179 | self._access_type = None 180 | self._mount_spec = None 181 | self._credentials = None 182 | self._encryption = None 183 | self._mount_options = None 184 | self._permissions = None 185 | self._time_created = None 186 | self._time_updated = None 187 | self._created_by = None 188 | self._is_assigned = None 189 | self._lifecycle_state = None 190 | self._lifecycle_details = None 191 | self._description = None 192 | self._lake_id = None 193 | self._compartment_id = None 194 | self._freeform_tags = None 195 | self._defined_tags = None 196 | self._system_tags = None 197 | 198 | @property 199 | def key(self): 200 | """ 201 | **[Required]** Gets the key of this LakeMount. 202 | A unique key for the Mount, that is immutable on creation. 203 | 204 | 205 | :return: The key of this LakeMount. 206 | :rtype: str 207 | """ 208 | return self._key 209 | 210 | @key.setter 211 | def key(self, key): 212 | """ 213 | Sets the key of this LakeMount. 214 | A unique key for the Mount, that is immutable on creation. 215 | 216 | 217 | :param key: The key of this LakeMount. 218 | :type: str 219 | """ 220 | self._key = key 221 | 222 | @property 223 | def display_name(self): 224 | """ 225 | **[Required]** Gets the display_name of this LakeMount. 226 | The Mount name. It can't be changed. 227 | 228 | 229 | :return: The display_name of this LakeMount. 230 | :rtype: str 231 | """ 232 | return self._display_name 233 | 234 | @display_name.setter 235 | def display_name(self, display_name): 236 | """ 237 | Sets the display_name of this LakeMount. 238 | The Mount name. It can't be changed. 239 | 240 | 241 | :param display_name: The display_name of this LakeMount. 242 | :type: str 243 | """ 244 | self._display_name = display_name 245 | 246 | @property 247 | def is_operation_pending(self): 248 | """ 249 | Gets the is_operation_pending of this LakeMount. 250 | Identifies if there is a ongoing operation on the mount. 251 | 252 | 253 | :return: The is_operation_pending of this LakeMount. 254 | :rtype: bool 255 | """ 256 | return self._is_operation_pending 257 | 258 | @is_operation_pending.setter 259 | def is_operation_pending(self, is_operation_pending): 260 | """ 261 | Sets the is_operation_pending of this LakeMount. 262 | Identifies if there is a ongoing operation on the mount. 263 | 264 | 265 | :param is_operation_pending: The is_operation_pending of this LakeMount. 266 | :type: bool 267 | """ 268 | self._is_operation_pending = is_operation_pending 269 | 270 | @property 271 | def mount_type(self): 272 | """ 273 | Gets the mount_type of this LakeMount. 274 | Type of the Mount. 275 | 276 | 277 | :return: The mount_type of this LakeMount. 278 | :rtype: str 279 | """ 280 | return self._mount_type 281 | 282 | @mount_type.setter 283 | def mount_type(self, mount_type): 284 | """ 285 | Sets the mount_type of this LakeMount. 286 | Type of the Mount. 287 | 288 | 289 | :param mount_type: The mount_type of this LakeMount. 290 | :type: str 291 | """ 292 | self._mount_type = mount_type 293 | 294 | @property 295 | def storage_type(self): 296 | """ 297 | Gets the storage_type of this LakeMount. 298 | Type of the Storage being used by the Mount. 299 | 300 | 301 | :return: The storage_type of this LakeMount. 302 | :rtype: str 303 | """ 304 | return self._storage_type 305 | 306 | @storage_type.setter 307 | def storage_type(self, storage_type): 308 | """ 309 | Sets the storage_type of this LakeMount. 310 | Type of the Storage being used by the Mount. 311 | 312 | 313 | :param storage_type: The storage_type of this LakeMount. 314 | :type: str 315 | """ 316 | self._storage_type = storage_type 317 | 318 | @property 319 | def access_type(self): 320 | """ 321 | Gets the access_type of this LakeMount. 322 | Access Type of the Mount. 323 | 324 | 325 | :return: The access_type of this LakeMount. 326 | :rtype: str 327 | """ 328 | return self._access_type 329 | 330 | @access_type.setter 331 | def access_type(self, access_type): 332 | """ 333 | Sets the access_type of this LakeMount. 334 | Access Type of the Mount. 335 | 336 | 337 | :param access_type: The access_type of this LakeMount. 338 | :type: str 339 | """ 340 | self._access_type = access_type 341 | 342 | @property 343 | def mount_spec(self): 344 | """ 345 | Gets the mount_spec of this LakeMount. 346 | 347 | :return: The mount_spec of this LakeMount. 348 | :rtype: oci.lakehouse.models.MountSpecification 349 | """ 350 | return self._mount_spec 351 | 352 | @mount_spec.setter 353 | def mount_spec(self, mount_spec): 354 | """ 355 | Sets the mount_spec of this LakeMount. 356 | 357 | :param mount_spec: The mount_spec of this LakeMount. 358 | :type: oci.lakehouse.models.MountSpecification 359 | """ 360 | self._mount_spec = mount_spec 361 | 362 | @property 363 | def credentials(self): 364 | """ 365 | Gets the credentials of this LakeMount. 366 | Credential for the Mount. 367 | 368 | 369 | :return: The credentials of this LakeMount. 370 | :rtype: str 371 | """ 372 | return self._credentials 373 | 374 | @credentials.setter 375 | def credentials(self, credentials): 376 | """ 377 | Sets the credentials of this LakeMount. 378 | Credential for the Mount. 379 | 380 | 381 | :param credentials: The credentials of this LakeMount. 382 | :type: str 383 | """ 384 | self._credentials = credentials 385 | 386 | @property 387 | def encryption(self): 388 | """ 389 | Gets the encryption of this LakeMount. 390 | Encryption for the Mount. 391 | 392 | 393 | :return: The encryption of this LakeMount. 394 | :rtype: str 395 | """ 396 | return self._encryption 397 | 398 | @encryption.setter 399 | def encryption(self, encryption): 400 | """ 401 | Sets the encryption of this LakeMount. 402 | Encryption for the Mount. 403 | 404 | 405 | :param encryption: The encryption of this LakeMount. 406 | :type: str 407 | """ 408 | self._encryption = encryption 409 | 410 | @property 411 | def mount_options(self): 412 | """ 413 | Gets the mount_options of this LakeMount. 414 | Additional Options for the Mount. 415 | 416 | 417 | :return: The mount_options of this LakeMount. 418 | :rtype: str 419 | """ 420 | return self._mount_options 421 | 422 | @mount_options.setter 423 | def mount_options(self, mount_options): 424 | """ 425 | Sets the mount_options of this LakeMount. 426 | Additional Options for the Mount. 427 | 428 | 429 | :param mount_options: The mount_options of this LakeMount. 430 | :type: str 431 | """ 432 | self._mount_options = mount_options 433 | 434 | @property 435 | def permissions(self): 436 | """ 437 | Gets the permissions of this LakeMount. 438 | The permissions in the Mount. 439 | 440 | 441 | :return: The permissions of this LakeMount. 442 | :rtype: list[oci.lakehouse.models.PermissionLine] 443 | """ 444 | return self._permissions 445 | 446 | @permissions.setter 447 | def permissions(self, permissions): 448 | """ 449 | Sets the permissions of this LakeMount. 450 | The permissions in the Mount. 451 | 452 | 453 | :param permissions: The permissions of this LakeMount. 454 | :type: list[oci.lakehouse.models.PermissionLine] 455 | """ 456 | self._permissions = permissions 457 | 458 | @property 459 | def time_created(self): 460 | """ 461 | Gets the time_created of this LakeMount. 462 | The time the Mount was created. An RFC3339 formatted datetime string. 463 | 464 | 465 | :return: The time_created of this LakeMount. 466 | :rtype: datetime 467 | """ 468 | return self._time_created 469 | 470 | @time_created.setter 471 | def time_created(self, time_created): 472 | """ 473 | Sets the time_created of this LakeMount. 474 | The time the Mount was created. An RFC3339 formatted datetime string. 475 | 476 | 477 | :param time_created: The time_created of this LakeMount. 478 | :type: datetime 479 | """ 480 | self._time_created = time_created 481 | 482 | @property 483 | def time_updated(self): 484 | """ 485 | Gets the time_updated of this LakeMount. 486 | The time the Mount was updated. An RFC3339 formatted datetime string. 487 | 488 | 489 | :return: The time_updated of this LakeMount. 490 | :rtype: datetime 491 | """ 492 | return self._time_updated 493 | 494 | @time_updated.setter 495 | def time_updated(self, time_updated): 496 | """ 497 | Sets the time_updated of this LakeMount. 498 | The time the Mount was updated. An RFC3339 formatted datetime string. 499 | 500 | 501 | :param time_updated: The time_updated of this LakeMount. 502 | :type: datetime 503 | """ 504 | self._time_updated = time_updated 505 | 506 | @property 507 | def created_by(self): 508 | """ 509 | Gets the created_by of this LakeMount. 510 | OCID of the Principal who created the Mount. 511 | 512 | 513 | :return: The created_by of this LakeMount. 514 | :rtype: str 515 | """ 516 | return self._created_by 517 | 518 | @created_by.setter 519 | def created_by(self, created_by): 520 | """ 521 | Sets the created_by of this LakeMount. 522 | OCID of the Principal who created the Mount. 523 | 524 | 525 | :param created_by: The created_by of this LakeMount. 526 | :type: str 527 | """ 528 | self._created_by = created_by 529 | 530 | @property 531 | def is_assigned(self): 532 | """ 533 | Gets the is_assigned of this LakeMount. 534 | The Mount is assigned to the current user or a group that the user is part of. 535 | 536 | 537 | :return: The is_assigned of this LakeMount. 538 | :rtype: bool 539 | """ 540 | return self._is_assigned 541 | 542 | @is_assigned.setter 543 | def is_assigned(self, is_assigned): 544 | """ 545 | Sets the is_assigned of this LakeMount. 546 | The Mount is assigned to the current user or a group that the user is part of. 547 | 548 | 549 | :param is_assigned: The is_assigned of this LakeMount. 550 | :type: bool 551 | """ 552 | self._is_assigned = is_assigned 553 | 554 | @property 555 | def lifecycle_state(self): 556 | """ 557 | Gets the lifecycle_state of this LakeMount. 558 | The state of the Lake Mount. 559 | 560 | Allowed values for this property are: "ACTIVE", 'UNKNOWN_ENUM_VALUE'. 561 | Any unrecognized values returned by a service will be mapped to 'UNKNOWN_ENUM_VALUE'. 562 | 563 | 564 | :return: The lifecycle_state of this LakeMount. 565 | :rtype: str 566 | """ 567 | return self._lifecycle_state 568 | 569 | @lifecycle_state.setter 570 | def lifecycle_state(self, lifecycle_state): 571 | """ 572 | Sets the lifecycle_state of this LakeMount. 573 | The state of the Lake Mount. 574 | 575 | 576 | :param lifecycle_state: The lifecycle_state of this LakeMount. 577 | :type: str 578 | """ 579 | allowed_values = ["ACTIVE"] 580 | if not value_allowed_none_or_none_sentinel(lifecycle_state, allowed_values): 581 | lifecycle_state = "UNKNOWN_ENUM_VALUE" 582 | self._lifecycle_state = lifecycle_state 583 | 584 | @property 585 | def lifecycle_details(self): 586 | """ 587 | Gets the lifecycle_details of this LakeMount. 588 | A message describing the current state in more detail. For example, it can be used to provide actionable information for a resource in Failed state. 589 | 590 | 591 | :return: The lifecycle_details of this LakeMount. 592 | :rtype: str 593 | """ 594 | return self._lifecycle_details 595 | 596 | @lifecycle_details.setter 597 | def lifecycle_details(self, lifecycle_details): 598 | """ 599 | Sets the lifecycle_details of this LakeMount. 600 | A message describing the current state in more detail. For example, it can be used to provide actionable information for a resource in Failed state. 601 | 602 | 603 | :param lifecycle_details: The lifecycle_details of this LakeMount. 604 | :type: str 605 | """ 606 | self._lifecycle_details = lifecycle_details 607 | 608 | @property 609 | def description(self): 610 | """ 611 | Gets the description of this LakeMount. 612 | The description of the Mount. 613 | 614 | 615 | :return: The description of this LakeMount. 616 | :rtype: str 617 | """ 618 | return self._description 619 | 620 | @description.setter 621 | def description(self, description): 622 | """ 623 | Sets the description of this LakeMount. 624 | The description of the Mount. 625 | 626 | 627 | :param description: The description of this LakeMount. 628 | :type: str 629 | """ 630 | self._description = description 631 | 632 | @property 633 | def lake_id(self): 634 | """ 635 | Gets the lake_id of this LakeMount. 636 | The Lake wherein the Mount resides. 637 | 638 | 639 | :return: The lake_id of this LakeMount. 640 | :rtype: str 641 | """ 642 | return self._lake_id 643 | 644 | @lake_id.setter 645 | def lake_id(self, lake_id): 646 | """ 647 | Sets the lake_id of this LakeMount. 648 | The Lake wherein the Mount resides. 649 | 650 | 651 | :param lake_id: The lake_id of this LakeMount. 652 | :type: str 653 | """ 654 | self._lake_id = lake_id 655 | 656 | @property 657 | def compartment_id(self): 658 | """ 659 | Gets the compartment_id of this LakeMount. 660 | The compartment id. 661 | 662 | 663 | :return: The compartment_id of this LakeMount. 664 | :rtype: str 665 | """ 666 | return self._compartment_id 667 | 668 | @compartment_id.setter 669 | def compartment_id(self, compartment_id): 670 | """ 671 | Sets the compartment_id of this LakeMount. 672 | The compartment id. 673 | 674 | 675 | :param compartment_id: The compartment_id of this LakeMount. 676 | :type: str 677 | """ 678 | self._compartment_id = compartment_id 679 | 680 | @property 681 | def freeform_tags(self): 682 | """ 683 | Gets the freeform_tags of this LakeMount. 684 | Simple key-value pair that is applied without any predefined name, type or scope. Exists for cross-compatibility only. 685 | Example: `{\"bar-key\": \"value\"}` 686 | 687 | 688 | :return: The freeform_tags of this LakeMount. 689 | :rtype: dict(str, str) 690 | """ 691 | return self._freeform_tags 692 | 693 | @freeform_tags.setter 694 | def freeform_tags(self, freeform_tags): 695 | """ 696 | Sets the freeform_tags of this LakeMount. 697 | Simple key-value pair that is applied without any predefined name, type or scope. Exists for cross-compatibility only. 698 | Example: `{\"bar-key\": \"value\"}` 699 | 700 | 701 | :param freeform_tags: The freeform_tags of this LakeMount. 702 | :type: dict(str, str) 703 | """ 704 | self._freeform_tags = freeform_tags 705 | 706 | @property 707 | def defined_tags(self): 708 | """ 709 | Gets the defined_tags of this LakeMount. 710 | Defined tags for this resource. Each key is predefined and scoped to a namespace. 711 | Example: `{\"foo-namespace\": {\"bar-key\": \"value\"}}` 712 | 713 | 714 | :return: The defined_tags of this LakeMount. 715 | :rtype: dict(str, dict(str, object)) 716 | """ 717 | return self._defined_tags 718 | 719 | @defined_tags.setter 720 | def defined_tags(self, defined_tags): 721 | """ 722 | Sets the defined_tags of this LakeMount. 723 | Defined tags for this resource. Each key is predefined and scoped to a namespace. 724 | Example: `{\"foo-namespace\": {\"bar-key\": \"value\"}}` 725 | 726 | 727 | :param defined_tags: The defined_tags of this LakeMount. 728 | :type: dict(str, dict(str, object)) 729 | """ 730 | self._defined_tags = defined_tags 731 | 732 | @property 733 | def system_tags(self): 734 | """ 735 | Gets the system_tags of this LakeMount. 736 | Usage of system tag keys. These predefined keys are scoped to namespaces. 737 | Example: `{\"orcl-cloud\": {\"free-tier-retained\": \"true\"}}` 738 | 739 | 740 | :return: The system_tags of this LakeMount. 741 | :rtype: dict(str, dict(str, object)) 742 | """ 743 | return self._system_tags 744 | 745 | @system_tags.setter 746 | def system_tags(self, system_tags): 747 | """ 748 | Sets the system_tags of this LakeMount. 749 | Usage of system tag keys. These predefined keys are scoped to namespaces. 750 | Example: `{\"orcl-cloud\": {\"free-tier-retained\": \"true\"}}` 751 | 752 | 753 | :param system_tags: The system_tags of this LakeMount. 754 | :type: dict(str, dict(str, object)) 755 | """ 756 | self._system_tags = system_tags 757 | 758 | def __repr__(self): 759 | return formatted_flat_dict(self) 760 | 761 | def __eq__(self, other): 762 | if other is None: 763 | return False 764 | 765 | return self.__dict__ == other.__dict__ 766 | 767 | def __ne__(self, other): 768 | return not self == other 769 | -------------------------------------------------------------------------------- /ocifs/data_lake/lakehouse.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | from oci.util import ( 5 | formatted_flat_dict, 6 | NONE_SENTINEL, 7 | value_allowed_none_or_none_sentinel, 8 | ) # noqa: F401 9 | from oci.decorators import init_model_state_from_kwargs 10 | 11 | 12 | @init_model_state_from_kwargs 13 | class Lakehouse(object): 14 | """ 15 | Description of Lakehouse. 16 | """ 17 | 18 | #: A constant which can be used with the lifecycle_state property of a Lakehouse. 19 | #: This constant has a value of "CREATING" 20 | LIFECYCLE_STATE_CREATING = "CREATING" 21 | 22 | #: A constant which can be used with the lifecycle_state property of a Lakehouse. 23 | #: This constant has a value of "UPDATING" 24 | LIFECYCLE_STATE_UPDATING = "UPDATING" 25 | 26 | #: A constant which can be used with the lifecycle_state property of a Lakehouse. 27 | #: This constant has a value of "ACTIVE" 28 | LIFECYCLE_STATE_ACTIVE = "ACTIVE" 29 | 30 | #: A constant which can be used with the lifecycle_state property of a Lakehouse. 31 | #: This constant has a value of "DELETING" 32 | LIFECYCLE_STATE_DELETING = "DELETING" 33 | 34 | #: A constant which can be used with the lifecycle_state property of a Lakehouse. 35 | #: This constant has a value of "DELETED" 36 | LIFECYCLE_STATE_DELETED = "DELETED" 37 | 38 | #: A constant which can be used with the lifecycle_state property of a Lakehouse. 39 | #: This constant has a value of "FAILED" 40 | LIFECYCLE_STATE_FAILED = "FAILED" 41 | 42 | def __init__(self, **kwargs): 43 | """ 44 | Initializes a new Lakehouse object with values from keyword arguments. 45 | The following keyword arguments are supported (corresponding to the getters/setters of this class): 46 | 47 | :param id: 48 | The value to assign to the id property of this Lakehouse. 49 | :type id: str 50 | 51 | :param display_name: 52 | The value to assign to the display_name property of this Lakehouse. 53 | :type display_name: str 54 | 55 | :param lakeshare_endpoint: 56 | The value to assign to the lakeshare_endpoint property of this Lakehouse. 57 | :type lakeshare_endpoint: str 58 | 59 | :param lakeproxy_endpoint: 60 | The value to assign to the lakeproxy_endpoint property of this Lakehouse. 61 | :type lakeproxy_endpoint: str 62 | 63 | :param compartment_id: 64 | The value to assign to the compartment_id property of this Lakehouse. 65 | :type compartment_id: str 66 | 67 | :param time_created: 68 | The value to assign to the time_created property of this Lakehouse. 69 | :type time_created: datetime 70 | 71 | :param time_updated: 72 | The value to assign to the time_updated property of this Lakehouse. 73 | :type time_updated: datetime 74 | 75 | :param lifecycle_state: 76 | The value to assign to the lifecycle_state property of this Lakehouse. 77 | Allowed values for this property are: "CREATING", "UPDATING", "ACTIVE", "DELETING", "DELETED", "FAILED", 'UNKNOWN_ENUM_VALUE'. 78 | Any unrecognized values returned by a service will be mapped to 'UNKNOWN_ENUM_VALUE'. 79 | :type lifecycle_state: str 80 | 81 | :param lifecycle_details: 82 | The value to assign to the lifecycle_details property of this Lakehouse. 83 | :type lifecycle_details: str 84 | 85 | :param metastore_id: 86 | The value to assign to the metastore_id property of this Lakehouse. 87 | :type metastore_id: str 88 | 89 | :param dis_work_space_details: 90 | The value to assign to the dis_work_space_details property of this Lakehouse. 91 | :type dis_work_space_details: oci.lakehouse.models.DisWorkSpaceDetails 92 | 93 | :param catalog_details: 94 | The value to assign to the catalog_details property of this Lakehouse. 95 | :type catalog_details: oci.lakehouse.models.CatalogDetails 96 | 97 | :param managed_bucket_uri: 98 | The value to assign to the managed_bucket_uri property of this Lakehouse. 99 | :type managed_bucket_uri: str 100 | 101 | :param policy_ids: 102 | The value to assign to the policy_ids property of this Lakehouse. 103 | :type policy_ids: list[str] 104 | 105 | :param freeform_tags: 106 | The value to assign to the freeform_tags property of this Lakehouse. 107 | :type freeform_tags: dict(str, str) 108 | 109 | :param defined_tags: 110 | The value to assign to the defined_tags property of this Lakehouse. 111 | :type defined_tags: dict(str, dict(str, object)) 112 | 113 | :param system_tags: 114 | The value to assign to the system_tags property of this Lakehouse. 115 | :type system_tags: dict(str, dict(str, object)) 116 | 117 | """ 118 | self.swagger_types = { 119 | "id": "str", 120 | "display_name": "str", 121 | "lakeshare_endpoint": "str", 122 | "lakeproxy_endpoint": "str", 123 | "compartment_id": "str", 124 | "time_created": "datetime", 125 | "time_updated": "datetime", 126 | "lifecycle_state": "str", 127 | "lifecycle_details": "str", 128 | "metastore_id": "str", 129 | "dis_work_space_details": "DisWorkSpaceDetails", 130 | "catalog_details": "CatalogDetails", 131 | "managed_bucket_uri": "str", 132 | "policy_ids": "list[str]", 133 | "freeform_tags": "dict(str, str)", 134 | "defined_tags": "dict(str, dict(str, object))", 135 | "system_tags": "dict(str, dict(str, object))", 136 | } 137 | 138 | self.attribute_map = { 139 | "id": "id", 140 | "display_name": "displayName", 141 | "lakeshare_endpoint": "lakeshareEndpoint", 142 | "lakeproxy_endpoint": "lakeproxyEndpoint", 143 | "compartment_id": "compartmentId", 144 | "time_created": "timeCreated", 145 | "time_updated": "timeUpdated", 146 | "lifecycle_state": "lifecycleState", 147 | "lifecycle_details": "lifecycleDetails", 148 | "metastore_id": "metastoreId", 149 | "dis_work_space_details": "disWorkSpaceDetails", 150 | "catalog_details": "catalogDetails", 151 | "managed_bucket_uri": "managedBucketUri", 152 | "policy_ids": "policyIds", 153 | "freeform_tags": "freeformTags", 154 | "defined_tags": "definedTags", 155 | "system_tags": "systemTags", 156 | } 157 | 158 | self._id = None 159 | self._display_name = None 160 | self._lakeshare_endpoint = None 161 | self._lakeproxy_endpoint = None 162 | self._compartment_id = None 163 | self._time_created = None 164 | self._time_updated = None 165 | self._lifecycle_state = None 166 | self._lifecycle_details = None 167 | self._metastore_id = None 168 | self._dis_work_space_details = None 169 | self._catalog_details = None 170 | self._managed_bucket_uri = None 171 | self._policy_ids = None 172 | self._freeform_tags = None 173 | self._defined_tags = None 174 | self._system_tags = None 175 | 176 | @property 177 | def id(self): 178 | """ 179 | **[Required]** Gets the id of this Lakehouse. 180 | Unique identifier that is immutable on creation 181 | 182 | 183 | :return: The id of this Lakehouse. 184 | :rtype: str 185 | """ 186 | return self._id 187 | 188 | @id.setter 189 | def id(self, id): 190 | """ 191 | Sets the id of this Lakehouse. 192 | Unique identifier that is immutable on creation 193 | 194 | 195 | :param id: The id of this Lakehouse. 196 | :type: str 197 | """ 198 | self._id = id 199 | 200 | @property 201 | def display_name(self): 202 | """ 203 | **[Required]** Gets the display_name of this Lakehouse. 204 | Lakehouse Identifier, can be renamed 205 | 206 | 207 | :return: The display_name of this Lakehouse. 208 | :rtype: str 209 | """ 210 | return self._display_name 211 | 212 | @display_name.setter 213 | def display_name(self, display_name): 214 | """ 215 | Sets the display_name of this Lakehouse. 216 | Lakehouse Identifier, can be renamed 217 | 218 | 219 | :param display_name: The display_name of this Lakehouse. 220 | :type: str 221 | """ 222 | self._display_name = display_name 223 | 224 | @property 225 | def lakeshare_endpoint(self): 226 | """ 227 | Gets the lakeshare_endpoint of this Lakehouse. 228 | Lakeshare access endpoint 229 | 230 | 231 | :return: The lakeshare_endpoint of this Lakehouse. 232 | :rtype: str 233 | """ 234 | return self._lakeshare_endpoint 235 | 236 | @lakeshare_endpoint.setter 237 | def lakeshare_endpoint(self, lakeshare_endpoint): 238 | """ 239 | Sets the lakeshare_endpoint of this Lakehouse. 240 | Lakeshare access endpoint 241 | 242 | 243 | :param lakeshare_endpoint: The lakeshare_endpoint of this Lakehouse. 244 | :type: str 245 | """ 246 | self._lakeshare_endpoint = lakeshare_endpoint 247 | 248 | @property 249 | def lakeproxy_endpoint(self): 250 | """ 251 | Gets the lakeproxy_endpoint of this Lakehouse. 252 | Lakeproxy access endpoint 253 | 254 | 255 | :return: The lakeproxy_endpoint of this Lakehouse. 256 | :rtype: str 257 | """ 258 | return self._lakeproxy_endpoint 259 | 260 | @lakeproxy_endpoint.setter 261 | def lakeproxy_endpoint(self, lakeproxy_endpoint): 262 | """ 263 | Sets the lakeproxy_endpoint of this Lakehouse. 264 | Lakeproxy access endpoint 265 | 266 | 267 | :param lakeproxy_endpoint: The lakeproxy_endpoint of this Lakehouse. 268 | :type: str 269 | """ 270 | self._lakeproxy_endpoint = lakeproxy_endpoint 271 | 272 | @property 273 | def compartment_id(self): 274 | """ 275 | **[Required]** Gets the compartment_id of this Lakehouse. 276 | Compartment Identifier 277 | 278 | 279 | :return: The compartment_id of this Lakehouse. 280 | :rtype: str 281 | """ 282 | return self._compartment_id 283 | 284 | @compartment_id.setter 285 | def compartment_id(self, compartment_id): 286 | """ 287 | Sets the compartment_id of this Lakehouse. 288 | Compartment Identifier 289 | 290 | 291 | :param compartment_id: The compartment_id of this Lakehouse. 292 | :type: str 293 | """ 294 | self._compartment_id = compartment_id 295 | 296 | @property 297 | def time_created(self): 298 | """ 299 | **[Required]** Gets the time_created of this Lakehouse. 300 | The time the the Lakehouse was created. An RFC3339 formatted datetime string 301 | 302 | 303 | :return: The time_created of this Lakehouse. 304 | :rtype: datetime 305 | """ 306 | return self._time_created 307 | 308 | @time_created.setter 309 | def time_created(self, time_created): 310 | """ 311 | Sets the time_created of this Lakehouse. 312 | The time the the Lakehouse was created. An RFC3339 formatted datetime string 313 | 314 | 315 | :param time_created: The time_created of this Lakehouse. 316 | :type: datetime 317 | """ 318 | self._time_created = time_created 319 | 320 | @property 321 | def time_updated(self): 322 | """ 323 | Gets the time_updated of this Lakehouse. 324 | The time the Lakehouse was updated. An RFC3339 formatted datetime string 325 | 326 | 327 | :return: The time_updated of this Lakehouse. 328 | :rtype: datetime 329 | """ 330 | return self._time_updated 331 | 332 | @time_updated.setter 333 | def time_updated(self, time_updated): 334 | """ 335 | Sets the time_updated of this Lakehouse. 336 | The time the Lakehouse was updated. An RFC3339 formatted datetime string 337 | 338 | 339 | :param time_updated: The time_updated of this Lakehouse. 340 | :type: datetime 341 | """ 342 | self._time_updated = time_updated 343 | 344 | @property 345 | def lifecycle_state(self): 346 | """ 347 | **[Required]** Gets the lifecycle_state of this Lakehouse. 348 | The current state of the Lakehouse. 349 | 350 | Allowed values for this property are: "CREATING", "UPDATING", "ACTIVE", "DELETING", "DELETED", "FAILED", 'UNKNOWN_ENUM_VALUE'. 351 | Any unrecognized values returned by a service will be mapped to 'UNKNOWN_ENUM_VALUE'. 352 | 353 | 354 | :return: The lifecycle_state of this Lakehouse. 355 | :rtype: str 356 | """ 357 | return self._lifecycle_state 358 | 359 | @lifecycle_state.setter 360 | def lifecycle_state(self, lifecycle_state): 361 | """ 362 | Sets the lifecycle_state of this Lakehouse. 363 | The current state of the Lakehouse. 364 | 365 | 366 | :param lifecycle_state: The lifecycle_state of this Lakehouse. 367 | :type: str 368 | """ 369 | allowed_values = [ 370 | "CREATING", 371 | "UPDATING", 372 | "ACTIVE", 373 | "DELETING", 374 | "DELETED", 375 | "FAILED", 376 | ] 377 | if not value_allowed_none_or_none_sentinel(lifecycle_state, allowed_values): 378 | lifecycle_state = "UNKNOWN_ENUM_VALUE" 379 | self._lifecycle_state = lifecycle_state 380 | 381 | @property 382 | def lifecycle_details(self): 383 | """ 384 | Gets the lifecycle_details of this Lakehouse. 385 | A message describing the current state in more detail. For example, can be used to provide actionable information for a resource in Failed state. 386 | 387 | 388 | :return: The lifecycle_details of this Lakehouse. 389 | :rtype: str 390 | """ 391 | return self._lifecycle_details 392 | 393 | @lifecycle_details.setter 394 | def lifecycle_details(self, lifecycle_details): 395 | """ 396 | Sets the lifecycle_details of this Lakehouse. 397 | A message describing the current state in more detail. For example, can be used to provide actionable information for a resource in Failed state. 398 | 399 | 400 | :param lifecycle_details: The lifecycle_details of this Lakehouse. 401 | :type: str 402 | """ 403 | self._lifecycle_details = lifecycle_details 404 | 405 | @property 406 | def metastore_id(self): 407 | """ 408 | **[Required]** Gets the metastore_id of this Lakehouse. 409 | Metastore OCID 410 | 411 | 412 | :return: The metastore_id of this Lakehouse. 413 | :rtype: str 414 | """ 415 | return self._metastore_id 416 | 417 | @metastore_id.setter 418 | def metastore_id(self, metastore_id): 419 | """ 420 | Sets the metastore_id of this Lakehouse. 421 | Metastore OCID 422 | 423 | 424 | :param metastore_id: The metastore_id of this Lakehouse. 425 | :type: str 426 | """ 427 | self._metastore_id = metastore_id 428 | 429 | @property 430 | def dis_work_space_details(self): 431 | """ 432 | **[Required]** Gets the dis_work_space_details of this Lakehouse. 433 | 434 | :return: The dis_work_space_details of this Lakehouse. 435 | :rtype: oci.lakehouse.models.DisWorkSpaceDetails 436 | """ 437 | return self._dis_work_space_details 438 | 439 | @dis_work_space_details.setter 440 | def dis_work_space_details(self, dis_work_space_details): 441 | """ 442 | Sets the dis_work_space_details of this Lakehouse. 443 | 444 | :param dis_work_space_details: The dis_work_space_details of this Lakehouse. 445 | :type: oci.lakehouse.models.DisWorkSpaceDetails 446 | """ 447 | self._dis_work_space_details = dis_work_space_details 448 | 449 | @property 450 | def catalog_details(self): 451 | """ 452 | **[Required]** Gets the catalog_details of this Lakehouse. 453 | 454 | :return: The catalog_details of this Lakehouse. 455 | :rtype: oci.lakehouse.models.CatalogDetails 456 | """ 457 | return self._catalog_details 458 | 459 | @catalog_details.setter 460 | def catalog_details(self, catalog_details): 461 | """ 462 | Sets the catalog_details of this Lakehouse. 463 | 464 | :param catalog_details: The catalog_details of this Lakehouse. 465 | :type: oci.lakehouse.models.CatalogDetails 466 | """ 467 | self._catalog_details = catalog_details 468 | 469 | @property 470 | def managed_bucket_uri(self): 471 | """ 472 | **[Required]** Gets the managed_bucket_uri of this Lakehouse. 473 | The warehouse bucket URI. It is a Oracle Cloud Infrastructure Object Storage bucket URI as defined here https://docs.oracle.com/en/cloud/paas/atp-cloud/atpud/object-storage-uris.html 474 | 475 | 476 | :return: The managed_bucket_uri of this Lakehouse. 477 | :rtype: str 478 | """ 479 | return self._managed_bucket_uri 480 | 481 | @managed_bucket_uri.setter 482 | def managed_bucket_uri(self, managed_bucket_uri): 483 | """ 484 | Sets the managed_bucket_uri of this Lakehouse. 485 | The warehouse bucket URI. It is a Oracle Cloud Infrastructure Object Storage bucket URI as defined here https://docs.oracle.com/en/cloud/paas/atp-cloud/atpud/object-storage-uris.html 486 | 487 | 488 | :param managed_bucket_uri: The managed_bucket_uri of this Lakehouse. 489 | :type: str 490 | """ 491 | self._managed_bucket_uri = managed_bucket_uri 492 | 493 | @property 494 | def policy_ids(self): 495 | """ 496 | **[Required]** Gets the policy_ids of this Lakehouse. 497 | All policy ids, created by lakehouse in customer tenancy 498 | 499 | 500 | :return: The policy_ids of this Lakehouse. 501 | :rtype: list[str] 502 | """ 503 | return self._policy_ids 504 | 505 | @policy_ids.setter 506 | def policy_ids(self, policy_ids): 507 | """ 508 | Sets the policy_ids of this Lakehouse. 509 | All policy ids, created by lakehouse in customer tenancy 510 | 511 | 512 | :param policy_ids: The policy_ids of this Lakehouse. 513 | :type: list[str] 514 | """ 515 | self._policy_ids = policy_ids 516 | 517 | @property 518 | def freeform_tags(self): 519 | """ 520 | **[Required]** Gets the freeform_tags of this Lakehouse. 521 | Simple key-value pair that is applied without any predefined name, type or scope. Exists for cross-compatibility only. 522 | Example: `{\"bar-key\": \"value\"}` 523 | 524 | 525 | :return: The freeform_tags of this Lakehouse. 526 | :rtype: dict(str, str) 527 | """ 528 | return self._freeform_tags 529 | 530 | @freeform_tags.setter 531 | def freeform_tags(self, freeform_tags): 532 | """ 533 | Sets the freeform_tags of this Lakehouse. 534 | Simple key-value pair that is applied without any predefined name, type or scope. Exists for cross-compatibility only. 535 | Example: `{\"bar-key\": \"value\"}` 536 | 537 | 538 | :param freeform_tags: The freeform_tags of this Lakehouse. 539 | :type: dict(str, str) 540 | """ 541 | self._freeform_tags = freeform_tags 542 | 543 | @property 544 | def defined_tags(self): 545 | """ 546 | **[Required]** Gets the defined_tags of this Lakehouse. 547 | Defined tags for this resource. Each key is predefined and scoped to a namespace. 548 | Example: `{\"foo-namespace\": {\"bar-key\": \"value\"}}` 549 | 550 | 551 | :return: The defined_tags of this Lakehouse. 552 | :rtype: dict(str, dict(str, object)) 553 | """ 554 | return self._defined_tags 555 | 556 | @defined_tags.setter 557 | def defined_tags(self, defined_tags): 558 | """ 559 | Sets the defined_tags of this Lakehouse. 560 | Defined tags for this resource. Each key is predefined and scoped to a namespace. 561 | Example: `{\"foo-namespace\": {\"bar-key\": \"value\"}}` 562 | 563 | 564 | :param defined_tags: The defined_tags of this Lakehouse. 565 | :type: dict(str, dict(str, object)) 566 | """ 567 | self._defined_tags = defined_tags 568 | 569 | @property 570 | def system_tags(self): 571 | """ 572 | Gets the system_tags of this Lakehouse. 573 | Usage of system tag keys. These predefined keys are scoped to namespaces. 574 | Example: `{\"orcl-cloud\": {\"free-tier-retained\": \"true\"}}` 575 | 576 | 577 | :return: The system_tags of this Lakehouse. 578 | :rtype: dict(str, dict(str, object)) 579 | """ 580 | return self._system_tags 581 | 582 | @system_tags.setter 583 | def system_tags(self, system_tags): 584 | """ 585 | Sets the system_tags of this Lakehouse. 586 | Usage of system tag keys. These predefined keys are scoped to namespaces. 587 | Example: `{\"orcl-cloud\": {\"free-tier-retained\": \"true\"}}` 588 | 589 | 590 | :param system_tags: The system_tags of this Lakehouse. 591 | :type: dict(str, dict(str, object)) 592 | """ 593 | self._system_tags = system_tags 594 | 595 | def __repr__(self): 596 | return formatted_flat_dict(self) 597 | 598 | def __eq__(self, other): 599 | if other is None: 600 | return False 601 | 602 | return self.__dict__ == other.__dict__ 603 | 604 | def __ne__(self, other): 605 | return not self == other 606 | -------------------------------------------------------------------------------- /ocifs/data_lake/lakehouse_client.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2024 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | from __future__ import absolute_import 5 | 6 | import logging 7 | import os 8 | 9 | from oci._vendor import requests # noqa: F401 10 | from oci._vendor import six 11 | 12 | from oci import retry, circuit_breaker # noqa: F401 13 | from oci.base_client import BaseClient 14 | from oci.config import get_config_value_or_default, validate_config 15 | from oci.signer import Signer 16 | from oci.util import ( 17 | Sentinel, 18 | get_signer_from_authentication_type, 19 | AUTHENTICATION_TYPE_FIELD_NAME, 20 | ) 21 | from ocifs.errors import translate_oci_error 22 | 23 | from .lakehouse import Lakehouse 24 | from .lake_mount import LakeMount 25 | from .mount_specification import MountSpecification 26 | 27 | missing = Sentinel("Missing") 28 | 29 | # Maps type name to class for lakehouse services. 30 | lakehouse_type_mapping = { 31 | "Lakehouse": Lakehouse, 32 | "LakeMount": LakeMount, 33 | "MountSpecification": MountSpecification, 34 | } 35 | 36 | logger = logging.getLogger("lakehouse") 37 | 38 | 39 | def setup_logging_for_lakehouse(level=None): 40 | level = level or os.environ["OCIFS_LOGGING_LEVEL"] 41 | handle = logging.StreamHandler() 42 | formatter = logging.Formatter( 43 | "%(asctime)s - %(name)s - %(levelname)s " "- %(message)s" 44 | ) 45 | handle.setFormatter(formatter) 46 | logger.addHandler(handle) 47 | logger.setLevel(level) 48 | 49 | 50 | # To quickly see all messages, you can set the environment variable OCIFS_LOGGING_LEVEL=DEBUG. 51 | if "OCIFS_LOGGING_LEVEL" in os.environ: 52 | setup_logging_for_lakehouse() 53 | 54 | 55 | class LakehouseClient(object): 56 | """ 57 | Data Lake helps secure and govern data stored in Object Storage and other Oracle databases. 58 | """ 59 | 60 | def __init__(self, config, lake_service_api_endpoint: str = None, **kwargs): 61 | """ 62 | Creates a new service client 63 | 64 | :param dict config: 65 | Configuration keys and values as per `SDK and Tool Configuration `__. 66 | The :py:meth:`~oci.config.from_file` method can be used to load configuration from a file. Alternatively, a ``dict`` can be passed. You can validate_config 67 | the dict using :py:meth:`~oci.config.validate_config` 68 | 69 | :param str service_endpoint: (optional) 70 | The endpoint of the service to call using this client. For example ``https://iaas.us-ashburn-1.oraclecloud.com``. If this keyword argument is 71 | not provided then it will be derived using the region in the config parameter. You should only provide this keyword argument if you have an explicit 72 | need to specify a service endpoint. 73 | 74 | :param timeout: (optional) 75 | The connection and read timeouts for the client. The default values are connection timeout 10 seconds and read timeout 60 seconds. This keyword argument can be provided 76 | as a single float, in which case the value provided is used for both the read and connection timeouts, or as a tuple of two floats. If 77 | a tuple is provided then the first value is used as the connection timeout and the second value as the read timeout. 78 | :type timeout: float or tuple(float, float) 79 | 80 | :param signer: (optional) 81 | The signer to use when signing requests made by the service client. The default is to use a :py:class:`~oci.signer.Signer` based on the values 82 | provided in the config parameter. 83 | 84 | One use case for this parameter is for `Instance Principals authentication `__ 85 | by passing an instance of :py:class:`~oci.auth.signers.InstancePrincipalsSecurityTokenSigner` as the value for this keyword argument 86 | :type signer: :py:class:`~oci.signer.AbstractBaseSigner` 87 | 88 | :param obj retry_strategy: (optional) 89 | A retry strategy to apply to all calls made by this service client (i.e. at the client level). There is no retry strategy applied by default. 90 | Retry strategies can also be applied at the operation level by passing a ``retry_strategy`` keyword argument as part of calling the operation. 91 | Any value provided at the operation level will override whatever is specified at the client level. 92 | 93 | This should be one of the strategies available in the :py:mod:`~oci.retry` module. A convenience :py:data:`~oci.retry.DEFAULT_RETRY_STRATEGY` 94 | is also available. The specifics of the default retry strategy are described `here `__. 95 | """ 96 | validate_config(config, signer=kwargs.get("signer")) 97 | if "signer" in kwargs: 98 | signer = kwargs["signer"] 99 | elif AUTHENTICATION_TYPE_FIELD_NAME in config: 100 | signer = get_signer_from_authentication_type(config) 101 | else: 102 | signer = Signer( 103 | tenancy=config["tenancy"], 104 | user=config["user"], 105 | fingerprint=config["fingerprint"], 106 | private_key_file_location=config.get("key_file"), 107 | pass_phrase=get_config_value_or_default(config, "pass_phrase"), 108 | private_key_content=config.get("key_content"), 109 | ) 110 | self.lake_service_api_endpoint = lake_service_api_endpoint 111 | logger.debug( 112 | f"lakehouse service api endpoint is: {self.lake_service_api_endpoint}" 113 | ) 114 | base_client_init_kwargs = { 115 | "regional_client": True, 116 | "service_endpoint": self.lake_service_api_endpoint, 117 | "base_path": "/20221010", 118 | "service_endpoint_template": "https://lake.{region}.oci.{secondLevelDomain}", 119 | "service_endpoint_template": "https://objectstorage.{region}.{secondLevelDomain}", 120 | "endpoint_service_name": "lakehouse", 121 | "skip_deserialization": kwargs.get("skip_deserialization", False), 122 | "circuit_breaker_strategy": kwargs.get( 123 | "circuit_breaker_strategy", 124 | circuit_breaker.GLOBAL_CIRCUIT_BREAKER_STRATEGY, 125 | ), 126 | } 127 | if "timeout" in kwargs: 128 | base_client_init_kwargs["timeout"] = kwargs.get("timeout") 129 | if base_client_init_kwargs.get("circuit_breaker_strategy") is None: 130 | base_client_init_kwargs["circuit_breaker_strategy"] = ( 131 | circuit_breaker.DEFAULT_CIRCUIT_BREAKER_STRATEGY 132 | ) 133 | if "allow_control_chars" in kwargs: 134 | base_client_init_kwargs["allow_control_chars"] = kwargs.get( 135 | "allow_control_chars" 136 | ) 137 | self.retry_strategy = kwargs.get("retry_strategy") 138 | self.circuit_breaker_callback = kwargs.get("circuit_breaker_callback") 139 | self.base_client = BaseClient( 140 | "lakehouse", 141 | config, 142 | signer, 143 | lakehouse_type_mapping, 144 | **base_client_init_kwargs, 145 | ) 146 | logger.debug(f"lakehouse client got initialized !!!!!!!!") 147 | 148 | def get_lakeshare_endpoint(self, lake_id, **kwargs): 149 | """ 150 | Gets a lakesharing endpoint by for the given 151 | :param str lake_id: (required) 152 | unique Lake identifier 153 | :return: A :class:`~oci.response.Response` object with data of type :class:`~oci.lakehouse.models.Lakehouse` 154 | :rtype: :class:`~oci.response.Response` 155 | """ 156 | lake_resource_path = "/lakes/" + lake_id 157 | method = "GET" 158 | header_params = { 159 | "accept": "application/json", 160 | "content-type": "application/json", 161 | "opc-request-id": kwargs.get("opc_request_id", missing), 162 | } 163 | header_params = { 164 | k: v 165 | for (k, v) in six.iteritems(header_params) 166 | if v is not missing and v is not None 167 | } 168 | lakeresponse = None 169 | lakeshare_endpoint = None 170 | try: 171 | if self.retry_strategy: 172 | lakeresponse = self.retry_strategy.make_retrying_call( 173 | self.base_client.call_api, 174 | resource_path=lake_resource_path, 175 | method=method, 176 | header_params=header_params, 177 | response_type="Lakehouse", 178 | ) 179 | else: 180 | lakeresponse = self.base_client.call_api( 181 | resource_path=lake_resource_path, 182 | method=method, 183 | header_params=header_params, 184 | response_type="Lakehouse", 185 | ) 186 | lakeshare_endpoint = lakeresponse.data.lakeshare_endpoint 187 | logger.debug( 188 | f"lake response status:{lakeresponse.status}" 189 | f",lakeresponse data:{lakeresponse.data}" 190 | f",lakeshare_endpoint data:{lakeshare_endpoint}" 191 | f" and it's opc-request-id:{lakeresponse.headers['opc-request-id']}" 192 | ) 193 | 194 | except Exception as excep: 195 | logger.error( 196 | "Exception encountered when fetching lakesharing endpoint from lakeshouse details request fetch call " 197 | "for the given lake Ocid: " 198 | f"{lake_id} and exception details:{excep}" 199 | ) 200 | raise translate_oci_error(excep) from excep 201 | return lakeshare_endpoint 202 | 203 | def get_lakehouse_mount( 204 | self, 205 | lakehouse_id, 206 | mount_key: str, 207 | mount_type: str = None, 208 | mount_scope_entity_type: str = None, 209 | mount_scope_schema_key: str = None, 210 | mount_scope_table_key: str = None, 211 | mount_scope_user_id: str = None, 212 | **kwargs, 213 | ): 214 | """ 215 | Returns a Mount identified by the Mount Key. 216 | :param str lake_id: (required) 217 | unique Lake identifier 218 | :param str mount_key: (required) 219 | The unique key of the Mount. 220 | :param str mount_type: (required) 221 | mandatory parameter which decides type of Mount. 222 | Allowed values are: "EXTERNAL", "MANAGED" 223 | :param str mount_scope_entity_type: (optional) 224 | Scope for the Managed Mount to be queried 225 | Allowed values are: "DATABASE", "TABLE", "USER" 226 | :param str mount_scope_schema_key: (optional) 227 | Entity key for the selected scope of the Managed Mount. 228 | :param str mount_scope_table_key: (optional) 229 | Entity key for the selected scope of the Managed Mount. 230 | :param str mount_scope_user_id: (optional) 231 | Entity Id for the selected scope of the Managed Mount. 232 | :return: A :class:`~oci.response.Response` object with data of type :class:`~oci.lakehouse.models.LakehouseMountCollection` 233 | :rtype: :class:`~oci.response.Response` 234 | 235 | Parameters 236 | ---------- 237 | mount_name 238 | """ 239 | lake_resource_path = "/lakes/" + lakehouse_id + "/lakeMounts/" + mount_key 240 | method = "GET" 241 | if not mount_type: 242 | mount_type = "EXTERNAL" 243 | mount_type_allowed_values = ["EXTERNAL", "MANAGED"] 244 | if mount_type not in mount_type_allowed_values: 245 | raise ValueError( 246 | "Invalid value for `mount_type`, must be one of {0}".format( 247 | mount_type_allowed_values 248 | ) 249 | ) 250 | query_params = {} 251 | if mount_type == "MANAGED": 252 | mount_scope_entity_type_allowed_values = ["DATABASE", "TABLE", "USER"] 253 | if mount_scope_entity_type not in mount_scope_entity_type_allowed_values: 254 | raise ValueError( 255 | "Invalid value for `mount_scope_entity_type`, must be one of {0}".format( 256 | mount_scope_entity_type_allowed_values 257 | ) 258 | ) 259 | if mount_scope_entity_type == "DATABASE" and not mount_scope_schema_key: 260 | raise ValueError( 261 | "mount_scope_schema_key parameter cannot be None, whitespace or empty string" 262 | ) 263 | if mount_scope_entity_type == "TABLE": 264 | if not mount_scope_schema_key or not mount_scope_table_key: 265 | raise ValueError( 266 | "mount_scope_schema_key and mount_scope_table_key parameters cannot be None, " 267 | "whitespace or empty string" 268 | ) 269 | if mount_scope_entity_type == "USER": 270 | if not mount_scope_user_id: 271 | raise ValueError( 272 | "mount_scope_user_id parameter cannot be None, whitespace or empty string" 273 | ) 274 | query_params = { 275 | "mountType": mount_type, 276 | "mountScopeEntityType": mount_scope_entity_type, 277 | "mountScopeSchemaKey": mount_scope_schema_key, 278 | "mountScopeTableKey": mount_scope_table_key, 279 | "mountScopeUserId": mount_scope_user_id, 280 | } 281 | else: 282 | query_params = {"mountType": mount_type} 283 | header_params = { 284 | "accept": "application/json", 285 | "content-type": "application/json", 286 | "opc-request-id": kwargs.get("opc_request_id", missing), 287 | } 288 | header_params = { 289 | k: v 290 | for (k, v) in six.iteritems(header_params) 291 | if v is not missing and v is not None 292 | } 293 | 294 | retry_strategy = self.retry_strategy 295 | lake_mount = None 296 | try: 297 | if self.retry_strategy: 298 | lake_mount = retry_strategy.make_retrying_call( 299 | self.base_client.call_api, 300 | resource_path=lake_resource_path, 301 | method=method, 302 | query_params=query_params, 303 | header_params=header_params, 304 | response_type="LakeMount", 305 | ) 306 | else: 307 | lake_mount = self.base_client.call_api( 308 | resource_path=lake_resource_path, 309 | method=method, 310 | query_params=query_params, 311 | header_params=header_params, 312 | response_type="LakeMount", 313 | ) 314 | logger.debug( 315 | f"lake_mount response status:{lake_mount.status}" 316 | f",lake_mount data:{lake_mount.data}" 317 | ) 318 | logger.debug(f"lake_mount response mount_type:{lake_mount.data.mount_type}") 319 | except Exception as excep: 320 | logger.error( 321 | "Exception encountered when fetching bucket and namespace for the given mountName " 322 | "for the given lake Ocid: " 323 | f"{lakehouse_id}, mountName: {mount_key} and exception details:{excep}" 324 | ) 325 | raise translate_oci_error(excep) from excep 326 | 327 | return lake_mount 328 | -------------------------------------------------------------------------------- /ocifs/data_lake/managed_prefix_collection.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | from oci.util import ( 5 | formatted_flat_dict, 6 | NONE_SENTINEL, 7 | value_allowed_none_or_none_sentinel, 8 | ) # noqa: F401 9 | from oci.decorators import init_model_state_from_kwargs 10 | 11 | 12 | @init_model_state_from_kwargs 13 | class ManagedPrefixCollection(object): 14 | """ 15 | Results of a managedprefix path search for a bucket/namespace. Contains both ManagedPrefixPathSummary items and the fsUri. 16 | """ 17 | 18 | def __init__(self, **kwargs): 19 | """ 20 | Initializes a new ManagedPrefixCollection object with values from keyword arguments. 21 | The following keyword arguments are supported (corresponding to the getters/setters of this class): 22 | 23 | :param items: 24 | The value to assign to the items property of this ManagedPrefixCollection. 25 | :type items: list[oci.lakesharing.models.ManagedPrefixSummary] 26 | 27 | :param fs_uri: 28 | The value to assign to the fs_uri property of this ManagedPrefixCollection. 29 | :type fs_uri: str 30 | 31 | :param service_namespace: 32 | The value to assign to the service_namespace property of this ManagedPrefixCollection. 33 | :type service_namespace: str 34 | 35 | :param is_force_fallback_for_managed: 36 | The value to assign to the is_force_fallback_for_managed property of this ManagedPrefixCollection. 37 | :type is_force_fallback_for_managed: bool 38 | 39 | """ 40 | self.swagger_types = { 41 | "items": "list[ManagedPrefixSummary]", 42 | "fs_uri": "str", 43 | "service_namespace": "str", 44 | "is_force_fallback_for_managed": "bool", 45 | } 46 | 47 | self.attribute_map = { 48 | "items": "items", 49 | "fs_uri": "fsUri", 50 | "service_namespace": "serviceNamespace", 51 | "is_force_fallback_for_managed": "isForceFallbackForManaged", 52 | } 53 | 54 | self._items = None 55 | self._fs_uri = None 56 | self._service_namespace = None 57 | self._is_force_fallback_for_managed = None 58 | 59 | @property 60 | def items(self): 61 | """ 62 | **[Required]** Gets the items of this ManagedPrefixCollection. 63 | list of managedprefixes summary 64 | 65 | 66 | :return: The items of this ManagedPrefixCollection. 67 | :rtype: list[oci.lakesharing.models.ManagedPrefixSummary] 68 | """ 69 | return self._items 70 | 71 | @items.setter 72 | def items(self, items): 73 | """ 74 | Sets the items of this ManagedPrefixCollection. 75 | list of managedprefixes summary 76 | 77 | 78 | :param items: The items of this ManagedPrefixCollection. 79 | :type: list[oci.lakesharing.models.ManagedPrefixSummary] 80 | """ 81 | self._items = items 82 | 83 | @property 84 | def fs_uri(self): 85 | """ 86 | **[Required]** Gets the fs_uri of this ManagedPrefixCollection. 87 | fs Uri 88 | 89 | 90 | :return: The fs_uri of this ManagedPrefixCollection. 91 | :rtype: str 92 | """ 93 | return self._fs_uri 94 | 95 | @fs_uri.setter 96 | def fs_uri(self, fs_uri): 97 | """ 98 | Sets the fs_uri of this ManagedPrefixCollection. 99 | fs Uri 100 | 101 | 102 | :param fs_uri: The fs_uri of this ManagedPrefixCollection. 103 | :type: str 104 | """ 105 | self._fs_uri = fs_uri 106 | 107 | @property 108 | def service_namespace(self): 109 | """ 110 | **[Required]** Gets the service_namespace of this ManagedPrefixCollection. 111 | lakehouse service namespace 112 | 113 | 114 | :return: The service_namespace of this ManagedPrefixCollection. 115 | :rtype: str 116 | """ 117 | return self._service_namespace 118 | 119 | @service_namespace.setter 120 | def service_namespace(self, service_namespace): 121 | """ 122 | Sets the service_namespace of this ManagedPrefixCollection. 123 | lakehouse service namespace 124 | 125 | 126 | :param service_namespace: The service_namespace of this ManagedPrefixCollection. 127 | :type: str 128 | """ 129 | self._service_namespace = service_namespace 130 | 131 | @property 132 | def is_force_fallback_for_managed(self): 133 | """ 134 | **[Required]** Gets the is_force_fallback_for_managed of this ManagedPrefixCollection. 135 | If the request is from trusted entity server returns this flag true, otherwise false. 136 | 137 | 138 | :return: The is_force_fallback_for_managed of this ManagedPrefixCollection. 139 | :rtype: bool 140 | """ 141 | return self._is_force_fallback_for_managed 142 | 143 | @is_force_fallback_for_managed.setter 144 | def is_force_fallback_for_managed(self, is_force_fallback_for_managed): 145 | """ 146 | Sets the is_force_fallback_for_managed of this ManagedPrefixCollection. 147 | If the request is from trusted entity server returns this flag true, otherwise false. 148 | 149 | 150 | :param is_force_fallback_for_managed: The is_force_fallback_for_managed of this ManagedPrefixCollection. 151 | :type: bool 152 | """ 153 | self._is_force_fallback_for_managed = is_force_fallback_for_managed 154 | 155 | def __repr__(self): 156 | return formatted_flat_dict(self) 157 | 158 | def __eq__(self, other): 159 | if other is None: 160 | return False 161 | 162 | return self.__dict__ == other.__dict__ 163 | 164 | def __ne__(self, other): 165 | return not self == other 166 | -------------------------------------------------------------------------------- /ocifs/data_lake/managed_prefix_summary.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | from oci.util import ( 5 | formatted_flat_dict, 6 | NONE_SENTINEL, 7 | value_allowed_none_or_none_sentinel, 8 | ) # noqa: F401 9 | from oci.decorators import init_model_state_from_kwargs 10 | 11 | 12 | @init_model_state_from_kwargs 13 | class ManagedPrefixSummary(object): 14 | """ 15 | Returns the list of Managed prefix paths for a given bucket/namespace 16 | """ 17 | 18 | def __init__(self, **kwargs): 19 | """ 20 | Initializes a new ManagedPrefixSummary object with values from keyword arguments. 21 | The following keyword arguments are supported (corresponding to the getters/setters of this class): 22 | 23 | :param prefix_path: 24 | The value to assign to the prefix_path property of this ManagedPrefixSummary. 25 | :type prefix_path: str 26 | 27 | """ 28 | self.swagger_types = {"prefix_path": "str"} 29 | 30 | self.attribute_map = {"prefix_path": "prefixPath"} 31 | 32 | self._prefix_path = None 33 | 34 | @property 35 | def prefix_path(self): 36 | """ 37 | **[Required]** Gets the prefix_path of this ManagedPrefixSummary. 38 | managed prefix path 39 | 40 | 41 | :return: The prefix_path of this ManagedPrefixSummary. 42 | :rtype: str 43 | """ 44 | return self._prefix_path 45 | 46 | @prefix_path.setter 47 | def prefix_path(self, prefix_path): 48 | """ 49 | Sets the prefix_path of this ManagedPrefixSummary. 50 | managed prefix path 51 | 52 | 53 | :param prefix_path: The prefix_path of this ManagedPrefixSummary. 54 | :type: str 55 | """ 56 | self._prefix_path = prefix_path 57 | 58 | def __repr__(self): 59 | return formatted_flat_dict(self) 60 | 61 | def __eq__(self, other): 62 | if other is None: 63 | return False 64 | 65 | return self.__dict__ == other.__dict__ 66 | 67 | def __ne__(self, other): 68 | return not self == other 69 | -------------------------------------------------------------------------------- /ocifs/data_lake/mount_specification.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | from oci.util import ( 5 | formatted_flat_dict, 6 | NONE_SENTINEL, 7 | value_allowed_none_or_none_sentinel, 8 | ) # noqa: F401 9 | from oci.decorators import init_model_state_from_kwargs 10 | 11 | 12 | @init_model_state_from_kwargs 13 | class MountSpecification(object): 14 | """ 15 | Specification related to a new Mount. 16 | """ 17 | 18 | #: A constant which can be used with the mount_scope_entity_type property of a MountSpecification. 19 | #: This constant has a value of "DATABASE" 20 | MOUNT_SCOPE_ENTITY_TYPE_DATABASE = "DATABASE" 21 | 22 | #: A constant which can be used with the mount_scope_entity_type property of a MountSpecification. 23 | #: This constant has a value of "TABLE" 24 | MOUNT_SCOPE_ENTITY_TYPE_TABLE = "TABLE" 25 | 26 | #: A constant which can be used with the mount_scope_entity_type property of a MountSpecification. 27 | #: This constant has a value of "USER" 28 | MOUNT_SCOPE_ENTITY_TYPE_USER = "USER" 29 | 30 | def __init__(self, **kwargs): 31 | """ 32 | Initializes a new MountSpecification object with values from keyword arguments. 33 | The following keyword arguments are supported (corresponding to the getters/setters of this class): 34 | 35 | :param obj_store_location: 36 | The value to assign to the obj_store_location property of this MountSpecification. 37 | :type obj_store_location: str 38 | 39 | :param file_path: 40 | The value to assign to the file_path property of this MountSpecification. 41 | :type file_path: str 42 | 43 | :param namespace: 44 | The value to assign to the namespace property of this MountSpecification. 45 | :type namespace: str 46 | 47 | :param bucket_name: 48 | The value to assign to the bucket_name property of this MountSpecification. 49 | :type bucket_name: str 50 | 51 | :param mount_scope_entity_type: 52 | The value to assign to the mount_scope_entity_type property of this MountSpecification. 53 | Allowed values for this property are: "DATABASE", "TABLE", "USER", 'UNKNOWN_ENUM_VALUE'. 54 | Any unrecognized values returned by a service will be mapped to 'UNKNOWN_ENUM_VALUE'. 55 | :type mount_scope_entity_type: str 56 | 57 | :param mount_scope_database_key: 58 | The value to assign to the mount_scope_database_key property of this MountSpecification. 59 | :type mount_scope_database_key: str 60 | 61 | :param mount_scope_schema_key: 62 | The value to assign to the mount_scope_schema_key property of this MountSpecification. 63 | :type mount_scope_schema_key: str 64 | 65 | :param mount_scope_table_key: 66 | The value to assign to the mount_scope_table_key property of this MountSpecification. 67 | :type mount_scope_table_key: str 68 | 69 | :param mount_scope_user_id: 70 | The value to assign to the mount_scope_user_id property of this MountSpecification. 71 | :type mount_scope_user_id: str 72 | 73 | :param oci_fs_uri: 74 | The value to assign to the oci_fs_uri property of this MountSpecification. 75 | :type oci_fs_uri: str 76 | 77 | """ 78 | self.swagger_types = { 79 | "obj_store_location": "str", 80 | "file_path": "str", 81 | "namespace": "str", 82 | "bucket_name": "str", 83 | "mount_scope_entity_type": "str", 84 | "mount_scope_database_key": "str", 85 | "mount_scope_schema_key": "str", 86 | "mount_scope_table_key": "str", 87 | "mount_scope_user_id": "str", 88 | "oci_fs_uri": "str", 89 | } 90 | 91 | self.attribute_map = { 92 | "obj_store_location": "objStoreLocation", 93 | "file_path": "filePath", 94 | "namespace": "namespace", 95 | "bucket_name": "bucketName", 96 | "mount_scope_entity_type": "mountScopeEntityType", 97 | "mount_scope_database_key": "mountScopeDatabaseKey", 98 | "mount_scope_schema_key": "mountScopeSchemaKey", 99 | "mount_scope_table_key": "mountScopeTableKey", 100 | "mount_scope_user_id": "mountScopeUserId", 101 | "oci_fs_uri": "ociFsUri", 102 | } 103 | 104 | self._obj_store_location = None 105 | self._file_path = None 106 | self._namespace = None 107 | self._bucket_name = None 108 | self._mount_scope_entity_type = None 109 | self._mount_scope_database_key = None 110 | self._mount_scope_schema_key = None 111 | self._mount_scope_table_key = None 112 | self._mount_scope_user_id = None 113 | self._oci_fs_uri = None 114 | 115 | @property 116 | def obj_store_location(self): 117 | """ 118 | Gets the obj_store_location of this MountSpecification. 119 | path for the Object Storage Path. 120 | 121 | 122 | :return: The obj_store_location of this MountSpecification. 123 | :rtype: str 124 | """ 125 | return self._obj_store_location 126 | 127 | @obj_store_location.setter 128 | def obj_store_location(self, obj_store_location): 129 | """ 130 | Sets the obj_store_location of this MountSpecification. 131 | path for the Object Storage Path. 132 | 133 | 134 | :param obj_store_location: The obj_store_location of this MountSpecification. 135 | :type: str 136 | """ 137 | self._obj_store_location = obj_store_location 138 | 139 | @property 140 | def file_path(self): 141 | """ 142 | Gets the file_path of this MountSpecification. 143 | path to the File relative to the bucket. 144 | 145 | 146 | :return: The file_path of this MountSpecification. 147 | :rtype: str 148 | """ 149 | return self._file_path 150 | 151 | @file_path.setter 152 | def file_path(self, file_path): 153 | """ 154 | Sets the file_path of this MountSpecification. 155 | path to the File relative to the bucket. 156 | 157 | 158 | :param file_path: The file_path of this MountSpecification. 159 | :type: str 160 | """ 161 | self._file_path = file_path 162 | 163 | @property 164 | def namespace(self): 165 | """ 166 | Gets the namespace of this MountSpecification. 167 | Namespace where to create the new External Mount. 168 | 169 | 170 | :return: The namespace of this MountSpecification. 171 | :rtype: str 172 | """ 173 | return self._namespace 174 | 175 | @namespace.setter 176 | def namespace(self, namespace): 177 | """ 178 | Sets the namespace of this MountSpecification. 179 | Namespace where to create the new External Mount. 180 | 181 | 182 | :param namespace: The namespace of this MountSpecification. 183 | :type: str 184 | """ 185 | self._namespace = namespace 186 | 187 | @property 188 | def bucket_name(self): 189 | """ 190 | Gets the bucket_name of this MountSpecification. 191 | Bucket to be used to create new External Mount. 192 | 193 | 194 | :return: The bucket_name of this MountSpecification. 195 | :rtype: str 196 | """ 197 | return self._bucket_name 198 | 199 | @bucket_name.setter 200 | def bucket_name(self, bucket_name): 201 | """ 202 | Sets the bucket_name of this MountSpecification. 203 | Bucket to be used to create new External Mount. 204 | 205 | 206 | :param bucket_name: The bucket_name of this MountSpecification. 207 | :type: str 208 | """ 209 | self._bucket_name = bucket_name 210 | 211 | @property 212 | def mount_scope_entity_type(self): 213 | """ 214 | Gets the mount_scope_entity_type of this MountSpecification. 215 | Scope of the new Managed Mount. 216 | 217 | Allowed values for this property are: "DATABASE", "TABLE", "USER", 'UNKNOWN_ENUM_VALUE'. 218 | Any unrecognized values returned by a service will be mapped to 'UNKNOWN_ENUM_VALUE'. 219 | 220 | 221 | :return: The mount_scope_entity_type of this MountSpecification. 222 | :rtype: str 223 | """ 224 | return self._mount_scope_entity_type 225 | 226 | @mount_scope_entity_type.setter 227 | def mount_scope_entity_type(self, mount_scope_entity_type): 228 | """ 229 | Sets the mount_scope_entity_type of this MountSpecification. 230 | Scope of the new Managed Mount. 231 | 232 | 233 | :param mount_scope_entity_type: The mount_scope_entity_type of this MountSpecification. 234 | :type: str 235 | """ 236 | allowed_values = ["DATABASE", "TABLE", "USER"] 237 | if not value_allowed_none_or_none_sentinel( 238 | mount_scope_entity_type, allowed_values 239 | ): 240 | mount_scope_entity_type = "UNKNOWN_ENUM_VALUE" 241 | self._mount_scope_entity_type = mount_scope_entity_type 242 | 243 | @property 244 | def mount_scope_database_key(self): 245 | """ 246 | Gets the mount_scope_database_key of this MountSpecification. 247 | Name of the entity as per the Scope selected. 248 | 249 | 250 | :return: The mount_scope_database_key of this MountSpecification. 251 | :rtype: str 252 | """ 253 | return self._mount_scope_database_key 254 | 255 | @mount_scope_database_key.setter 256 | def mount_scope_database_key(self, mount_scope_database_key): 257 | """ 258 | Sets the mount_scope_database_key of this MountSpecification. 259 | Name of the entity as per the Scope selected. 260 | 261 | 262 | :param mount_scope_database_key: The mount_scope_database_key of this MountSpecification. 263 | :type: str 264 | """ 265 | self._mount_scope_database_key = mount_scope_database_key 266 | 267 | @property 268 | def mount_scope_schema_key(self): 269 | """ 270 | Gets the mount_scope_schema_key of this MountSpecification. 271 | Name of the entity as per the Scope selected. 272 | 273 | 274 | :return: The mount_scope_schema_key of this MountSpecification. 275 | :rtype: str 276 | """ 277 | return self._mount_scope_schema_key 278 | 279 | @mount_scope_schema_key.setter 280 | def mount_scope_schema_key(self, mount_scope_schema_key): 281 | """ 282 | Sets the mount_scope_schema_key of this MountSpecification. 283 | Name of the entity as per the Scope selected. 284 | 285 | 286 | :param mount_scope_schema_key: The mount_scope_schema_key of this MountSpecification. 287 | :type: str 288 | """ 289 | self._mount_scope_schema_key = mount_scope_schema_key 290 | 291 | @property 292 | def mount_scope_table_key(self): 293 | """ 294 | Gets the mount_scope_table_key of this MountSpecification. 295 | Name of the entity as per the Scope selected. 296 | 297 | 298 | :return: The mount_scope_table_key of this MountSpecification. 299 | :rtype: str 300 | """ 301 | return self._mount_scope_table_key 302 | 303 | @mount_scope_table_key.setter 304 | def mount_scope_table_key(self, mount_scope_table_key): 305 | """ 306 | Sets the mount_scope_table_key of this MountSpecification. 307 | Name of the entity as per the Scope selected. 308 | 309 | 310 | :param mount_scope_table_key: The mount_scope_table_key of this MountSpecification. 311 | :type: str 312 | """ 313 | self._mount_scope_table_key = mount_scope_table_key 314 | 315 | @property 316 | def mount_scope_user_id(self): 317 | """ 318 | Gets the mount_scope_user_id of this MountSpecification. 319 | Name of the entity as per the Scope selected. 320 | 321 | 322 | :return: The mount_scope_user_id of this MountSpecification. 323 | :rtype: str 324 | """ 325 | return self._mount_scope_user_id 326 | 327 | @mount_scope_user_id.setter 328 | def mount_scope_user_id(self, mount_scope_user_id): 329 | """ 330 | Sets the mount_scope_user_id of this MountSpecification. 331 | Name of the entity as per the Scope selected. 332 | 333 | 334 | :param mount_scope_user_id: The mount_scope_user_id of this MountSpecification. 335 | :type: str 336 | """ 337 | self._mount_scope_user_id = mount_scope_user_id 338 | 339 | @property 340 | def oci_fs_uri(self): 341 | """ 342 | Gets the oci_fs_uri of this MountSpecification. 343 | OCI FS URI which is used as URI for OCI FS (Python) 344 | 345 | 346 | :return: The oci_fs_uri of this MountSpecification. 347 | :rtype: str 348 | """ 349 | return self._oci_fs_uri 350 | 351 | @oci_fs_uri.setter 352 | def oci_fs_uri(self, oci_fs_uri): 353 | """ 354 | Sets the oci_fs_uri of this MountSpecification. 355 | OCI FS URI which is used as URI for OCI FS (Python) 356 | 357 | 358 | :param oci_fs_uri: The oci_fs_uri of this MountSpecification. 359 | :type: str 360 | """ 361 | self._oci_fs_uri = oci_fs_uri 362 | 363 | def __repr__(self): 364 | return formatted_flat_dict(self) 365 | 366 | def __eq__(self, other): 367 | if other is None: 368 | return False 369 | 370 | return self.__dict__ == other.__dict__ 371 | 372 | def __ne__(self, other): 373 | return not self == other 374 | -------------------------------------------------------------------------------- /ocifs/data_lake/par_response.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | from oci.util import ( 5 | formatted_flat_dict, 6 | NONE_SENTINEL, 7 | value_allowed_none_or_none_sentinel, 8 | ) # noqa: F401 9 | from oci.decorators import init_model_state_from_kwargs 10 | 11 | 12 | @init_model_state_from_kwargs 13 | class ParResponse(object): 14 | """ 15 | LakeSharing Par Hash 16 | """ 17 | 18 | #: A constant which can be used with the par_type property of a ParResponse. 19 | #: This constant has a value of "DATABASE" 20 | PAR_TYPE_DATABASE = "DATABASE" 21 | 22 | #: A constant which can be used with the par_type property of a ParResponse. 23 | #: This constant has a value of "TABLE" 24 | PAR_TYPE_TABLE = "TABLE" 25 | 26 | #: A constant which can be used with the par_type property of a ParResponse. 27 | #: This constant has a value of "MOUNT" 28 | PAR_TYPE_MOUNT = "MOUNT" 29 | 30 | #: A constant which can be used with the par_type property of a ParResponse. 31 | #: This constant has a value of "PATH" 32 | PAR_TYPE_PATH = "PATH" 33 | 34 | def __init__(self, **kwargs): 35 | """ 36 | Initializes a new ParResponse object with values from keyword arguments. 37 | The following keyword arguments are supported (corresponding to the getters/setters of this class): 38 | 39 | :param par_hash: 40 | The value to assign to the par_hash property of this ParResponse. 41 | :type par_hash: str 42 | 43 | :param prefix_path: 44 | The value to assign to the prefix_path property of this ParResponse. 45 | :type prefix_path: str 46 | 47 | :param par_type: 48 | The value to assign to the par_type property of this ParResponse. 49 | Allowed values for this property are: "DATABASE", "TABLE", "MOUNT", "PATH", 'UNKNOWN_ENUM_VALUE'. 50 | Any unrecognized values returned by a service will be mapped to 'UNKNOWN_ENUM_VALUE'. 51 | :type par_type: str 52 | 53 | """ 54 | self.swagger_types = { 55 | "par_hash": "str", 56 | "prefix_path": "str", 57 | "par_type": "str", 58 | } 59 | 60 | self.attribute_map = { 61 | "par_hash": "parHash", 62 | "prefix_path": "prefixPath", 63 | "par_type": "parType", 64 | } 65 | 66 | self._par_hash = None 67 | self._prefix_path = None 68 | self._par_type = None 69 | 70 | @property 71 | def par_hash(self): 72 | """ 73 | **[Required]** Gets the par_hash of this ParResponse. 74 | PAR Hash for the bucket/object for a given namespace, scoped to the operation and exipry time 75 | 76 | 77 | :return: The par_hash of this ParResponse. 78 | :rtype: str 79 | """ 80 | return self._par_hash 81 | 82 | @par_hash.setter 83 | def par_hash(self, par_hash): 84 | """ 85 | Sets the par_hash of this ParResponse. 86 | PAR Hash for the bucket/object for a given namespace, scoped to the operation and exipry time 87 | 88 | 89 | :param par_hash: The par_hash of this ParResponse. 90 | :type: str 91 | """ 92 | self._par_hash = par_hash 93 | 94 | @property 95 | def prefix_path(self): 96 | """ 97 | Gets the prefix_path of this ParResponse. 98 | Prefix path for the parHash. 99 | 100 | 101 | :return: The prefix_path of this ParResponse. 102 | :rtype: str 103 | """ 104 | return self._prefix_path 105 | 106 | @prefix_path.setter 107 | def prefix_path(self, prefix_path): 108 | """ 109 | Sets the prefix_path of this ParResponse. 110 | Prefix path for the parHash. 111 | 112 | 113 | :param prefix_path: The prefix_path of this ParResponse. 114 | :type: str 115 | """ 116 | self._prefix_path = prefix_path 117 | 118 | @property 119 | def par_type(self): 120 | """ 121 | Gets the par_type of this ParResponse. 122 | Type of Par (DB/TABLE) 123 | 124 | Allowed values for this property are: "DATABASE", "TABLE", "MOUNT", "PATH", 'UNKNOWN_ENUM_VALUE'. 125 | Any unrecognized values returned by a service will be mapped to 'UNKNOWN_ENUM_VALUE'. 126 | 127 | 128 | :return: The par_type of this ParResponse. 129 | :rtype: str 130 | """ 131 | return self._par_type 132 | 133 | @par_type.setter 134 | def par_type(self, par_type): 135 | """ 136 | Sets the par_type of this ParResponse. 137 | Type of Par (DB/TABLE) 138 | 139 | 140 | :param par_type: The par_type of this ParResponse. 141 | :type: str 142 | """ 143 | allowed_values = ["DATABASE", "TABLE", "MOUNT", "PATH"] 144 | if not value_allowed_none_or_none_sentinel(par_type, allowed_values): 145 | par_type = "UNKNOWN_ENUM_VALUE" 146 | self._par_type = par_type 147 | 148 | def __repr__(self): 149 | return formatted_flat_dict(self) 150 | 151 | def __eq__(self, other): 152 | if other is None: 153 | return False 154 | 155 | return self.__dict__ == other.__dict__ 156 | 157 | def __ne__(self, other): 158 | return not self == other 159 | -------------------------------------------------------------------------------- /ocifs/data_lake/rename_object_details.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | from oci.util import ( 5 | formatted_flat_dict, 6 | NONE_SENTINEL, 7 | value_allowed_none_or_none_sentinel, 8 | ) # noqa: F401 9 | from oci.decorators import init_model_state_from_kwargs 10 | 11 | 12 | @init_model_state_from_kwargs 13 | class RenameObjectDetails(object): 14 | """ 15 | Rename object details 16 | """ 17 | 18 | def __init__(self, **kwargs): 19 | """ 20 | Initializes a new RenameObjectDetails object with values from keyword arguments. 21 | The following keyword arguments are supported (corresponding to the getters/setters of this class): 22 | 23 | :param source_name: 24 | The value to assign to the source_name property of this RenameObjectDetails. 25 | :type source_name: str 26 | 27 | :param new_name: 28 | The value to assign to the new_name property of this RenameObjectDetails. 29 | :type new_name: str 30 | 31 | :param src_obj_if_match_e_tag: 32 | The value to assign to the src_obj_if_match_e_tag property of this RenameObjectDetails. 33 | :type src_obj_if_match_e_tag: str 34 | 35 | :param new_obj_if_match_e_tag: 36 | The value to assign to the new_obj_if_match_e_tag property of this RenameObjectDetails. 37 | :type new_obj_if_match_e_tag: str 38 | 39 | :param new_obj_if_none_match_e_tag: 40 | The value to assign to the new_obj_if_none_match_e_tag property of this RenameObjectDetails. 41 | :type new_obj_if_none_match_e_tag: str 42 | 43 | """ 44 | self.swagger_types = { 45 | "source_name": "str", 46 | "new_name": "str", 47 | "src_obj_if_match_e_tag": "str", 48 | "new_obj_if_match_e_tag": "str", 49 | "new_obj_if_none_match_e_tag": "str", 50 | } 51 | 52 | self.attribute_map = { 53 | "source_name": "sourceName", 54 | "new_name": "newName", 55 | "src_obj_if_match_e_tag": "srcObjIfMatchETag", 56 | "new_obj_if_match_e_tag": "newObjIfMatchETag", 57 | "new_obj_if_none_match_e_tag": "newObjIfNoneMatchETag", 58 | } 59 | 60 | self._source_name = None 61 | self._new_name = None 62 | self._src_obj_if_match_e_tag = None 63 | self._new_obj_if_match_e_tag = None 64 | self._new_obj_if_none_match_e_tag = None 65 | 66 | @property 67 | def source_name(self): 68 | """ 69 | **[Required]** Gets the source_name of this RenameObjectDetails. 70 | Name of source object 71 | 72 | 73 | :return: The source_name of this RenameObjectDetails. 74 | :rtype: str 75 | """ 76 | return self._source_name 77 | 78 | @source_name.setter 79 | def source_name(self, source_name): 80 | """ 81 | Sets the source_name of this RenameObjectDetails. 82 | Name of source object 83 | 84 | 85 | :param source_name: The source_name of this RenameObjectDetails. 86 | :type: str 87 | """ 88 | self._source_name = source_name 89 | 90 | @property 91 | def new_name(self): 92 | """ 93 | **[Required]** Gets the new_name of this RenameObjectDetails. 94 | Name of destination object 95 | 96 | 97 | :return: The new_name of this RenameObjectDetails. 98 | :rtype: str 99 | """ 100 | return self._new_name 101 | 102 | @new_name.setter 103 | def new_name(self, new_name): 104 | """ 105 | Sets the new_name of this RenameObjectDetails. 106 | Name of destination object 107 | 108 | 109 | :param new_name: The new_name of this RenameObjectDetails. 110 | :type: str 111 | """ 112 | self._new_name = new_name 113 | 114 | @property 115 | def src_obj_if_match_e_tag(self): 116 | """ 117 | Gets the src_obj_if_match_e_tag of this RenameObjectDetails. 118 | srcObjIfMatchETag 119 | 120 | 121 | :return: The src_obj_if_match_e_tag of this RenameObjectDetails. 122 | :rtype: str 123 | """ 124 | return self._src_obj_if_match_e_tag 125 | 126 | @src_obj_if_match_e_tag.setter 127 | def src_obj_if_match_e_tag(self, src_obj_if_match_e_tag): 128 | """ 129 | Sets the src_obj_if_match_e_tag of this RenameObjectDetails. 130 | srcObjIfMatchETag 131 | 132 | 133 | :param src_obj_if_match_e_tag: The src_obj_if_match_e_tag of this RenameObjectDetails. 134 | :type: str 135 | """ 136 | self._src_obj_if_match_e_tag = src_obj_if_match_e_tag 137 | 138 | @property 139 | def new_obj_if_match_e_tag(self): 140 | """ 141 | Gets the new_obj_if_match_e_tag of this RenameObjectDetails. 142 | newObjIfMatchETag 143 | 144 | 145 | :return: The new_obj_if_match_e_tag of this RenameObjectDetails. 146 | :rtype: str 147 | """ 148 | return self._new_obj_if_match_e_tag 149 | 150 | @new_obj_if_match_e_tag.setter 151 | def new_obj_if_match_e_tag(self, new_obj_if_match_e_tag): 152 | """ 153 | Sets the new_obj_if_match_e_tag of this RenameObjectDetails. 154 | newObjIfMatchETag 155 | 156 | 157 | :param new_obj_if_match_e_tag: The new_obj_if_match_e_tag of this RenameObjectDetails. 158 | :type: str 159 | """ 160 | self._new_obj_if_match_e_tag = new_obj_if_match_e_tag 161 | 162 | @property 163 | def new_obj_if_none_match_e_tag(self): 164 | """ 165 | Gets the new_obj_if_none_match_e_tag of this RenameObjectDetails. 166 | newObjIfNoneMatchETag 167 | 168 | 169 | :return: The new_obj_if_none_match_e_tag of this RenameObjectDetails. 170 | :rtype: str 171 | """ 172 | return self._new_obj_if_none_match_e_tag 173 | 174 | @new_obj_if_none_match_e_tag.setter 175 | def new_obj_if_none_match_e_tag(self, new_obj_if_none_match_e_tag): 176 | """ 177 | Sets the new_obj_if_none_match_e_tag of this RenameObjectDetails. 178 | newObjIfNoneMatchETag 179 | 180 | 181 | :param new_obj_if_none_match_e_tag: The new_obj_if_none_match_e_tag of this RenameObjectDetails. 182 | :type: str 183 | """ 184 | self._new_obj_if_none_match_e_tag = new_obj_if_none_match_e_tag 185 | 186 | def __repr__(self): 187 | return formatted_flat_dict(self) 188 | 189 | def __eq__(self, other): 190 | if other is None: 191 | return False 192 | 193 | return self.__dict__ == other.__dict__ 194 | 195 | def __ne__(self, other): 196 | return not self == other 197 | -------------------------------------------------------------------------------- /ocifs/errors.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | import errno 5 | import functools 6 | 7 | 8 | # Fallback values since some systems might not have these. 9 | EREMOTEIO = getattr(errno, "EREMOTEIO", errno.EIO) 10 | 11 | 12 | ERROR_CODE_TO_EXCEPTION = { 13 | "CannotParseRequest": functools.partial(IOError, errno.EINVAL), 14 | "InvalidParameter": functools.partial(IOError, errno.EINVAL), 15 | "LimitExceeded": functools.partial(IOError, errno.EINVAL), 16 | "MissingParameter": functools.partial(IOError, errno.EINVAL), 17 | "QuotaExceeded": functools.partial(IOError, errno.EINVAL), 18 | "RelatedResourceNotAuthorizedOrNotFound": functools.partial(IOError, errno.EINVAL), 19 | "NotAuthenticated": PermissionError, 20 | "SignUpRequired": PermissionError, 21 | "NotAuthorized": PermissionError, 22 | "NotAuthorizedOrNotFound": FileNotFoundError, 23 | "NotFound": FileNotFoundError, 24 | "MethodNotAllowed": functools.partial(IOError, errno.EPERM), 25 | "IncorrectState": functools.partial(IOError, errno.EBUSY), 26 | "InvalidatedRetryToken": functools.partial(IOError, errno.EBUSY), 27 | "NotAuthorizedOr ResourceAlreadyExists": functools.partial(IOError, errno.EBUSY), 28 | "NotAuthorizedOrResourceAlreadyExists": functools.partial(IOError, errno.EBUSY), 29 | "NoEtagMatch": functools.partial(IOError, errno.EINVAL), 30 | "TooManyRequests": functools.partial(IOError, errno.EBUSY), 31 | "InternalServerError": functools.partial(IOError, EREMOTEIO), 32 | "MethodNotImplemented": functools.partial(IOError, errno.ENOSYS), 33 | "ServiceUnavailable": functools.partial(IOError, errno.EBUSY), 34 | "400": functools.partial(IOError, errno.EINVAL), 35 | "401": PermissionError, 36 | "402": PermissionError, 37 | "403": PermissionError, 38 | "404": FileNotFoundError, 39 | "405": functools.partial(IOError, errno.EPERM), 40 | "409": functools.partial(IOError, errno.EBUSY), 41 | "412": functools.partial(IOError, errno.EINVAL), # PreconditionFailed 42 | "429": functools.partial(IOError, errno.EBUSY), # SlowDown 43 | "500": functools.partial(IOError, EREMOTEIO), # InternalError 44 | "501": functools.partial(IOError, errno.ENOSYS), # NotImplemented 45 | "503": functools.partial(IOError, errno.EBUSY), # SlowDown 46 | } 47 | 48 | 49 | def translate_oci_error(error, message=None, *args, **kwargs): 50 | """Convert a ClientError exception into a Python one. 51 | Parameters 52 | ---------- 53 | error : oci.exceptions.ServiceError 54 | The exception returned by the OCI Object Storage API. 55 | message : str 56 | An error message to use for the returned exception. If not given, the 57 | error message returned by the server is used instead. 58 | *args, **kwargs : 59 | Additional arguments to pass to the exception constructor, after the 60 | error message. Useful for passing the filename arguments to ``IOError``. 61 | Returns 62 | ------- 63 | An instantiated exception ready to be thrown. If the error code isn't 64 | recognized, an IOError with the original error message is returned. 65 | """ 66 | if not hasattr(error, "code") or not hasattr(error, "status"): 67 | return error 68 | constructor = ERROR_CODE_TO_EXCEPTION.get( 69 | error.code, ERROR_CODE_TO_EXCEPTION.get(str(error.status)) 70 | ) 71 | if not constructor: 72 | # No match found, wrap this in an IOError with the appropriate message. 73 | return IOError(errno.EIO, message or str(error), *args) 74 | 75 | if not message: 76 | message = error.message if error.message is not None else str(error) 77 | 78 | return constructor(message, *args, **kwargs) 79 | -------------------------------------------------------------------------------- /ocifs/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | from ..core import OCIFileSystem 5 | -------------------------------------------------------------------------------- /ocifs/tests/test-requirements.txt: -------------------------------------------------------------------------------- 1 | ruff 2 | pytest 3 | -------------------------------------------------------------------------------- /ocifs/tests/test_integration.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2025 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | import pandas as pd 5 | import numpy as np 6 | import os 7 | import pytest 8 | import oci 9 | from ocifs import OCIFileSystem 10 | 11 | storage_options = {} 12 | config = {} 13 | 14 | namespace_name = os.environ["OCIFS_TEST_NAMESPACE"] 15 | test_bucket_name = os.environ["OCIFS_TEST_BUCKET"] 16 | remote_folder = f"oci://{test_bucket_name}-int@{namespace_name}/sample_data" 17 | 18 | 19 | @pytest.fixture(autouse=True) 20 | def reset_folder(): 21 | oci_fs = OCIFileSystem(config=config) 22 | try: 23 | oci_fs.rm(remote_folder, recursive=True) 24 | except FileNotFoundError: 25 | pass 26 | yield 27 | 28 | 29 | def test_rw_small(): 30 | sample_data = {"A": [1, 2], "B": [3, 4]} 31 | small_fn = os.path.join(remote_folder, "small.csv") 32 | 33 | df = pd.DataFrame(sample_data) 34 | df.to_csv(small_fn, index=False, storage_options=storage_options) 35 | df_reloaded = pd.read_csv(small_fn, storage_options=storage_options) 36 | assert df_reloaded.equals(df) 37 | 38 | 39 | def test_rw_large(): 40 | sample_data = { 41 | "A": np.arange(10000), 42 | "B": np.arange(10000) * 2, 43 | "C": np.arange(10000) * 3, 44 | } 45 | large_fn = os.path.join(remote_folder, "large.csv") 46 | 47 | df = pd.DataFrame(sample_data) 48 | df.to_csv(large_fn, index=False, storage_options=storage_options) 49 | df_reloaded = pd.read_csv(large_fn, storage_options=storage_options) 50 | assert df_reloaded.equals(df) 51 | 52 | 53 | def test_new_bucket(): 54 | storage_options2 = { 55 | "create_parents": True, 56 | } 57 | storage_options2.update(storage_options) 58 | sample_data = {"A": [1, 2], "B": [3, 4]} 59 | small_fn = os.path.join(remote_folder, "small.csv") 60 | 61 | df = pd.DataFrame(sample_data) 62 | df.to_csv(small_fn, index=False, storage_options=storage_options2) 63 | 64 | df_reloaded = pd.read_csv(small_fn, storage_options=storage_options2) 65 | assert df_reloaded.equals(df) 66 | -------------------------------------------------------------------------------- /ocifs/tests/test_integration_lake.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | import pandas as pd 5 | import os 6 | import pytest 7 | import oci 8 | from ocifs import OCIFileSystem 9 | 10 | config = oci.config.from_file("~/.oci/config", profile_name="iad_prod") 11 | full_external_mount_name = os.environ["OCIFS_EXTERNAL_MOUNT_URI"] 12 | storage_options = {"config": config} 13 | 14 | 15 | @pytest.fixture(autouse=True) 16 | def reset_folder(): 17 | oci_fs = OCIFileSystem(config=config, profile="iad_prod") 18 | try: 19 | if oci_fs.lexists(full_external_mount_name + "/a/employees.csv"): 20 | oci_fs.rm(full_external_mount_name + "/a/employees.csv") 21 | except FileNotFoundError: 22 | pass 23 | yield 24 | 25 | 26 | def test_rw_small(): 27 | users = {"Name": ["Amit", "Cody", "Drew"], "Age": [20, 21, 25]} 28 | # create DataFrame 29 | df = pd.DataFrame(users, columns=["Name", "Age"]) 30 | df.to_csv( 31 | full_external_mount_name + "/a/employees.csv", 32 | index=False, 33 | header=True, 34 | storage_options=storage_options, 35 | ) 36 | df_reloaded = pd.read_csv( 37 | full_external_mount_name + "/a/employees.csv", storage_options=storage_options 38 | ) 39 | assert df_reloaded.equals(df) 40 | -------------------------------------------------------------------------------- /ocifs/tests/test_spec_lake.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023, 2024 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | 5 | import io 6 | import fsspec 7 | import pytest 8 | import oci 9 | import os 10 | from ocifs import OCIFileSystem 11 | from ocifs.errors import translate_oci_error 12 | from oci._vendor.requests.structures import CaseInsensitiveDict 13 | from oci.exceptions import ServiceError, ProfileNotFound 14 | from ocifs.data_lake.lake_sharing_object_storage_client import ( 15 | LakeSharingObjectStorageClient, 16 | ) 17 | 18 | 19 | profile_name = os.environ["OCIFS_CONFIG_PROFILE"] 20 | config = oci.config.from_file("~/.oci/config", profile_name=profile_name) 21 | full_external_mount_name = os.environ["OCIFS_EXTERNAL_MOUNT_URI"] 22 | storage_options = {"config": config} 23 | test_bucket_with_namespace = "" 24 | os.environ["OCIFS_IAM_TYPE"] = "api_key" 25 | files = { 26 | "test/accounts.1.json": ( 27 | b'{"amount": 100, "name": "Alice"}\n' 28 | b'{"amount": 200, "name": "Bob"}\n' 29 | b'{"amount": 300, "name": "Charlie"}\n' 30 | b'{"amount": 400, "name": "Dennis"}\n' 31 | ), 32 | "test/accounts.2.json": ( 33 | b'{"amount": 500, "name": "Alice"}\n' 34 | b'{"amount": 600, "name": "Bob"}\n' 35 | b'{"amount": 700, "name": "Charlie"}\n' 36 | b'{"amount": 800, "name": "Dennis"}\n' 37 | ), 38 | } 39 | 40 | csv_files = { 41 | "2014-01-01.csv": ( 42 | b"name,amount,id\n" b"Alice,100,1\n" b"Bob,200,2\n" b"Charlie,300,3\n" 43 | ), 44 | "2014-01-02.csv": (b"name,amount,id\n"), 45 | "2014-01-03.csv": ( 46 | b"name,amount,id\n" b"Dennis,400,4\n" b"Edith,500,5\n" b"Frank,600,6\n" 47 | ), 48 | } 49 | text_files = { 50 | "nested/file1": b"hello\n", 51 | "nested/file2": b"world", 52 | "nested/nested2/file1": b"hello\n", 53 | "nested/nested2/file2": b"world", 54 | } 55 | glob_files = {"file.dat": b"", "filexdat": b""} 56 | a_path = "tmp/test/a" 57 | b_path = "tmp/test/b" 58 | c_path = "tmp/test/c" 59 | a = os.path.join(full_external_mount_name, a_path) 60 | b = os.path.join(full_external_mount_name, b_path) 61 | c = os.path.join(full_external_mount_name, c_path) 62 | 63 | 64 | @pytest.fixture 65 | def fs(): 66 | OCIFileSystem.clear_instance_cache() 67 | fs = OCIFileSystem( 68 | config="~/.oci/config", profile=profile_name 69 | ) # Using env var to set IAM type 70 | try: 71 | client = LakeSharingObjectStorageClient(config) 72 | except ServiceError as e: 73 | raise translate_oci_error(e) from e 74 | for flist in [files, csv_files, text_files, glob_files]: 75 | for f, values in flist.items(): 76 | file_path = os.path.join(full_external_mount_name, f) 77 | fs.touch(file_path, truncate=True, data=values) 78 | fs.invalidate_cache() 79 | yield fs 80 | 81 | 82 | def test_simple(fs): 83 | data = b"a" * (10 * 2**20) 84 | 85 | with fs.open(a, "wb") as f: 86 | f.write(data) 87 | 88 | with fs.open(a, "rb") as f: 89 | out = f.read(len(data)) 90 | assert len(data) == len(out) 91 | assert out == data 92 | 93 | 94 | @pytest.mark.parametrize("default_cache_type", ["none", "bytes"]) 95 | def test_default_cache_type(default_cache_type): 96 | data = b"a" * (10 * 2**20) 97 | oci_fs = OCIFileSystem(config=config, default_cache_type=default_cache_type) 98 | 99 | with oci_fs.open(a, "wb") as f: 100 | f.write(data) 101 | 102 | with oci_fs.open(a, "rb") as f: 103 | assert isinstance(f.cache, fsspec.core.caches[default_cache_type]) 104 | out = f.read(len(data)) 105 | assert len(data) == len(out) 106 | assert out == data 107 | 108 | 109 | def test_idempotent_connect(fs): 110 | con1 = fs.connect() 111 | con2 = fs.connect(refresh=False) 112 | con3 = fs.connect(refresh=True) 113 | assert con1 is con2 114 | assert con1 is not con3 115 | 116 | 117 | def test_find(fs): 118 | assert fs.find(full_external_mount_name) 119 | 120 | 121 | def test_oci_file_access(fs): 122 | fn = full_external_mount_name + "/nested/file1" 123 | data = b"hello\n" 124 | assert fs.cat(fn) == data 125 | assert fs.head(fn, 3) == data[:3] 126 | assert fs.tail(fn, 3) == data[-3:] 127 | assert fs.tail(fn, 10000) == data 128 | 129 | 130 | def test_multiple_objects(fs): 131 | fs.connect() 132 | assert fs.ls(full_external_mount_name) 133 | fs2 = OCIFileSystem(storage_options["config"]) 134 | assert fs.ls(full_external_mount_name) == fs2.ls(full_external_mount_name) 135 | 136 | 137 | def test_ls(fs): 138 | bucket, namespace, key = fs.split_path(full_external_mount_name + "/test") 139 | test_bucket_with_namespace = bucket + "@" + namespace + "/" 140 | fn = test_bucket_with_namespace + "test/accounts.1.json" 141 | assert fn in fs.ls(full_external_mount_name + "/test") 142 | 143 | 144 | def test_ls_touch(fs): 145 | bucket, namespace, key = fs.split_path(full_external_mount_name + "/test") 146 | test_bucket_with_namespace = bucket + "@" + namespace + "/" 147 | test_dir_path = test_bucket_with_namespace + "tmp/test/" 148 | fs.touch(a) 149 | fs.touch(b) 150 | L = fs.ls(full_external_mount_name + "/tmp/test", detail=True) 151 | assert {d["name"] for d in L} == {test_dir_path + "a", test_dir_path + "b"} 152 | 153 | 154 | def test_isfile(fs): 155 | assert not fs.isfile(full_external_mount_name) 156 | assert not fs.isfile(full_external_mount_name + "/test") 157 | assert fs.isfile(full_external_mount_name + "/test/accounts.1.json") 158 | assert fs.isfile(full_external_mount_name + "/test/accounts.2.json") 159 | 160 | 161 | def test_isdir(fs): 162 | assert fs.isdir(full_external_mount_name) 163 | assert fs.isdir(full_external_mount_name + "/test") 164 | assert not fs.isdir(full_external_mount_name + "/test/accounts.1.json") 165 | assert not fs.isdir(full_external_mount_name + "/test/accounts.2.json") 166 | assert not fs.isdir(b + "/") 167 | assert not fs.isdir(c) 168 | 169 | 170 | def test_oci_file_info(fs): 171 | bucket, namespace, key = fs.split_path(full_external_mount_name + "/test") 172 | test_bucket_with_namespace = bucket + "@" + namespace + "/" 173 | nested_file1_path = test_bucket_with_namespace + "nested/file1" 174 | fn = full_external_mount_name + "/nested/file1" 175 | data = b"hello\n" 176 | assert nested_file1_path in fs.find(full_external_mount_name) 177 | assert fs.exists(fn) 178 | assert not fs.exists(fn + "another") 179 | assert fs.info(fn)["size"] == len(data) 180 | with pytest.raises(FileNotFoundError): 181 | fs.info(fn + "another") 182 | 183 | 184 | def test_du(fs): 185 | d = fs.du(full_external_mount_name, total=False) 186 | assert all(isinstance(v, int) and v >= 0 for v in d.values()) 187 | bucket, namespace, key = fs.split_path(full_external_mount_name + "/test") 188 | test_bucket_with_namespace = bucket + "@" + namespace + "/" 189 | nested_file1_path = test_bucket_with_namespace + "nested/file1" 190 | assert nested_file1_path in d 191 | assert fs.du(full_external_mount_name + "/test/", total=True) == sum( 192 | map(len, files.values()) 193 | ) 194 | 195 | 196 | def test_oci_ls(fs): 197 | bucket, namespace, key = fs.split_path(full_external_mount_name + "/test") 198 | test_bucket_with_namespace = bucket + "@" + namespace + "/" 199 | nested_file1_path = test_bucket_with_namespace + "nested/file1" 200 | assert nested_file1_path not in fs.ls(full_external_mount_name + "/") 201 | assert nested_file1_path in fs.ls(full_external_mount_name + "/nested/") 202 | assert nested_file1_path in fs.ls(full_external_mount_name + "/nested") 203 | 204 | 205 | def test_oci_ls_detail(fs): 206 | L = fs.ls(full_external_mount_name + "/nested", detail=True) 207 | assert all(isinstance(item, CaseInsensitiveDict) for item in L) 208 | 209 | 210 | def test_get(fs): 211 | data = files["test/accounts.1.json"] 212 | with fs.open(full_external_mount_name + "/test/accounts.1.json", "rb") as f: 213 | assert f.read() == data 214 | 215 | 216 | def test_read_small(fs): 217 | fn = full_external_mount_name + "/2014-01-01.csv" 218 | with fs.open(fn, "rb", block_size=10) as f: 219 | out = [] 220 | while True: 221 | data = f.read(3) 222 | if data == b"": 223 | break 224 | out.append(data) 225 | assert fs.cat(fn) == b"".join(out) 226 | assert len(f.cache) < len(out) 227 | 228 | 229 | def test_read_oci_block(fs): 230 | data = files["test/accounts.1.json"] 231 | lines = io.BytesIO(data).readlines() 232 | path = full_external_mount_name + "/test/accounts.1.json" 233 | assert fs.read_block(path, 1, 35, b"\n") == lines[1] 234 | assert fs.read_block(path, 0, 30, b"\n") == lines[0] 235 | assert fs.read_block(path, 0, 35, b"\n") == lines[0] + lines[1] 236 | assert fs.read_block(path, 0, 5000, b"\n") == data 237 | assert len(fs.read_block(path, 0, 5)) == 5 238 | assert len(fs.read_block(path, 4, 5000)) == len(data) - 4 239 | assert fs.read_block(path, 5000, 5010) == b"" 240 | assert fs.read_block(path, 5, None) == fs.read_block(path, 5, 1000) 241 | 242 | 243 | def test_write_small(fs): 244 | with fs.open(a, "wb") as f: 245 | f.write(b"hello") 246 | assert fs.cat(a) == b"hello" 247 | fs.open(a, "wb").close() 248 | assert fs.info(a)["size"] == 0 249 | fs.rm(a, recursive=True) 250 | assert not fs.exists(a) 251 | 252 | 253 | def test_write_fails(fs): 254 | with pytest.raises(ValueError): 255 | fs.touch(full_external_mount_name + "/temp") 256 | fs.open(full_external_mount_name + "/temp", "rb").write(b"hello") 257 | with pytest.raises(ValueError): 258 | fs.open(full_external_mount_name + "/temp", "wb", block_size=10) 259 | f = fs.open(full_external_mount_name + "/temp", "wb") 260 | f.close() 261 | with pytest.raises(ValueError): 262 | f.write(b"hello") 263 | with pytest.raises(FileNotFoundError): 264 | fs.open("nonexistentbucket@ns/temp", "wb").close() 265 | fs.rm(full_external_mount_name + "/temp", recursive=True) 266 | assert not fs.exists(full_external_mount_name + "/temp") 267 | 268 | 269 | def test_write_blocks(fs): 270 | with fs.open(full_external_mount_name + "/temp", "wb") as f: 271 | f.write(b"a" * 2 * 2**20) 272 | assert f.buffer.tell() == 2 * 2**20 273 | assert not (f.parts) 274 | f.flush() 275 | assert f.buffer.tell() == 2 * 2**20 276 | assert not (f.parts) 277 | f.write(b"a" * 2 * 2**20) 278 | f.write(b"a" * 2 * 2**20) 279 | assert f.mpu 280 | assert f.parts 281 | assert fs.info(full_external_mount_name + "/temp")["size"] == 6 * 2**20 282 | with fs.open(full_external_mount_name + "/temp", "wb", block_size=10 * 2**20) as f: 283 | f.write(b"a" * 15 * 2**20) 284 | assert f.buffer.tell() == 0 285 | assert fs.info(full_external_mount_name + "/temp")["size"] == 15 * 2**20 286 | fs.rm(full_external_mount_name + "/temp", recursive=True) 287 | assert not fs.exists(full_external_mount_name + "/temp") 288 | 289 | 290 | def test_readline_empty(fs): 291 | data = b"" 292 | with fs.open(a, "wb") as f: 293 | f.write(data) 294 | with fs.open(a, "rb") as f: 295 | result = f.readline() 296 | assert result == data 297 | 298 | 299 | def test_readline_blocksize(fs): 300 | data = b"ab\n" + b"a" * (10 * 2**20) + b"\nab" 301 | with fs.open(a, "wb") as f: 302 | f.write(data) 303 | with fs.open(a, "rb") as f: 304 | result = f.readline() 305 | expected = b"ab\n" 306 | assert result == expected 307 | 308 | result = f.readline() 309 | expected = b"a" * (10 * 2**20) + b"\n" 310 | assert result == expected 311 | 312 | result = f.readline() 313 | expected = b"ab" 314 | assert result == expected 315 | 316 | 317 | def test_next(fs): 318 | expected = csv_files["2014-01-01.csv"].split(b"\n")[0] + b"\n" 319 | with fs.open(full_external_mount_name + "/2014-01-01.csv") as f: 320 | result = next(f) 321 | assert result == expected 322 | 323 | 324 | def test_iterable(fs): 325 | data = b"abc\n123" 326 | with fs.open(a, "wb") as f: 327 | f.write(data) 328 | with fs.open(a) as f, io.BytesIO(data) as g: 329 | for fromoci, fromio in zip(f, g): 330 | assert fromoci == fromio 331 | f.seek(0) 332 | assert f.readline() == b"abc\n" 333 | assert f.readline() == b"123" 334 | f.seek(1) 335 | assert f.readline() == b"bc\n" 336 | 337 | with fs.open(a) as f: 338 | out = list(f) 339 | with fs.open(a) as f: 340 | out2 = f.readlines() 341 | assert out == out2 342 | assert b"".join(out) == data 343 | 344 | 345 | def test_readable(fs): 346 | with fs.open(a, "wb") as f: 347 | assert not f.readable() 348 | 349 | with fs.open(a, "rb") as f: 350 | assert f.readable() 351 | 352 | 353 | def test_seekable(fs): 354 | with fs.open(a, "wb") as f: 355 | assert not f.seekable() 356 | 357 | with fs.open(a, "rb") as f: 358 | assert f.seekable() 359 | 360 | 361 | def test_writable(fs): 362 | with fs.open(a, "wb") as f: 363 | assert f.writable() 364 | 365 | with fs.open(a, "rb") as f: 366 | assert not f.writable() 367 | 368 | 369 | def test_multipart_upload_blocksize(fs): 370 | blocksize = 5 * (2**20) 371 | expected_parts = 3 372 | 373 | fs2 = fs.open(a, "wb", block_size=blocksize) 374 | for _ in range(3): 375 | data = b"b" * blocksize 376 | fs2.write(data) 377 | 378 | # Ensure that the multipart upload consists of only 3 parts 379 | assert len(fs2.parts) == expected_parts 380 | fs2.close() 381 | 382 | 383 | def test_default_pars(): 384 | fs = OCIFileSystem(storage_options["config"], default_block_size=20) 385 | fn = full_external_mount_name + "/" + list(files)[0] 386 | with fs.open(fn) as f: 387 | assert f.blocksize == 20 388 | with fs.open(fn, block_size=40) as f: 389 | assert f.blocksize == 40 390 | 391 | 392 | def test_text_io__stream_wrapper_works(fs): 393 | with fs.open(f"{full_external_mount_name}/file.txt", "wb") as fd: 394 | fd.write("\u00af\\_(\u30c4)_/\u00af".encode("utf-16-le")) 395 | 396 | with fs.open(f"{full_external_mount_name}/file.txt", "rb") as fd: 397 | with io.TextIOWrapper(fd, "utf-16-le") as stream: 398 | assert stream.readline() == "\u00af\\_(\u30c4)_/\u00af" 399 | 400 | fs.rm(f"{full_external_mount_name}/file.txt") 401 | assert not fs.exists(f"{full_external_mount_name}/file.txt") 402 | 403 | 404 | def test_text_io__basic(fs): 405 | """Text mode is now allowed.""" 406 | with fs.open(f"{full_external_mount_name}/file.txt", "w") as fd: 407 | fd.write("\u00af\\_(\u30c4)_/\u00af") 408 | 409 | with fs.open(f"{full_external_mount_name}/file.txt", "r") as fd: 410 | assert fd.read() == "\u00af\\_(\u30c4)_/\u00af" 411 | 412 | fs.rm(f"{full_external_mount_name}/file.txt") 413 | assert not fs.exists(f"{full_external_mount_name}/file.txt") 414 | 415 | 416 | def test_text_io__override_encoding(fs): 417 | """Allow overriding the default text encoding.""" 418 | with fs.open(f"{full_external_mount_name}/file.txt", "w", encoding="ibm500") as fd: 419 | fd.write("Hello, World!") 420 | 421 | with fs.open(f"{full_external_mount_name}/file.txt", "r", encoding="ibm500") as fd: 422 | assert fd.read() == "Hello, World!" 423 | 424 | fs.rm(f"{full_external_mount_name}/file.txt") 425 | assert not fs.exists(f"{full_external_mount_name}/file.txt") 426 | 427 | 428 | def test_readinto(fs): 429 | with fs.open(f"{full_external_mount_name}/file.txt", "wb") as fd: 430 | fd.write(b"Hello, World!") 431 | 432 | contents = bytearray(15) 433 | 434 | with fs.open(f"{full_external_mount_name}/file.txt", "rb") as fd: 435 | assert fd.readinto(contents) == 13 436 | 437 | assert contents.startswith(b"Hello, World!") 438 | 439 | fs.rm(f"{full_external_mount_name}/file.txt") 440 | assert not fs.exists(f"{full_external_mount_name}/file.txt") 441 | 442 | 443 | def test_autocommit(): 444 | auto_file = full_external_mount_name + "/auto_file" 445 | committed_file = full_external_mount_name + "/commit_file" 446 | fs = OCIFileSystem(storage_options["config"], version_aware=True) 447 | 448 | def write_and_flush(path, autocommit): 449 | with fs.open(path, "wb", autocommit=autocommit) as fo: 450 | fo.write(b"1") 451 | return fo 452 | 453 | # regular behavior 454 | fo = write_and_flush(auto_file, autocommit=True) 455 | assert fo.autocommit 456 | assert fs.exists(auto_file) 457 | fs.rm(auto_file) 458 | assert not fs.exists(auto_file) 459 | 460 | fo = write_and_flush(committed_file, autocommit=False) 461 | assert not fo.autocommit 462 | assert not fs.exists(committed_file) 463 | fo.commit() 464 | assert fs.exists(committed_file) 465 | fs.rm(committed_file) 466 | assert not fs.exists(committed_file) 467 | with pytest.raises(Exception): 468 | fo.commit() 469 | 470 | 471 | def test_autocommit_mpu(fs): 472 | """When not autocommitting we always want to use multipart uploads""" 473 | path = full_external_mount_name + "/auto_commit_with_mpu" 474 | with fs.open(path, "wb", autocommit=False) as fo: 475 | fo.write(b"1") 476 | assert fo.mpu is not None 477 | assert len(fo.parts) == 1 478 | 479 | 480 | def test_touch(fs): 481 | with fs.open(a, "wb") as f: 482 | f.write(b"data") 483 | assert fs.size(a) == 4 484 | fs.touch(a, truncate=True) 485 | assert fs.size(a) == 0 486 | 487 | 488 | def test_seek_reads(fs): 489 | fn = full_external_mount_name + "/myfile" 490 | with fs.open(fn, "wb") as f: 491 | f.write(b"a" * 175627146) 492 | with fs.open(fn, "rb", blocksize=100) as f: 493 | f.seek(175561610) 494 | d1 = f.read(65536) 495 | 496 | f.seek(4) 497 | size = 17562198 498 | d2 = f.read(size) 499 | assert len(d2) == size 500 | 501 | f.seek(17562288) 502 | size = 17562187 503 | d3 = f.read(size) 504 | assert len(d3) == size 505 | fs.rm(fn) 506 | assert not fs.exists(fn) 507 | 508 | 509 | def test_rename(fs): 510 | with fs.open(a, "wb") as f: 511 | f.write(b"data") 512 | assert fs.size(a) == 4 513 | fs.rename(a, b) 514 | assert fs.exists(b) 515 | assert fs.size(b) == 4 516 | 517 | 518 | def test_rm(fs): 519 | assert not fs.exists(a) 520 | fs.touch(a) 521 | assert fs.exists(a) 522 | fs.rm(a) 523 | assert not fs.exists(a) 524 | 525 | 526 | def test_bulk_delete(fs): 527 | filelist = fs.find(full_external_mount_name + "/nested") 528 | fs.bulk_delete([]) 529 | fs.bulk_delete(filelist) 530 | assert not fs.exists(full_external_mount_name + "/nested/file1") 531 | -------------------------------------------------------------------------------- /ocifs/utils.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 3 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 4 | 5 | import sys 6 | 7 | 8 | __version__ = "UNKNOWN" 9 | # https://packaging.python.org/en/latest/guides/single-sourcing-package-version/#single-sourcing-the-package-version 10 | if sys.version_info >= (3, 8): 11 | from importlib import metadata 12 | else: 13 | import importlib_metadata as metadata 14 | 15 | __version__ = metadata.version("ocifs") 16 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | ## This file created and used instead of setup.py for building and installing ads package. This change is to 2 | ## follow best practive to "not invoke setup.py directly", see detailed explanation why here: 3 | ## https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html. 4 | ## Check README-development.md and Makefile for instruction how to install or build ADS locally. 5 | 6 | [build-system] 7 | # These are the assumed default build requirements from pip: 8 | # https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support 9 | # PEP 517 – A build-system independent format for source trees - https://peps.python.org/pep-0517/ 10 | requires = ["flit-core >= 3.8"] 11 | build-backend = "flit_core.buildapi" 12 | 13 | 14 | [project] 15 | # Declaring project metadata 16 | # https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ 17 | # PEP 621 – Storing project metadata in pyproject.toml - https://peps.python.org/pep-0621/ 18 | # PEP 518 – Specifying Minimum Build System Requirements for Python Projects https://peps.python.org/pep-0518/ 19 | 20 | # Required 21 | name = "ocifs" # the install (PyPI) name 22 | version = "1.3.2" 23 | 24 | # Optional 25 | description = "Convenient filesystem interface over Oracle Cloud's Object Storage" 26 | readme = {file = "README.md", content-type = "text/markdown"} 27 | requires-python = ">=3.6" 28 | license = {file = "LICENSE.txt"} 29 | authors = [ 30 | {name = "Oracle Cloud Infrastructure Data Science"} 31 | ] 32 | maintainers = [ 33 | {name = "Allen Hosler", email = "allen.hosler@oracle.com"} 34 | ] 35 | keywords = [ 36 | "Oracle Cloud Infrastructure", 37 | "OCI", 38 | "Object Storage", 39 | ] 40 | classifiers = [ 41 | "Development Status :: 5 - Production/Stable", 42 | "Intended Audience :: Developers", 43 | "License :: OSI Approved :: Universal Permissive License (UPL)", 44 | "Operating System :: OS Independent", 45 | "Programming Language :: Python :: 3.6", 46 | "Programming Language :: Python :: 3.7", 47 | "Programming Language :: Python :: 3.8", 48 | "Programming Language :: Python :: 3.9", 49 | "Programming Language :: Python :: 3.10", 50 | "Programming Language :: Python :: 3.11", 51 | "Programming Language :: Python :: 3.12", 52 | ] 53 | 54 | # PEP 508 – Dependency specification for Python Software Packages - https://peps.python.org/pep-0508/ 55 | # In dependencies se "; platform_machine == 'aarch64'" to specify ARM underlying platform 56 | # Copied from install_requires list in setup.py, setup.py got removed in favor of this config file 57 | dependencies = [ 58 | "fsspec>=0.8.7", 59 | "oci>=2.43.1", 60 | "requests", 61 | ] 62 | 63 | [project.urls] 64 | "Github" = "https://github.com/oracle/ocifs" 65 | "Documentation" = "https://ocifs.readthedocs.io/en/latest/index.html" 66 | 67 | # Configuring Ruff (https://docs.astral.sh/ruff/configuration/) 68 | [tool.ruff] 69 | fix = true 70 | 71 | [tool.ruff.lint] 72 | exclude = ["*.yaml", "*jinja2"] 73 | # rules - https://docs.astral.sh/ruff/rules/ 74 | select = [ 75 | # pyflakes 76 | "F", 77 | # subset of the pycodestyle 78 | "E4", "E7", "E9", 79 | # warnings 80 | "W", 81 | # isort 82 | "I" 83 | ] 84 | ignore = [ 85 | # module level import not at top of file 86 | "E402", 87 | # line-length violations 88 | "E501", 89 | ] 90 | 91 | [tool.ruff.lint.per-file-ignores] 92 | "__init__.py" = ["F401"] 93 | -------------------------------------------------------------------------------- /ruff.toml: -------------------------------------------------------------------------------- 1 | [lint.per-file-ignores] 2 | "__init__.py" = ["F401"] 3 | 4 | [lint] 5 | extend-ignore = ["E402", "N806", "N803"] 6 | ignore = [ 7 | "S101", # use of assert 8 | "B008", # function call in argument defaults 9 | "B017", # pytest.raises considered evil 10 | "B023", # function definition in loop (TODO: un-ignore this) 11 | "B028", # explicit stacklevel for warnings 12 | "C901", # function is too complex (TODO: un-ignore this) 13 | "E501", # from scripts/lint_backend.sh 14 | "PLR091", # complexity rules 15 | "PLR2004", # magic numbers 16 | "PLW2901", # `for` loop variable overwritten by assignment target 17 | "SIM105", # contextlib.suppress (has a performance cost) 18 | "SIM117", # multiple nested with blocks (doesn't look good with gr.Row etc) 19 | "UP006", # use `list` instead of `List` for type annotations (fails for 3.8) 20 | "UP007", # use X | Y for type annotations (TODO: can be enabled once Pydantic plays nice with them) 21 | ] 22 | extend-select = [ 23 | "ARG", 24 | "B", 25 | "C", 26 | "E", 27 | "F", 28 | "I", 29 | "N", 30 | "PL", 31 | "S101", 32 | "SIM", 33 | "UP", 34 | "W", 35 | ] 36 | -------------------------------------------------------------------------------- /sbom_generation.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. 2 | 3 | # This OCI DevOps build specification file [1] generates a Software Bill of Materials (SBOM) of the repository. 4 | # The file is needed to run checks for third-party vulnerabilities and business approval according to Oracle’s GitHub policies. 5 | # [1] https://docs.oracle.com/en-us/iaas/Content/devops/using/build_specs.htm 6 | 7 | version: 0.1 8 | component: build 9 | timeoutInSeconds: 1000 10 | shell: bash 11 | env: 12 | variables: 13 | PYTHON_CMD: "python3" 14 | CDXGEN_DEBUG_MODE: "debug" 15 | steps: 16 | - type: Command 17 | name: "Download the version 10.10.0 of cdxgen globally" 18 | command: | 19 | npm install -g @cyclonedx/cdxgen@10.10.0 20 | - type: Command 21 | name: "Workaround to let cdxgen run on nodejs 16" 22 | command: | 23 | # cdxgen relies on a fourth-party dependency that cannot be executed in a Node.js environment running version 16 24 | # (as installed on the build runner instance) 25 | # This is a workaround to ensure cdxgen functions correctly, even in an older Node.js environment. 26 | cd /node/node-v16.14.2-linux-x64/lib/node_modules/@cyclonedx/cdxgen && \ 27 | npm install cheerio@v1.0.0-rc.12 28 | - type: Command 29 | name: "Generate SBOM for Python " 30 | command: | 31 | # Search the test or dev requirements files, so that test and dev py packages can be excluded in the generated SBOM 32 | files=$(find . -type f -regex ".*\(test.*requirements\|requirements.*test\|dev.*requirements\|requirements.*dev\).*\.txt") && \ 33 | if [ -n "$files" ]; then \ 34 | cdxgen -t python -o artifactSBOM.json --spec-version 1.4 \ 35 | --exclude "*{requirements,dev,test}*{requirements,dev,test}*.txt" --project-name "$(basename $OCI_PRIMARY_SOURCE_URL)" --no-recurse 36 | else \ 37 | cdxgen -t python -o artifactSBOM.json --spec-version 1.4 --project-name "$(basename $OCI_PRIMARY_SOURCE_URL)" --no-recurse 38 | fi \ 39 | outputArtifacts: 40 | - name: artifactSBOM 41 | type: BINARY 42 | location: ${OCI_PRIMARY_SOURCE_DIR}/artifactSBOM.json -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = __init__.py 3 | max-line-length = 95 4 | ignore = 5 | E20, # Extra space in brackets 6 | E231,E241, # Multiple spaces around "," 7 | E26, # Comments 8 | E4, # Import formatting 9 | E721, # Comparing types instead of isinstance 10 | E731, # Assigning lambda expression 11 | E121, # continuation line under-indented for hanging indent 12 | E126, # continuation line over-indented for hanging indent 13 | E127, # continuation line over-indented for visual indent 14 | E128, # E128 continuation line under-indented for visual indent 15 | E702, # multiple statements on one line (semicolon) 16 | W503, # line break before binary operator 17 | E129, # visually indented line with same indent as next logical line 18 | E116, # unexpected indentation 19 | F811, # redefinition of unused 'loop' from line 10 20 | F841, # local variable is assigned to but never used 21 | E741 # Ambiguous variable names 22 | W504, # line break after binary operator 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, 2023 Oracle and/or its affiliates. 2 | # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ 3 | 4 | ### File setup.py obsolete and must not be used. Please update pyproject.toml instead. 5 | ### See detailed explanation why here: 6 | ### https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html. 7 | # PEP 621 – Storing project metadata in pyproject.toml - https://peps.python.org/pep-0621/ 8 | # PEP 518 – Specifying Minimum Build System Requirements for Python Projects https://peps.python.org/pep-0518/ 9 | # PEP 508 – Dependency specification for Python Software Packages - https://peps.python.org/pep-0508/ 10 | # PEP 517 – A build-system independent format for source trees - https://peps.python.org/pep-0517/ 11 | --------------------------------------------------------------------------------