├── .circleci └── config.yml ├── .editorconfig ├── .gitignore ├── .gitmodules ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── contributing.md ├── docs ├── Makefile ├── build │ ├── doctrees │ │ ├── environment.pickle │ │ ├── getting_started.doctree │ │ ├── index.doctree │ │ ├── modules.doctree │ │ ├── python_pachyderm.doctree │ │ ├── python_pachyderm.experimental.doctree │ │ ├── python_pachyderm.experimental.mixin.doctree │ │ ├── python_pachyderm.experimental.proto.doctree │ │ ├── python_pachyderm.experimental.proto.v2.admin_v2.doctree │ │ ├── python_pachyderm.experimental.proto.v2.auth_v2.doctree │ │ ├── python_pachyderm.experimental.proto.v2.debug_v2.doctree │ │ ├── python_pachyderm.experimental.proto.v2.doctree │ │ ├── python_pachyderm.experimental.proto.v2.enterprise_v2.doctree │ │ ├── python_pachyderm.experimental.proto.v2.identity_v2.doctree │ │ ├── python_pachyderm.experimental.proto.v2.license_v2.doctree │ │ ├── python_pachyderm.experimental.proto.v2.pfs_v2.doctree │ │ ├── python_pachyderm.experimental.proto.v2.pps_v2.doctree │ │ ├── python_pachyderm.experimental.proto.v2.proxy.doctree │ │ ├── python_pachyderm.experimental.proto.v2.transaction_v2.doctree │ │ ├── python_pachyderm.experimental.proto.v2.versionpb_v2.doctree │ │ └── python_pachyderm.mixin.doctree │ └── html │ │ ├── .buildinfo │ │ ├── _modules │ │ ├── betterproto.html │ │ ├── index.html │ │ └── python_pachyderm │ │ │ ├── client.html │ │ │ ├── experimental │ │ │ ├── client.html │ │ │ ├── mixin │ │ │ │ ├── admin.html │ │ │ │ ├── auth.html │ │ │ │ ├── debug.html │ │ │ │ ├── enterprise.html │ │ │ │ ├── health.html │ │ │ │ ├── identity.html │ │ │ │ ├── license.html │ │ │ │ ├── pfs.html │ │ │ │ ├── pps.html │ │ │ │ ├── transaction.html │ │ │ │ └── version.html │ │ │ ├── pfs.html │ │ │ ├── proto │ │ │ │ └── v2 │ │ │ │ │ ├── admin_v2.html │ │ │ │ │ ├── auth_v2.html │ │ │ │ │ ├── debug_v2.html │ │ │ │ │ ├── enterprise_v2.html │ │ │ │ │ ├── identity_v2.html │ │ │ │ │ ├── license_v2.html │ │ │ │ │ ├── pfs_v2.html │ │ │ │ │ ├── pps_v2.html │ │ │ │ │ ├── proxy.html │ │ │ │ │ ├── transaction_v2.html │ │ │ │ │ └── versionpb_v2.html │ │ │ └── util.html │ │ │ ├── mixin │ │ │ ├── admin.html │ │ │ ├── auth.html │ │ │ ├── debug.html │ │ │ ├── enterprise.html │ │ │ ├── health.html │ │ │ ├── identity.html │ │ │ ├── license.html │ │ │ ├── pfs.html │ │ │ ├── pps.html │ │ │ ├── transaction.html │ │ │ ├── util.html │ │ │ └── version.html │ │ │ ├── pfs.html │ │ │ ├── spout.html │ │ │ └── util.html │ │ ├── _sources │ │ ├── getting_started.md.txt │ │ ├── index.rst.txt │ │ ├── modules.rst.txt │ │ ├── python_pachyderm.experimental.mixin.rst.txt │ │ ├── python_pachyderm.experimental.proto.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.admin_v2.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.auth_v2.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.debug_v2.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.enterprise_v2.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.identity_v2.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.license_v2.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.pfs_v2.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.pps_v2.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.proxy.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.transaction_v2.rst.txt │ │ ├── python_pachyderm.experimental.proto.v2.versionpb_v2.rst.txt │ │ ├── python_pachyderm.experimental.rst.txt │ │ ├── python_pachyderm.mixin.rst.txt │ │ └── python_pachyderm.rst.txt │ │ ├── _static │ │ ├── alabaster.css │ │ ├── basic.css │ │ ├── css │ │ │ ├── badge_only.css │ │ │ ├── fonts │ │ │ │ ├── Roboto-Slab-Bold.woff │ │ │ │ ├── Roboto-Slab-Bold.woff2 │ │ │ │ ├── Roboto-Slab-Regular.woff │ │ │ │ ├── Roboto-Slab-Regular.woff2 │ │ │ │ ├── fontawesome-webfont.eot │ │ │ │ ├── fontawesome-webfont.svg │ │ │ │ ├── fontawesome-webfont.ttf │ │ │ │ ├── fontawesome-webfont.woff │ │ │ │ ├── fontawesome-webfont.woff2 │ │ │ │ ├── lato-bold-italic.woff │ │ │ │ ├── lato-bold-italic.woff2 │ │ │ │ ├── lato-bold.woff │ │ │ │ ├── lato-bold.woff2 │ │ │ │ ├── lato-normal-italic.woff │ │ │ │ ├── lato-normal-italic.woff2 │ │ │ │ ├── lato-normal.woff │ │ │ │ └── lato-normal.woff2 │ │ │ └── theme.css │ │ ├── custom.css │ │ ├── doctools.js │ │ ├── documentation_options.js │ │ ├── file.png │ │ ├── fonts │ │ │ ├── Inconsolata-Bold.ttf │ │ │ ├── Inconsolata-Regular.ttf │ │ │ ├── Inconsolata.ttf │ │ │ ├── Lato-Bold.ttf │ │ │ ├── Lato-Regular.ttf │ │ │ ├── Lato │ │ │ │ ├── lato-bold.eot │ │ │ │ ├── lato-bold.ttf │ │ │ │ ├── lato-bold.woff │ │ │ │ ├── lato-bold.woff2 │ │ │ │ ├── lato-bolditalic.eot │ │ │ │ ├── lato-bolditalic.ttf │ │ │ │ ├── lato-bolditalic.woff │ │ │ │ ├── lato-bolditalic.woff2 │ │ │ │ ├── lato-italic.eot │ │ │ │ ├── lato-italic.ttf │ │ │ │ ├── lato-italic.woff │ │ │ │ ├── lato-italic.woff2 │ │ │ │ ├── lato-regular.eot │ │ │ │ ├── lato-regular.ttf │ │ │ │ ├── lato-regular.woff │ │ │ │ └── lato-regular.woff2 │ │ │ ├── RobotoSlab-Bold.ttf │ │ │ ├── RobotoSlab-Regular.ttf │ │ │ ├── RobotoSlab │ │ │ │ ├── roboto-slab-v7-bold.eot │ │ │ │ ├── roboto-slab-v7-bold.ttf │ │ │ │ ├── roboto-slab-v7-bold.woff │ │ │ │ ├── roboto-slab-v7-bold.woff2 │ │ │ │ ├── roboto-slab-v7-regular.eot │ │ │ │ ├── roboto-slab-v7-regular.ttf │ │ │ │ ├── roboto-slab-v7-regular.woff │ │ │ │ └── roboto-slab-v7-regular.woff2 │ │ │ ├── fontawesome-webfont.eot │ │ │ ├── fontawesome-webfont.svg │ │ │ ├── fontawesome-webfont.ttf │ │ │ ├── fontawesome-webfont.woff │ │ │ └── fontawesome-webfont.woff2 │ │ ├── jquery-3.5.1.js │ │ ├── jquery.js │ │ ├── js │ │ │ ├── badge_only.js │ │ │ ├── html5shiv-printshiv.min.js │ │ │ ├── html5shiv.min.js │ │ │ ├── modernizr.min.js │ │ │ └── theme.js │ │ ├── language_data.js │ │ ├── minus.png │ │ ├── plus.png │ │ ├── pygments.css │ │ ├── searchtools.js │ │ ├── underscore-1.13.1.js │ │ └── underscore.js │ │ ├── genindex.html │ │ ├── getting_started.html │ │ ├── index.html │ │ ├── modules.html │ │ ├── objects.inv │ │ ├── py-modindex.html │ │ ├── python_pachyderm.experimental.html │ │ ├── python_pachyderm.experimental.mixin.html │ │ ├── python_pachyderm.experimental.proto.html │ │ ├── python_pachyderm.experimental.proto.v2.admin_v2.html │ │ ├── python_pachyderm.experimental.proto.v2.auth_v2.html │ │ ├── python_pachyderm.experimental.proto.v2.debug_v2.html │ │ ├── python_pachyderm.experimental.proto.v2.enterprise_v2.html │ │ ├── python_pachyderm.experimental.proto.v2.html │ │ ├── python_pachyderm.experimental.proto.v2.identity_v2.html │ │ ├── python_pachyderm.experimental.proto.v2.license_v2.html │ │ ├── python_pachyderm.experimental.proto.v2.pfs_v2.html │ │ ├── python_pachyderm.experimental.proto.v2.pps_v2.html │ │ ├── python_pachyderm.experimental.proto.v2.proxy.html │ │ ├── python_pachyderm.experimental.proto.v2.transaction_v2.html │ │ ├── python_pachyderm.experimental.proto.v2.versionpb_v2.html │ │ ├── python_pachyderm.html │ │ ├── python_pachyderm.mixin.html │ │ ├── search.html │ │ └── searchindex.js ├── make.bat ├── requirements.txt └── source │ ├── conf.py │ ├── getting_started.md │ ├── index.rst │ ├── modules.rst │ ├── python_pachyderm.experimental.mixin.rst │ ├── python_pachyderm.experimental.proto.rst │ ├── python_pachyderm.experimental.proto.v2.admin_v2.rst │ ├── python_pachyderm.experimental.proto.v2.auth_v2.rst │ ├── python_pachyderm.experimental.proto.v2.debug_v2.rst │ ├── python_pachyderm.experimental.proto.v2.enterprise_v2.rst │ ├── python_pachyderm.experimental.proto.v2.identity_v2.rst │ ├── python_pachyderm.experimental.proto.v2.license_v2.rst │ ├── python_pachyderm.experimental.proto.v2.pfs_v2.rst │ ├── python_pachyderm.experimental.proto.v2.pps_v2.rst │ ├── python_pachyderm.experimental.proto.v2.proxy.rst │ ├── python_pachyderm.experimental.proto.v2.rst │ ├── python_pachyderm.experimental.proto.v2.transaction_v2.rst │ ├── python_pachyderm.experimental.proto.v2.versionpb_v2.rst │ ├── python_pachyderm.experimental.rst │ ├── python_pachyderm.mixin.rst │ └── python_pachyderm.rst ├── etc ├── kube │ ├── check_ready.sh │ └── start-minikube.sh ├── proto_lint │ └── proto_lint.py └── testing │ └── circle │ ├── install.sh │ └── kube_debug.sh ├── examples ├── img │ └── pach_logo.svg ├── jupyter │ ├── Dockerfile │ ├── README.md │ ├── final_graph.png │ ├── investigate-unexpected-sales.ipynb │ ├── jupyter.json │ ├── merge.py │ ├── sales.png │ ├── trips_data │ │ ├── 7-01-16 │ │ ├── 7-02-16 │ │ ├── 7-03-16 │ │ ├── 7-04-16 │ │ ├── 7-05-16 │ │ ├── 7-06-16 │ │ ├── 7-07-16 │ │ ├── 7-08-16 │ │ ├── 7-09-16 │ │ ├── 7-10-16 │ │ ├── 7-11-16 │ │ ├── 7-12-16 │ │ ├── 7-13-16 │ │ ├── 7-14-16 │ │ ├── 7-15-16 │ │ ├── 7-16-16 │ │ ├── 7-17-16 │ │ ├── 7-18-16 │ │ ├── 7-19-16 │ │ ├── 7-20-16 │ │ ├── 7-21-16 │ │ ├── 7-22-16 │ │ ├── 7-23-16 │ │ ├── 7-24-16 │ │ ├── 7-25-16 │ │ ├── 7-26-16 │ │ ├── 7-27-16 │ │ ├── 7-28-16 │ │ ├── 7-29-16 │ │ ├── 7-30-16 │ │ ├── 7-31-16 │ │ ├── 8-01-16 │ │ ├── 8-02-16 │ │ ├── 8-03-16 │ │ ├── 8-04-16 │ │ ├── 8-05-16 │ │ ├── 8-06-16 │ │ ├── 8-07-16 │ │ ├── 8-08-16 │ │ ├── 8-09-16 │ │ ├── 8-10-16 │ │ ├── 8-11-16 │ │ ├── 8-12-16 │ │ ├── 8-13-16 │ │ ├── 8-14-16 │ │ ├── 8-15-16 │ │ ├── 8-16-16 │ │ ├── 8-17-16 │ │ ├── 8-18-16 │ │ ├── 8-19-16 │ │ ├── 8-20-16 │ │ ├── 8-21-16 │ │ ├── 8-22-16 │ │ ├── 8-23-16 │ │ ├── 8-24-16 │ │ ├── 8-25-16 │ │ ├── 8-26-16 │ │ ├── 8-27-16 │ │ ├── 8-28-16 │ │ ├── 8-29-16 │ │ ├── 8-30-16 │ │ ├── 8-31-16 │ │ ├── 9-01-16 │ │ ├── 9-02-16 │ │ ├── 9-03-16 │ │ ├── 9-04-16 │ │ ├── 9-05-16 │ │ ├── 9-06-16 │ │ ├── 9-07-16 │ │ ├── 9-08-16 │ │ ├── 9-09-16 │ │ ├── 9-10-16 │ │ ├── 9-11-16 │ │ ├── 9-12-16 │ │ ├── 9-13-16 │ │ ├── 9-14-16 │ │ ├── 9-15-16 │ │ ├── 9-16-16 │ │ ├── 9-17-16 │ │ ├── 9-18-16 │ │ ├── 9-19-16 │ │ ├── 9-20-16 │ │ ├── 9-21-16 │ │ ├── 9-22-16 │ │ ├── 9-23-16 │ │ ├── 9-24-16 │ │ ├── 9-25-16 │ │ ├── 9-26-16 │ │ ├── 9-27-16 │ │ ├── 9-28-16 │ │ ├── 9-29-16 │ │ └── 9-30-16 │ └── weather_data │ │ ├── 2016-07-01 │ │ ├── 2016-07-02 │ │ ├── 2016-07-03 │ │ ├── 2016-07-04 │ │ ├── 2016-07-05 │ │ ├── 2016-07-06 │ │ ├── 2016-07-07 │ │ ├── 2016-07-08 │ │ ├── 2016-07-09 │ │ ├── 2016-07-10 │ │ ├── 2016-07-11 │ │ ├── 2016-07-12 │ │ ├── 2016-07-13 │ │ ├── 2016-07-14 │ │ ├── 2016-07-15 │ │ ├── 2016-07-16 │ │ ├── 2016-07-17 │ │ ├── 2016-07-18 │ │ ├── 2016-07-19 │ │ ├── 2016-07-20 │ │ ├── 2016-07-21 │ │ ├── 2016-07-22 │ │ ├── 2016-07-23 │ │ ├── 2016-07-24 │ │ ├── 2016-07-25 │ │ ├── 2016-07-26 │ │ ├── 2016-07-27 │ │ ├── 2016-07-28 │ │ ├── 2016-07-29 │ │ ├── 2016-07-30 │ │ ├── 2016-07-31 │ │ ├── 2016-08-01 │ │ ├── 2016-08-02 │ │ ├── 2016-08-03 │ │ ├── 2016-08-04 │ │ ├── 2016-08-05 │ │ ├── 2016-08-06 │ │ ├── 2016-08-07 │ │ ├── 2016-08-08 │ │ ├── 2016-08-09 │ │ ├── 2016-08-10 │ │ ├── 2016-08-11 │ │ ├── 2016-08-12 │ │ ├── 2016-08-13 │ │ ├── 2016-08-14 │ │ ├── 2016-08-15 │ │ ├── 2016-08-16 │ │ ├── 2016-08-17 │ │ ├── 2016-08-18 │ │ ├── 2016-08-19 │ │ ├── 2016-08-20 │ │ ├── 2016-08-21 │ │ ├── 2016-08-22 │ │ ├── 2016-08-23 │ │ ├── 2016-08-24 │ │ ├── 2016-08-25 │ │ ├── 2016-08-26 │ │ ├── 2016-08-27 │ │ ├── 2016-08-28 │ │ ├── 2016-08-29 │ │ ├── 2016-08-30 │ │ ├── 2016-08-31 │ │ ├── 2016-09-01 │ │ ├── 2016-09-02 │ │ ├── 2016-09-03 │ │ ├── 2016-09-04 │ │ ├── 2016-09-05 │ │ ├── 2016-09-06 │ │ ├── 2016-09-07 │ │ ├── 2016-09-08 │ │ ├── 2016-09-09 │ │ ├── 2016-09-10 │ │ ├── 2016-09-11 │ │ ├── 2016-09-12 │ │ ├── 2016-09-13 │ │ ├── 2016-09-14 │ │ ├── 2016-09-15 │ │ ├── 2016-09-16 │ │ ├── 2016-09-17 │ │ ├── 2016-09-18 │ │ ├── 2016-09-19 │ │ ├── 2016-09-20 │ │ ├── 2016-09-21 │ │ ├── 2016-09-22 │ │ ├── 2016-09-23 │ │ ├── 2016-09-24 │ │ ├── 2016-09-25 │ │ ├── 2016-09-26 │ │ ├── 2016-09-27 │ │ ├── 2016-09-28 │ │ ├── 2016-09-29 │ │ └── 2016-09-30 ├── opencv │ ├── README.md │ ├── edges │ │ ├── main.py │ │ └── requirements.txt │ ├── images │ │ ├── 46Q8nDz.jpg │ │ ├── 8MN9Kg0.jpg │ │ └── g2QnNqa.jpg │ └── opencv.py └── spout │ ├── README.md │ └── spout.py ├── poetry.lock ├── poetry.toml ├── proto ├── Dockerfile └── run ├── pyproject.toml ├── setup.cfg ├── src └── python_pachyderm │ ├── __init__.py │ ├── client.py │ ├── datum_batching.py │ ├── errors.py │ ├── interceptor.py │ ├── mixin │ ├── __init__.py │ ├── admin.py │ ├── auth.py │ ├── debug.py │ ├── enterprise.py │ ├── health.py │ ├── identity.py │ ├── license.py │ ├── pfs.py │ ├── pps.py │ ├── transaction.py │ ├── version.py │ └── worker.py │ ├── pfs.py │ ├── proto │ └── v2 │ │ ├── __init__.py │ │ ├── admin │ │ ├── __init__.py │ │ ├── admin_pb2.py │ │ └── admin_pb2_grpc.py │ │ ├── auth │ │ ├── __init__.py │ │ ├── auth_pb2.py │ │ └── auth_pb2_grpc.py │ │ ├── debug │ │ ├── __init__.py │ │ ├── debug_pb2.py │ │ └── debug_pb2_grpc.py │ │ ├── enterprise │ │ ├── __init__.py │ │ ├── enterprise_pb2.py │ │ └── enterprise_pb2_grpc.py │ │ ├── identity │ │ ├── __init__.py │ │ ├── identity_pb2.py │ │ └── identity_pb2_grpc.py │ │ ├── license │ │ ├── __init__.py │ │ ├── license_pb2.py │ │ └── license_pb2_grpc.py │ │ ├── pfs │ │ ├── __init__.py │ │ ├── pfs_pb2.py │ │ └── pfs_pb2_grpc.py │ │ ├── pps │ │ ├── __init__.py │ │ ├── pps_pb2.py │ │ └── pps_pb2_grpc.py │ │ ├── proxy │ │ ├── __init__.py │ │ ├── proxy_pb2.py │ │ └── proxy_pb2_grpc.py │ │ ├── task │ │ ├── __init__.py │ │ ├── task_pb2.py │ │ └── task_pb2_grpc.py │ │ ├── transaction │ │ ├── __init__.py │ │ ├── transaction_pb2.py │ │ └── transaction_pb2_grpc.py │ │ ├── version │ │ ├── __init__.py │ │ └── versionpb │ │ │ ├── __init__.py │ │ │ ├── version_pb2.py │ │ │ └── version_pb2_grpc.py │ │ └── worker │ │ ├── __init__.py │ │ ├── worker_pb2.py │ │ └── worker_pb2_grpc.py │ ├── service.py │ └── util.py ├── tests ├── __init__.py ├── health.py ├── test_admin.py ├── test_auth.py ├── test_client.py ├── test_debug.py ├── test_enterprise.py ├── test_identity.py ├── test_pfs.py ├── test_pps.py ├── test_spout.py ├── test_transaction.py ├── test_util.py ├── test_version.py ├── test_worker.py └── util.py ├── tox.ini └── version.json /.editorconfig: -------------------------------------------------------------------------------- 1 | # see http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | end_of_line = lf 6 | trim_trailing_whitespace = true 7 | insert_final_newline = true 8 | indent_style = space 9 | indent_size = 4 10 | charset = utf-8 11 | 12 | [*.{bat,cmd,ps1}] 13 | end_of_line = crlf 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # general things to ignore 2 | build/ 3 | !docs/build/ 4 | dist/ 5 | *.egg-info/ 6 | *.egg 7 | .eggs/ 8 | *.py[cod] 9 | __pycache__/ 10 | *.so 11 | *~ 12 | .DS_Store 13 | 14 | # ignore vim temporary files 15 | .*.sw* 16 | 17 | # due to using tox and pytest 18 | .tox 19 | .cache 20 | 21 | # due to jupyter 22 | .ipynb_checkpoints/ 23 | 24 | # PyCharm 25 | .idea 26 | 27 | # VSCode 28 | .vscode 29 | 30 | # python-pachyderm 31 | src/python_pachyderm/version.py 32 | 33 | # virtualenv 34 | .venv 35 | .python-version 36 | venv 37 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "proto/pachyderm"] 2 | path = proto/pachyderm 3 | url = https://github.com/pachyderm/pachyderm.git 4 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 22.6.0 4 | hooks: 5 | - id: black 6 | language_version: python3 7 | - repo: https://gitlab.com/pycqa/flake8 8 | rev: 3.9.1 9 | hooks: 10 | - id: flake8 11 | language_version: python3 12 | - repo: local 13 | hooks: 14 | - id: proto_lint 15 | name: proto_lint 16 | language: system 17 | entry: etc/proto_lint/proto_lint.py 18 | pass_filenames: false 19 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build documentation in the docs/ directory with Sphinx 9 | sphinx: 10 | configuration: docs/source/conf.py 11 | 12 | # Optionally set the version of Python and requirements required to build your docs 13 | python: 14 | install: 15 | - requirements: docs/requirements.txt 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim 2 | 3 | ENV PYTHONUNBUFFERED=1 4 | 5 | COPY dist dist 6 | RUN python3 -m pip install `find dist/ -name \*.whl` -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | PACHYDERM_VERSION ?= $(shell jq -r .pachyderm version.json) 3 | 4 | docs: 5 | $(MAKE) -C docs html 6 | 7 | docker-build-proto: 8 | docker build -t pachyderm_python_proto proto 9 | 10 | src/python_pachyderm/proto/v2: docker-build-proto 11 | @echo "Building with pachyderm core v$(PACHYDERM_VERSION)" 12 | rm -rf src/python_pachyderm/proto/v2 && \ 13 | rm -rf src/python_pachyderm/experimental/proto/v2 14 | cd proto/pachyderm && \ 15 | git fetch --all && \ 16 | git checkout v$(PACHYDERM_VERSION) 17 | find ./proto/pachyderm/src -regex ".*\.proto" \ 18 | | grep -v 'internal' \ 19 | | grep -v 'server' \ 20 | | grep -v 'protoextensions' \ 21 | | xargs tar cf - \ 22 | | docker run -e VERSION=2 -i pachyderm_python_proto \ 23 | | tar -C src -xf - 24 | 25 | init: 26 | git submodule update --init 27 | pre-commit install 28 | 29 | release: 30 | git checkout origin/master 31 | rm -rf build dist 32 | poetry publish --build 33 | 34 | test-release: 35 | git checkout origin/master 36 | rm -rf build dist 37 | poetry publish --build --repository testpypi 38 | 39 | lint: 40 | black --check --diff . 41 | flake8 . 42 | PYTHONPATH=./src:$(PYTHONPATH) etc/proto_lint/proto_lint.py 43 | 44 | .PHONY: docs docker-build-proto init release test-release lint 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Python Pachyderm 2 | 3 | [](https://pypi.python.org/pypi/python-pachyderm) 4 | [](https://pypi.python.org/pypi/python-pachyderm) 5 | [](http://slack.pachyderm.io) 6 | 7 | ## **Deprecated** 8 | 9 | **WARNING! This package is no longer maintained nor supported. Please transition to the currently maintained Python SDK for Pachyderm: [pachyderm-sdk](https://github.com/pachyderm/pachyderm/tree/master/python-sdk)** 10 | 11 | --- 12 | 13 | Official Python Pachyderm client. 14 | 15 | This library provides the autogenerated gRPC/protobuf code for Pachyderm, along with a higher-level and more pythonic `Client` class. 16 | See the [API docs](https://python-pachyderm.readthedocs.io/en/stable/). 17 | 18 | ## Installation 19 | 20 | ```bash 21 | pip install python-pachyderm 22 | ``` 23 | 24 | ## A Small Taste 25 | 26 | Here's an example that creates a repo and adds a file: 27 | 28 | ```python 29 | import python_pachyderm 30 | 31 | # Connects to a pachyderm cluster on localhost:30650. 32 | # For other options, see the API docs. 33 | client = python_pachyderm.Client() 34 | 35 | # Create a pachyderm repo called `test` 36 | client.create_repo("test") 37 | 38 | # Create a file in `(repo="test", branch="master")` at `/dir_a/data.txt` 39 | # Similar to `pachctl put file test@master:/dir_a/data.txt` 40 | with client.commit("test", "master") as commit: 41 | client.put_file_bytes(commit, "/dir_a/data.txt", b"DATA") 42 | 43 | # Get back the file 44 | f = client.get_file(("test", "master"), "/dir_a/data.txt") 45 | print(f.read()) # >>> b"DATA" 46 | ``` 47 | 48 | How to load a CSV file into a Pandas dataframe 49 | 50 | ```python 51 | import pandas as pd 52 | 53 | f = client.get_file(("my_repo", "my_branch"), "/path_to/my_data.csv") 54 | df = pd.read_csv(f) 55 | ``` 56 | 57 | For more sophisticated examples, [see the examples directory](https://github.com/pachyderm/python-pachyderm/tree/master/examples). 58 | 59 | ## Versioning 60 | 61 | Prior to python-pachyderm 2.0, this library's versioning synced with pachyderm's core versioning; e.g. version 1.8.5 of this library synced with 1.8.5 of pachyderm core. python-pachyderm 2.0 onwards uses semver instead, so versions are not tied to pachyderm core. This was done for two reasons: 62 | 63 | 1. Sometimes this library makes breaking or backwards-incompatible changes, which aren't properly conveyed by revision changes. 64 | 2. Pachyderm core is stable enough that most features of this library will work for disparate versions of pachyderm clusters. To help ensure this, this library's CI tests against several versions of pachyderm core. 65 | 66 | However, if for whatever reason you need to know which version of pachyderm core a version of python-pachyderm was built with, consult `CHANGELOG.md`. As a broad rule of thumb, we recommend working with the latest version of both pachyderm core and python-pachyderm where possible. 67 | 68 | ## Contributing 69 | 70 | This driver is co-maintained by Pachyderm and the community. If you're looking to contribute to the project, this is a fantastic place to get involved. Take a look at [the contributing guide](./contributing.md) for more info (including testing instructions). 71 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/build/doctrees/environment.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/environment.pickle -------------------------------------------------------------------------------- /docs/build/doctrees/getting_started.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/getting_started.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/index.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/index.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/modules.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/modules.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.mixin.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.mixin.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.admin_v2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.admin_v2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.auth_v2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.auth_v2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.debug_v2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.debug_v2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.enterprise_v2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.enterprise_v2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.identity_v2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.identity_v2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.license_v2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.license_v2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.pfs_v2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.pfs_v2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.pps_v2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.pps_v2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.proxy.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.proxy.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.transaction_v2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.transaction_v2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.experimental.proto.v2.versionpb_v2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.experimental.proto.v2.versionpb_v2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/python_pachyderm.mixin.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/doctrees/python_pachyderm.mixin.doctree -------------------------------------------------------------------------------- /docs/build/html/.buildinfo: -------------------------------------------------------------------------------- 1 | # Sphinx build info version 1 2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. 3 | config: 7dba78a1c47e9c8477439178fca38ed3 4 | tags: 645f666f9bcd5a90fca523b33c5a78b7 5 | -------------------------------------------------------------------------------- /docs/build/html/_sources/getting_started.md.txt: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | ## Prerequisites 4 | * A workspace on [Pachyderm Hub](https://docs.pachyderm.com/latest/hub/hub_getting_started/) (recommended) or Pachyderm running [locally](https://docs.pachyderm.com/latest/getting_started/local_installation/) 5 | * Python 3.6 or higher 6 | 7 | ## Installation 8 | * via PyPI: `pip install python-pachyderm` 9 | * via source 10 | * download desired version [here](https://github.com/pachyderm/python-pachyderm/releases) (under assets) 11 | * `make init` in folder root-level 12 | 13 | ## Hello World example 14 | 15 | Creates a repo, commits data to the repo (versioning the data), and reads the data back from the repo. 16 | 17 | ```python 18 | import python_pachyderm 19 | 20 | # Connects to a pachyderm cluster on localhost:30650 21 | # For other connection options, see the API docs 22 | client = python_pachyderm.Client() 23 | 24 | # Create a pachyderm repo called `test` 25 | client.create_repo("test") 26 | 27 | # Create a file in (repo="test", branch="master") at `/dir_a/data.txt` 28 | # Similar to `pachctl put file test@master:/dir_a/data.txt` 29 | with client.commit("test", "master") as commit: 30 | client.put_file_bytes(commit, "/dir_a/data.txt", b"hello world") 31 | 32 | # Get the file 33 | f = client.get_file(("test", "master"), "/dir_a/data.txt") 34 | print(f.read()) # >>> b"hello world" 35 | ``` 36 | 37 | Since `client.get_file()` returns a file-like object, you can pass it into your favorite analysis packages. 38 | 39 | ```python 40 | import pandas as pd 41 | 42 | f = client.get_file(("my_repo", "my_branch"), "/path_to/my_data.csv") 43 | df = pd.read_csv(f) 44 | ``` 45 | 46 | ## Hello World example with Pachyderm Pipelines 47 | 48 | Creates a data-driven pipeline that transforms the data and outputs results to a new repo. This example pipeline counts the occurrences of the word `hello` in the repo. Continued from the example above... 49 | 50 | ```python 51 | from python_pachyderm.service import pps_proto 52 | 53 | # Create a pipeline that logs frequency of the word "hello" in `test` 54 | # repo to a file in the `word_count` repo (which is created automatically) 55 | # Any time data is committed to the `test` repo, this pipeline will 56 | # automatically trigger. 57 | client.create_pipeline( 58 | "word_count", 59 | transform=pps_proto.Transform( 60 | cmd=["bash"], 61 | stdin=[ 62 | "grep -roh hello /pfs/test/ | wc -w > /pfs/out/count.txt" 63 | ] 64 | ), 65 | input=pps_proto.Input( 66 | pfs=pps_proto.PFSInput(repo="test", branch="master", glob="/") 67 | ) 68 | ) 69 | 70 | # Wait for new commit, triggered by pipeline run, to finish 71 | client.wait_commit(("word_count", "master")) 72 | 73 | # Check `count.txt` for "hello" count 74 | f = client.get_file(("word_count", "master"), "count.txt") 75 | print(f.read()) # >>> b"1" 76 | 77 | # Add more data to the `test` repo 78 | with client.commit("test", "master") as commit: 79 | client.put_file_bytes(commit, "/data2.txt", b"hello hello from the top of the world") 80 | 81 | # Wait for commit to finish 82 | client.wait_commit(commit.id) 83 | 84 | # Check `count.txt` for "hello" count 85 | f = client.get_file(("word_count", "master"), "count.txt") 86 | print(f.read()) # >>> b"3" 87 | ``` 88 | 89 | For more sophisticated examples, [see the examples directory](https://github.com/pachyderm/python-pachyderm/tree/master/examples). 90 | To learn more about what you can do with Pachyderm, check out the [docs](https://docs.pachyderm.com/latest/how-tos/). -------------------------------------------------------------------------------- /docs/build/html/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | python-pachyderm 2 | ============================================ 3 | 4 | Overview 5 | -------- 6 | 7 | python-pachyderm is a Python client that interacts with `Pachyderm `_, 8 | a tool for version-controlled, automated, end-to-end data pipelines for data science. If you're not 9 | familiar with Pachyderm or its value, check out that first! 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | :caption: Docs 14 | 15 | getting_started.md 16 | modules.rst 17 | 18 | Links 19 | ----- 20 | 21 | * `python_pachyderm repo `_ 22 | * `pachyderm repo `_ 23 | 24 | Indices and tables 25 | ================== 26 | 27 | * :ref:`genindex` 28 | * :ref:`modindex` 29 | * :ref:`search` 30 | -------------------------------------------------------------------------------- /docs/build/html/_sources/modules.rst.txt: -------------------------------------------------------------------------------- 1 | python_pachyderm 2 | ================ 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | python_pachyderm 8 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.mixin.rst.txt: -------------------------------------------------------------------------------- 1 | Important 2 | --------- 3 | 4 | .. automodule:: python_pachyderm.experimental.mixin 5 | :members: 6 | :undoc-members: 7 | 8 | Admin mixin 9 | ----------- 10 | 11 | .. automodule:: python_pachyderm.experimental.mixin.admin 12 | :members: 13 | :undoc-members: 14 | 15 | Auth mixin 16 | ---------- 17 | 18 | .. automodule:: python_pachyderm.experimental.mixin.auth 19 | :members: 20 | :undoc-members: 21 | 22 | Debug mixin 23 | ----------- 24 | 25 | .. automodule:: python_pachyderm.experimental.mixin.debug 26 | :members: 27 | :undoc-members: 28 | 29 | Enterprise mixin 30 | ---------------- 31 | 32 | .. automodule:: python_pachyderm.experimental.mixin.enterprise 33 | :members: 34 | :undoc-members: 35 | 36 | Health mixin 37 | ------------ 38 | 39 | .. automodule:: python_pachyderm.experimental.mixin.health 40 | :members: 41 | :undoc-members: 42 | 43 | Identity mixin 44 | -------------- 45 | 46 | .. automodule:: python_pachyderm.experimental.mixin.identity 47 | :members: 48 | :undoc-members: 49 | 50 | License mixin 51 | ------------- 52 | 53 | .. automodule:: python_pachyderm.experimental.mixin.license 54 | :members: 55 | :undoc-members: 56 | 57 | PFS mixin 58 | --------- 59 | 60 | .. automodule:: python_pachyderm.experimental.mixin.pfs 61 | :members: 62 | :undoc-members: 63 | :exclude-members: FileTarstream 64 | 65 | PPS mixin 66 | --------- 67 | 68 | .. automodule:: python_pachyderm.experimental.mixin.pps 69 | :members: 70 | :undoc-members: 71 | 72 | Transaction mixin 73 | ----------------- 74 | 75 | .. automodule:: python_pachyderm.experimental.mixin.transaction 76 | :members: 77 | :undoc-members: 78 | 79 | Version mixin 80 | ------------- 81 | 82 | .. automodule:: python_pachyderm.experimental.mixin.version 83 | :members: 84 | :undoc-members: 85 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.rst.txt: -------------------------------------------------------------------------------- 1 | Important 2 | ---------- 3 | 4 | .. automodule:: python_pachyderm.experimental.proto 5 | :members: 6 | :undoc-members: 7 | 8 | Admin proto 9 | ----------- 10 | 11 | .. automodule:: python_pachyderm.experimental.proto.v2.admin_v2 12 | :members: 13 | :undoc-members: 14 | :exclude-members: ApiBase, ApiStub 15 | 16 | Auth proto 17 | ---------- 18 | 19 | .. automodule:: python_pachyderm.experimental.proto.v2.auth_v2 20 | :members: 21 | :undoc-members: 22 | :exclude-members: ApiBase, ApiStub 23 | 24 | Debug proto 25 | ----------- 26 | 27 | .. automodule:: python_pachyderm.experimental.proto.v2.debug_v2 28 | :members: 29 | :undoc-members: 30 | :exclude-members: DebugBase, DebugStub 31 | 32 | Enterprise proto 33 | ---------------- 34 | 35 | .. automodule:: python_pachyderm.experimental.proto.v2.enterprise_v2 36 | :members: 37 | :undoc-members: 38 | :exclude-members: ApiBase, ApiStub 39 | 40 | Identity proto 41 | -------------- 42 | 43 | .. automodule:: python_pachyderm.experimental.proto.v2.identity_v2 44 | :members: 45 | :undoc-members: 46 | :exclude-members: ApiBase, ApiStub 47 | 48 | License proto 49 | ------------- 50 | 51 | .. automodule:: python_pachyderm.experimental.proto.v2.license_v2 52 | :members: 53 | :undoc-members: 54 | :exclude-members: ApiBase, ApiStub 55 | 56 | PFS proto 57 | --------- 58 | 59 | .. automodule:: python_pachyderm.experimental.proto.v2.pfs_v2 60 | :members: 61 | :undoc-members: 62 | :exclude-members: ApiBase, ApiStub, ActivateAuthRequest, ActivateAuthResponse 63 | 64 | PPS proto 65 | --------- 66 | 67 | .. automodule:: python_pachyderm.experimental.proto.v2.pps_v2 68 | :members: 69 | :undoc-members: 70 | :exclude-members: ApiBase, ApiStub, ActivateAuthRequest, ActivateAuthResponse 71 | 72 | Transaction proto 73 | ----------------- 74 | 75 | .. automodule:: python_pachyderm.experimental.proto.v2.transaction_v2 76 | :members: 77 | :undoc-members: 78 | :exclude-members: ApiBase, ApiStub 79 | 80 | Version proto 81 | ------------- 82 | 83 | .. automodule:: python_pachyderm.experimental.proto.v2.versionpb_v2 84 | :members: 85 | :undoc-members: 86 | :exclude-members: ApiBase, ApiStub 87 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.admin_v2.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.admin\_v2 package 2 | ========================================================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.admin_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.auth_v2.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.auth\_v2 package 2 | ======================================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.auth_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.debug_v2.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.debug\_v2 package 2 | ========================================================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.debug_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.enterprise_v2.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.enterprise\_v2 package 2 | ============================================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.enterprise_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.identity_v2.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.identity\_v2 package 2 | ============================================================ 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.identity_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.license_v2.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.license\_v2 package 2 | =========================================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.license_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.pfs_v2.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.pfs\_v2 package 2 | ======================================================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.pfs_v2 8 | :members: 9 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.pps_v2.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.pps\_v2 package 2 | ======================================================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.pps_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.proxy.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.proxy package 2 | ===================================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.proxy 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2 package 2 | =============================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | python_pachyderm.experimental.proto.v2.admin_v2 11 | python_pachyderm.experimental.proto.v2.auth_v2 12 | python_pachyderm.experimental.proto.v2.debug_v2 13 | python_pachyderm.experimental.proto.v2.enterprise_v2 14 | python_pachyderm.experimental.proto.v2.identity_v2 15 | python_pachyderm.experimental.proto.v2.license_v2 16 | python_pachyderm.experimental.proto.v2.pfs_v2 17 | python_pachyderm.experimental.proto.v2.pps_v2 18 | python_pachyderm.experimental.proto.v2.proxy 19 | python_pachyderm.experimental.proto.v2.transaction_v2 20 | python_pachyderm.experimental.proto.v2.versionpb_v2 21 | 22 | Module contents 23 | --------------- 24 | 25 | .. automodule:: python_pachyderm.experimental.proto.v2 26 | :members: 27 | :undoc-members: 28 | :show-inheritance: 29 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.transaction_v2.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.transaction\_v2 package 2 | =============================================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.transaction_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.proto.v2.versionpb_v2.rst.txt: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.versionpb\_v2 package 2 | ============================================================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.versionpb_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.experimental.rst.txt: -------------------------------------------------------------------------------- 1 | Information 2 | ----------- 3 | .. automodule:: python_pachyderm.experimental 4 | :members: 5 | 6 | 7 | Experimental Mixins 8 | ------------------- 9 | .. toctree:: 10 | :maxdepth: 4 11 | 12 | python_pachyderm.experimental.mixin 13 | 14 | 15 | Experimental Protos 16 | ------------------- 17 | .. toctree:: 18 | :maxdepth: 4 19 | 20 | python_pachyderm.experimental.proto 21 | 22 | 23 | Experimental Client 24 | ------------------- 25 | .. automodule:: python_pachyderm.experimental.client 26 | :members: 27 | :show-inheritance: 28 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.mixin.rst.txt: -------------------------------------------------------------------------------- 1 | Information 2 | ----------- 3 | 4 | .. automodule:: python_pachyderm.mixin 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | python\_pachyderm.mixin.admin 10 | ----------------------------- 11 | 12 | .. automodule:: python_pachyderm.mixin.admin 13 | :members: 14 | :undoc-members: 15 | 16 | python\_pachyderm.mixin.auth 17 | ---------------------------- 18 | 19 | .. automodule:: python_pachyderm.mixin.auth 20 | :members: 21 | :undoc-members: 22 | 23 | python\_pachyderm.mixin.debug 24 | ----------------------------- 25 | 26 | .. automodule:: python_pachyderm.mixin.debug 27 | :members: 28 | :undoc-members: 29 | 30 | python\_pachyderm.mixin.enterprise 31 | ---------------------------------- 32 | 33 | .. automodule:: python_pachyderm.mixin.enterprise 34 | :members: 35 | :undoc-members: 36 | 37 | python\_pachyderm.mixin.health 38 | ------------------------------ 39 | 40 | .. automodule:: python_pachyderm.mixin.health 41 | :members: 42 | :undoc-members: 43 | 44 | python\_pachyderm.mixin.identity 45 | -------------------------------- 46 | 47 | .. automodule:: python_pachyderm.mixin.identity 48 | :members: 49 | :undoc-members: 50 | 51 | python\_pachyderm.mixin.license 52 | ------------------------------- 53 | 54 | .. automodule:: python_pachyderm.mixin.license 55 | :members: 56 | :undoc-members: 57 | 58 | python\_pachyderm.mixin.pfs 59 | --------------------------- 60 | 61 | .. automodule:: python_pachyderm.mixin.pfs 62 | :members: 63 | :undoc-members: 64 | :exclude-members: FileTarstream 65 | 66 | python\_pachyderm.mixin.pps 67 | --------------------------- 68 | 69 | .. automodule:: python_pachyderm.mixin.pps 70 | :members: 71 | :undoc-members: 72 | 73 | python\_pachyderm.mixin.transaction 74 | ----------------------------------- 75 | 76 | .. automodule:: python_pachyderm.mixin.transaction 77 | :members: 78 | :undoc-members: 79 | 80 | python\_pachyderm.mixin.version 81 | ------------------------------- 82 | 83 | .. automodule:: python_pachyderm.mixin.version 84 | :members: 85 | :undoc-members: 86 | -------------------------------------------------------------------------------- /docs/build/html/_sources/python_pachyderm.rst.txt: -------------------------------------------------------------------------------- 1 | Mixins 2 | ------ 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | python_pachyderm.mixin 8 | 9 | Client 10 | ------ 11 | 12 | .. automodule:: python_pachyderm.client 13 | :members: 14 | :show-inheritance: 15 | :exclude-members: BadClusterDeploymentID, ConfigError 16 | :special-members: __init__ 17 | 18 | PFS Helper 19 | ---------- 20 | 21 | .. automodule:: python_pachyderm.pfs 22 | :members: 23 | :show-inheritance: 24 | 25 | Util Helper 26 | ----------- 27 | 28 | .. automodule:: python_pachyderm.util 29 | :members: 30 | :undoc-members: 31 | 32 | Experimental Module 33 | ------------------- 34 | 35 | .. toctree:: 36 | :maxdepth: 4 37 | 38 | python_pachyderm.experimental 39 | -------------------------------------------------------------------------------- /docs/build/html/_static/css/badge_only.css: -------------------------------------------------------------------------------- 1 | .fa:before{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}} -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/Roboto-Slab-Bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/Roboto-Slab-Bold.woff -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/Roboto-Slab-Bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/Roboto-Slab-Bold.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/Roboto-Slab-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/Roboto-Slab-Regular.woff -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/Roboto-Slab-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/Roboto-Slab-Regular.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/lato-bold-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/lato-bold-italic.woff -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/lato-bold-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/lato-bold-italic.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/lato-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/lato-bold.woff -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/lato-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/lato-bold.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/lato-normal-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/lato-normal-italic.woff -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/lato-normal-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/lato-normal-italic.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/lato-normal.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/lato-normal.woff -------------------------------------------------------------------------------- /docs/build/html/_static/css/fonts/lato-normal.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/css/fonts/lato-normal.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/custom.css: -------------------------------------------------------------------------------- 1 | /* This file intentionally left blank. */ 2 | -------------------------------------------------------------------------------- /docs/build/html/_static/documentation_options.js: -------------------------------------------------------------------------------- 1 | var DOCUMENTATION_OPTIONS = { 2 | URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), 3 | VERSION: '', 4 | LANGUAGE: 'None', 5 | COLLAPSE_INDEX: false, 6 | BUILDER: 'html', 7 | FILE_SUFFIX: '.html', 8 | LINK_SUFFIX: '.html', 9 | HAS_SOURCE: true, 10 | SOURCELINK_SUFFIX: '.txt', 11 | NAVIGATION_WITH_KEYS: false 12 | }; -------------------------------------------------------------------------------- /docs/build/html/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/file.png -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Inconsolata-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Inconsolata-Bold.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Inconsolata-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Inconsolata-Regular.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Inconsolata.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Inconsolata.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato-Bold.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato-Regular.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-bold.eot -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-bold.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-bold.woff -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-bold.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-bolditalic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-bolditalic.eot -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-bolditalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-bolditalic.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-bolditalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-bolditalic.woff -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-bolditalic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-bolditalic.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-italic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-italic.eot -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-italic.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-italic.woff -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-italic.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-regular.eot -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-regular.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-regular.woff -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/Lato/lato-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/Lato/lato-regular.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/RobotoSlab-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/RobotoSlab-Bold.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/RobotoSlab-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/RobotoSlab-Regular.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.eot -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /docs/build/html/_static/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/docs/build/html/_static/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /docs/build/html/_static/js/badge_only.js: -------------------------------------------------------------------------------- 1 | !function(e){var t={};function r(n){if(t[n])return t[n].exports;var o=t[n]={i:n,l:!1,exports:{}};return e[n].call(o.exports,o,o.exports,r),o.l=!0,o.exports}r.m=e,r.c=t,r.d=function(e,t,n){r.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:n})},r.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.t=function(e,t){if(1&t&&(e=r(e)),8&t)return e;if(4&t&&"object"==typeof e&&e&&e.__esModule)return e;var n=Object.create(null);if(r.r(n),Object.defineProperty(n,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var o in e)r.d(n,o,function(t){return e[t]}.bind(null,o));return n},r.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return r.d(t,"a",t),t},r.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r.p="",r(r.s=4)}({4:function(e,t,r){}}); -------------------------------------------------------------------------------- /docs/build/html/_static/js/html5shiv-printshiv.min.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @preserve HTML5 Shiv 3.7.3-pre | @afarkas @jdalton @jon_neal @rem | MIT/GPL2 Licensed 3 | */ 4 | !function(a,b){function c(a,b){var c=a.createElement("p"),d=a.getElementsByTagName("head")[0]||a.documentElement;return c.innerHTML="x",d.insertBefore(c.lastChild,d.firstChild)}function d(){var a=y.elements;return"string"==typeof a?a.split(" "):a}function e(a,b){var c=y.elements;"string"!=typeof c&&(c=c.join(" ")),"string"!=typeof a&&(a=a.join(" ")),y.elements=c+" "+a,j(b)}function f(a){var b=x[a[v]];return b||(b={},w++,a[v]=w,x[w]=b),b}function g(a,c,d){if(c||(c=b),q)return c.createElement(a);d||(d=f(c));var e;return e=d.cache[a]?d.cache[a].cloneNode():u.test(a)?(d.cache[a]=d.createElem(a)).cloneNode():d.createElem(a),!e.canHaveChildren||t.test(a)||e.tagUrn?e:d.frag.appendChild(e)}function h(a,c){if(a||(a=b),q)return a.createDocumentFragment();c=c||f(a);for(var e=c.frag.cloneNode(),g=0,h=d(),i=h.length;i>g;g++)e.createElement(h[g]);return e}function i(a,b){b.cache||(b.cache={},b.createElem=a.createElement,b.createFrag=a.createDocumentFragment,b.frag=b.createFrag()),a.createElement=function(c){return y.shivMethods?g(c,a,b):b.createElem(c)},a.createDocumentFragment=Function("h,f","return function(){var n=f.cloneNode(),c=n.createElement;h.shivMethods&&("+d().join().replace(/[\w\-:]+/g,function(a){return b.createElem(a),b.frag.createElement(a),'c("'+a+'")'})+");return n}")(y,b.frag)}function j(a){a||(a=b);var d=f(a);return!y.shivCSS||p||d.hasCSS||(d.hasCSS=!!c(a,"article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}mark{background:#FF0;color:#000}template{display:none}")),q||i(a,d),a}function k(a){for(var b,c=a.getElementsByTagName("*"),e=c.length,f=RegExp("^(?:"+d().join("|")+")$","i"),g=[];e--;)b=c[e],f.test(b.nodeName)&&g.push(b.applyElement(l(b)));return g}function l(a){for(var b,c=a.attributes,d=c.length,e=a.ownerDocument.createElement(A+":"+a.nodeName);d--;)b=c[d],b.specified&&e.setAttribute(b.nodeName,b.nodeValue);return e.style.cssText=a.style.cssText,e}function m(a){for(var b,c=a.split("{"),e=c.length,f=RegExp("(^|[\\s,>+~])("+d().join("|")+")(?=[[\\s,>+~#.:]|$)","gi"),g="$1"+A+"\\:$2";e--;)b=c[e]=c[e].split("}"),b[b.length-1]=b[b.length-1].replace(f,g),c[e]=b.join("}");return c.join("{")}function n(a){for(var b=a.length;b--;)a[b].removeNode()}function o(a){function b(){clearTimeout(g._removeSheetTimer),d&&d.removeNode(!0),d=null}var d,e,g=f(a),h=a.namespaces,i=a.parentWindow;return!B||a.printShived?a:("undefined"==typeof h[A]&&h.add(A),i.attachEvent("onbeforeprint",function(){b();for(var f,g,h,i=a.styleSheets,j=[],l=i.length,n=Array(l);l--;)n[l]=i[l];for(;h=n.pop();)if(!h.disabled&&z.test(h.media)){try{f=h.imports,g=f.length}catch(o){g=0}for(l=0;g>l;l++)n.push(f[l]);try{j.push(h.cssText)}catch(o){}}j=m(j.reverse().join("")),e=k(a),d=c(a,j)}),i.attachEvent("onafterprint",function(){n(e),clearTimeout(g._removeSheetTimer),g._removeSheetTimer=setTimeout(b,500)}),a.printShived=!0,a)}var p,q,r="3.7.3",s=a.html5||{},t=/^<|^(?:button|map|select|textarea|object|iframe|option|optgroup)$/i,u=/^(?:a|b|code|div|fieldset|h1|h2|h3|h4|h5|h6|i|label|li|ol|p|q|span|strong|style|table|tbody|td|th|tr|ul)$/i,v="_html5shiv",w=0,x={};!function(){try{var a=b.createElement("a");a.innerHTML="",p="hidden"in a,q=1==a.childNodes.length||function(){b.createElement("a");var a=b.createDocumentFragment();return"undefined"==typeof a.cloneNode||"undefined"==typeof a.createDocumentFragment||"undefined"==typeof a.createElement}()}catch(c){p=!0,q=!0}}();var y={elements:s.elements||"abbr article aside audio bdi canvas data datalist details dialog figcaption figure footer header hgroup main mark meter nav output picture progress section summary template time video",version:r,shivCSS:s.shivCSS!==!1,supportsUnknownElements:q,shivMethods:s.shivMethods!==!1,type:"default",shivDocument:j,createElement:g,createDocumentFragment:h,addElements:e};a.html5=y,j(b);var z=/^$|\b(?:all|print)\b/,A="html5shiv",B=!q&&function(){var c=b.documentElement;return!("undefined"==typeof b.namespaces||"undefined"==typeof b.parentWindow||"undefined"==typeof c.applyElement||"undefined"==typeof c.removeNode||"undefined"==typeof a.attachEvent)}();y.type+=" print",y.shivPrint=o,o(b),"object"==typeof module&&module.exports&&(module.exports=y)}("undefined"!=typeof window?window:this,document); -------------------------------------------------------------------------------- /docs/build/html/_static/js/html5shiv.min.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @preserve HTML5 Shiv 3.7.3 | @afarkas @jdalton @jon_neal @rem | MIT/GPL2 Licensed 3 | */ 4 | !function(a,b){function c(a,b){var c=a.createElement("p"),d=a.getElementsByTagName("head")[0]||a.documentElement;return c.innerHTML="x",d.insertBefore(c.lastChild,d.firstChild)}function d(){var a=t.elements;return"string"==typeof a?a.split(" "):a}function e(a,b){var c=t.elements;"string"!=typeof c&&(c=c.join(" ")),"string"!=typeof a&&(a=a.join(" ")),t.elements=c+" "+a,j(b)}function f(a){var b=s[a[q]];return b||(b={},r++,a[q]=r,s[r]=b),b}function g(a,c,d){if(c||(c=b),l)return c.createElement(a);d||(d=f(c));var e;return e=d.cache[a]?d.cache[a].cloneNode():p.test(a)?(d.cache[a]=d.createElem(a)).cloneNode():d.createElem(a),!e.canHaveChildren||o.test(a)||e.tagUrn?e:d.frag.appendChild(e)}function h(a,c){if(a||(a=b),l)return a.createDocumentFragment();c=c||f(a);for(var e=c.frag.cloneNode(),g=0,h=d(),i=h.length;i>g;g++)e.createElement(h[g]);return e}function i(a,b){b.cache||(b.cache={},b.createElem=a.createElement,b.createFrag=a.createDocumentFragment,b.frag=b.createFrag()),a.createElement=function(c){return t.shivMethods?g(c,a,b):b.createElem(c)},a.createDocumentFragment=Function("h,f","return function(){var n=f.cloneNode(),c=n.createElement;h.shivMethods&&("+d().join().replace(/[\w\-:]+/g,function(a){return b.createElem(a),b.frag.createElement(a),'c("'+a+'")'})+");return n}")(t,b.frag)}function j(a){a||(a=b);var d=f(a);return!t.shivCSS||k||d.hasCSS||(d.hasCSS=!!c(a,"article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}mark{background:#FF0;color:#000}template{display:none}")),l||i(a,d),a}var k,l,m="3.7.3-pre",n=a.html5||{},o=/^<|^(?:button|map|select|textarea|object|iframe|option|optgroup)$/i,p=/^(?:a|b|code|div|fieldset|h1|h2|h3|h4|h5|h6|i|label|li|ol|p|q|span|strong|style|table|tbody|td|th|tr|ul)$/i,q="_html5shiv",r=0,s={};!function(){try{var a=b.createElement("a");a.innerHTML="",k="hidden"in a,l=1==a.childNodes.length||function(){b.createElement("a");var a=b.createDocumentFragment();return"undefined"==typeof a.cloneNode||"undefined"==typeof a.createDocumentFragment||"undefined"==typeof a.createElement}()}catch(c){k=!0,l=!0}}();var t={elements:n.elements||"abbr article aside audio bdi canvas data datalist details dialog figcaption figure footer header hgroup main mark meter nav output picture progress section summary template time video",version:m,shivCSS:n.shivCSS!==!1,supportsUnknownElements:l,shivMethods:n.shivMethods!==!1,type:"default",shivDocument:j,createElement:g,createDocumentFragment:h,addElements:e};a.html5=t,j(b),"object"==typeof module&&module.exports&&(module.exports=t)}("undefined"!=typeof window?window:this,document); -------------------------------------------------------------------------------- /docs/build/html/_static/js/theme.js: -------------------------------------------------------------------------------- 1 | !function(n){var e={};function t(i){if(e[i])return e[i].exports;var o=e[i]={i:i,l:!1,exports:{}};return n[i].call(o.exports,o,o.exports,t),o.l=!0,o.exports}t.m=n,t.c=e,t.d=function(n,e,i){t.o(n,e)||Object.defineProperty(n,e,{enumerable:!0,get:i})},t.r=function(n){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(n,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(n,"__esModule",{value:!0})},t.t=function(n,e){if(1&e&&(n=t(n)),8&e)return n;if(4&e&&"object"==typeof n&&n&&n.__esModule)return n;var i=Object.create(null);if(t.r(i),Object.defineProperty(i,"default",{enumerable:!0,value:n}),2&e&&"string"!=typeof n)for(var o in n)t.d(i,o,function(e){return n[e]}.bind(null,o));return i},t.n=function(n){var e=n&&n.__esModule?function(){return n.default}:function(){return n};return t.d(e,"a",e),e},t.o=function(n,e){return Object.prototype.hasOwnProperty.call(n,e)},t.p="",t(t.s=0)}([function(n,e,t){t(1),n.exports=t(3)},function(n,e,t){(function(){var e="undefined"!=typeof window?window.jQuery:t(2);n.exports.ThemeNav={navBar:null,win:null,winScroll:!1,winResize:!1,linkScroll:!1,winPosition:0,winHeight:null,docHeight:null,isRunning:!1,enable:function(n){var t=this;void 0===n&&(n=!0),t.isRunning||(t.isRunning=!0,e((function(e){t.init(e),t.reset(),t.win.on("hashchange",t.reset),n&&t.win.on("scroll",(function(){t.linkScroll||t.winScroll||(t.winScroll=!0,requestAnimationFrame((function(){t.onScroll()})))})),t.win.on("resize",(function(){t.winResize||(t.winResize=!0,requestAnimationFrame((function(){t.onResize()})))})),t.onResize()})))},enableSticky:function(){this.enable(!0)},init:function(n){n(document);var e=this;this.navBar=n("div.wy-side-scroll:first"),this.win=n(window),n(document).on("click","[data-toggle='wy-nav-top']",(function(){n("[data-toggle='wy-nav-shift']").toggleClass("shift"),n("[data-toggle='rst-versions']").toggleClass("shift")})).on("click",".wy-menu-vertical .current ul li a",(function(){var t=n(this);n("[data-toggle='wy-nav-shift']").removeClass("shift"),n("[data-toggle='rst-versions']").toggleClass("shift"),e.toggleCurrent(t),e.hashChange()})).on("click","[data-toggle='rst-current-version']",(function(){n("[data-toggle='rst-versions']").toggleClass("shift-up")})),n("table.docutils:not(.field-list,.footnote,.citation)").wrap(""),n("table.docutils.footnote").wrap(""),n("table.docutils.citation").wrap(""),n(".wy-menu-vertical ul").not(".simple").siblings("a").each((function(){var t=n(this);expand=n(''),expand.on("click",(function(n){return e.toggleCurrent(t),n.stopPropagation(),!1})),t.prepend(expand)}))},reset:function(){var n=encodeURI(window.location.hash)||"#";try{var e=$(".wy-menu-vertical"),t=e.find('[href="'+n+'"]');if(0===t.length){var i=$('.document [id="'+n.substring(1)+'"]').closest("div.section");0===(t=e.find('[href="#'+i.attr("id")+'"]')).length&&(t=e.find('[href="#"]'))}if(t.length>0){$(".wy-menu-vertical .current").removeClass("current").attr("aria-expanded","false"),t.addClass("current").attr("aria-expanded","true"),t.closest("li.toctree-l1").parent().addClass("current").attr("aria-expanded","true");for(let n=1;n<=10;n++)t.closest("li.toctree-l"+n).addClass("current").attr("aria-expanded","true");t[0].scrollIntoView()}}catch(n){console.log("Error expanding nav for anchor",n)}},onScroll:function(){this.winScroll=!1;var n=this.win.scrollTop(),e=n+this.winHeight,t=this.navBar.scrollTop()+(n-this.winPosition);n<0||e>this.docHeight||(this.navBar.scrollTop(t),this.winPosition=n)},onResize:function(){this.winResize=!1,this.winHeight=this.win.height(),this.docHeight=$(document).height()},hashChange:function(){this.linkScroll=!0,this.win.one("hashchange",(function(){this.linkScroll=!1}))},toggleCurrent:function(n){var e=n.closest("li");e.siblings("li.current").removeClass("current").attr("aria-expanded","false"),e.siblings().find("li.current").removeClass("current").attr("aria-expanded","false");var t=e.find("> ul li");t.length&&(t.removeClass("current").attr("aria-expanded","false"),e.toggleClass("current").attr("aria-expanded",(function(n,e){return"true"==e?"false":"true"})))}},"undefined"!=typeof window&&(window.SphinxRtdTheme={Navigation:n.exports.ThemeNav,StickyNav:n.exports.ThemeNav}),function(){for(var n=0,e=["ms","moz","webkit","o"],t=0;t 2 | 3 | 4 | 5 | 6 | Search — python-pachyderm documentation 7 | 8 | 9 | 10 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | python-pachyderm 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | Docs 41 | 42 | Getting Started 43 | python_pachyderm 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | python-pachyderm 53 | 54 | 55 | 56 | 57 | 58 | 59 | » 60 | Search 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | Please activate JavaScript to enable the search functionality. 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 98 | 99 | 100 | 101 | 102 | 107 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | 16 | sys.path.insert(0, os.path.abspath("../../src")) 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = "python-pachyderm" 22 | copyright = "2021, Joe Doliner" 23 | author = "Joe Doliner" 24 | 25 | 26 | # -- General configuration --------------------------------------------------- 27 | # Add any Sphinx extension module names here, as strings. They can be 28 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 29 | # ones. 30 | extensions = ["sphinx.ext.autodoc", "numpydoc", "sphinx.ext.viewcode", "myst_parser"] 31 | 32 | # Remove type hints in method signatures as they clutter up docs. 33 | autodoc_typehints = "none" 34 | 35 | # Add any paths that contain templates here, relative to this directory. 36 | templates_path = ["_templates"] 37 | 38 | # List of patterns, relative to source directory, that match files and 39 | # directories to ignore when looking for source files. 40 | # This pattern also affects html_static_path and html_extra_path. 41 | exclude_patterns = [] 42 | 43 | # The master toctree document. 44 | master_doc = "index" 45 | 46 | # If True, module will be added to title of every member 47 | add_module_names = False 48 | 49 | 50 | # -- Options for HTML output ------------------------------------------------- 51 | 52 | # The theme to use for HTML and HTML Help pages. See the documentation for 53 | # a list of builtin themes. 54 | # 55 | html_theme = "sphinx_rtd_theme" 56 | 57 | # Add any paths that contain custom static files (such as style sheets) here, 58 | # relative to this directory. They are copied after the builtin static files, 59 | # so a file named "default.css" will overwrite the builtin "default.css". 60 | html_static_path = ["_static"] 61 | -------------------------------------------------------------------------------- /docs/source/getting_started.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | ## Prerequisites 4 | * A workspace on [Pachyderm Hub](https://docs.pachyderm.com/latest/hub/hub_getting_started/) (recommended) or Pachyderm running [locally](https://docs.pachyderm.com/latest/getting_started/local_installation/) 5 | * Python 3.6 or higher 6 | 7 | ## Installation 8 | * via PyPI: `pip install python-pachyderm` 9 | * via source 10 | * download desired version [here](https://github.com/pachyderm/python-pachyderm/releases) (under assets) 11 | * `make init` in folder root-level 12 | 13 | ## Hello World example 14 | 15 | Creates a repo, commits data to the repo (versioning the data), and reads the data back from the repo. 16 | 17 | ```python 18 | import python_pachyderm 19 | 20 | # Connects to a pachyderm cluster on localhost:30650 21 | # For other connection options, see the API docs 22 | client = python_pachyderm.Client() 23 | 24 | # Create a pachyderm repo called `test` 25 | client.create_repo("test") 26 | 27 | # Create a file in (repo="test", branch="master") at `/dir_a/data.txt` 28 | # Similar to `pachctl put file test@master:/dir_a/data.txt` 29 | with client.commit("test", "master") as commit: 30 | client.put_file_bytes(commit, "/dir_a/data.txt", b"hello world") 31 | 32 | # Get the file 33 | f = client.get_file(("test", "master"), "/dir_a/data.txt") 34 | print(f.read()) # >>> b"hello world" 35 | ``` 36 | 37 | Since `client.get_file()` returns a file-like object, you can pass it into your favorite analysis packages. 38 | 39 | ```python 40 | import pandas as pd 41 | 42 | f = client.get_file(("my_repo", "my_branch"), "/path_to/my_data.csv") 43 | df = pd.read_csv(f) 44 | ``` 45 | 46 | ## Hello World example with Pachyderm Pipelines 47 | 48 | Creates a data-driven pipeline that transforms the data and outputs results to a new repo. This example pipeline counts the occurrences of the word `hello` in the repo. Continued from the example above... 49 | 50 | ```python 51 | from python_pachyderm.service import pps_proto 52 | 53 | # Create a pipeline that logs frequency of the word "hello" in `test` 54 | # repo to a file in the `word_count` repo (which is created automatically) 55 | # Any time data is committed to the `test` repo, this pipeline will 56 | # automatically trigger. 57 | client.create_pipeline( 58 | "word_count", 59 | transform=pps_proto.Transform( 60 | cmd=["bash"], 61 | stdin=[ 62 | "grep -roh hello /pfs/test/ | wc -w > /pfs/out/count.txt" 63 | ] 64 | ), 65 | input=pps_proto.Input( 66 | pfs=pps_proto.PFSInput(repo="test", branch="master", glob="/") 67 | ) 68 | ) 69 | 70 | # Wait for new commit, triggered by pipeline run, to finish 71 | client.wait_commit(("word_count", "master")) 72 | 73 | # Check `count.txt` for "hello" count 74 | f = client.get_file(("word_count", "master"), "count.txt") 75 | print(f.read()) # >>> b"1" 76 | 77 | # Add more data to the `test` repo 78 | with client.commit("test", "master") as commit: 79 | client.put_file_bytes(commit, "/data2.txt", b"hello hello from the top of the world") 80 | 81 | # Wait for commit to finish 82 | client.wait_commit(commit.id) 83 | 84 | # Check `count.txt` for "hello" count 85 | f = client.get_file(("word_count", "master"), "count.txt") 86 | print(f.read()) # >>> b"3" 87 | ``` 88 | 89 | For more sophisticated examples, [see the examples directory](https://github.com/pachyderm/python-pachyderm/tree/master/examples). 90 | To learn more about what you can do with Pachyderm, check out the [docs](https://docs.pachyderm.com/latest/how-tos/). -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | python-pachyderm 2 | ============================================ 3 | 4 | Overview 5 | -------- 6 | 7 | python-pachyderm is a Python client that interacts with `Pachyderm `_, 8 | a tool for version-controlled, automated, end-to-end data pipelines for data science. If you're not 9 | familiar with Pachyderm or its value, check out that first! 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | :caption: Docs 14 | 15 | getting_started.md 16 | modules.rst 17 | 18 | Links 19 | ----- 20 | 21 | * `python_pachyderm repo `_ 22 | * `pachyderm repo `_ 23 | 24 | Indices and tables 25 | ================== 26 | 27 | * :ref:`genindex` 28 | * :ref:`modindex` 29 | * :ref:`search` 30 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | python_pachyderm 2 | ================ 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | python_pachyderm 8 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.mixin.rst: -------------------------------------------------------------------------------- 1 | Important 2 | --------- 3 | 4 | .. automodule:: python_pachyderm.experimental.mixin 5 | :members: 6 | :undoc-members: 7 | 8 | Admin mixin 9 | ----------- 10 | 11 | .. automodule:: python_pachyderm.experimental.mixin.admin 12 | :members: 13 | :undoc-members: 14 | 15 | Auth mixin 16 | ---------- 17 | 18 | .. automodule:: python_pachyderm.experimental.mixin.auth 19 | :members: 20 | :undoc-members: 21 | 22 | Debug mixin 23 | ----------- 24 | 25 | .. automodule:: python_pachyderm.experimental.mixin.debug 26 | :members: 27 | :undoc-members: 28 | 29 | Enterprise mixin 30 | ---------------- 31 | 32 | .. automodule:: python_pachyderm.experimental.mixin.enterprise 33 | :members: 34 | :undoc-members: 35 | 36 | Health mixin 37 | ------------ 38 | 39 | .. automodule:: python_pachyderm.experimental.mixin.health 40 | :members: 41 | :undoc-members: 42 | 43 | Identity mixin 44 | -------------- 45 | 46 | .. automodule:: python_pachyderm.experimental.mixin.identity 47 | :members: 48 | :undoc-members: 49 | 50 | License mixin 51 | ------------- 52 | 53 | .. automodule:: python_pachyderm.experimental.mixin.license 54 | :members: 55 | :undoc-members: 56 | 57 | PFS mixin 58 | --------- 59 | 60 | .. automodule:: python_pachyderm.experimental.mixin.pfs 61 | :members: 62 | :undoc-members: 63 | :exclude-members: FileTarstream 64 | 65 | PPS mixin 66 | --------- 67 | 68 | .. automodule:: python_pachyderm.experimental.mixin.pps 69 | :members: 70 | :undoc-members: 71 | 72 | Transaction mixin 73 | ----------------- 74 | 75 | .. automodule:: python_pachyderm.experimental.mixin.transaction 76 | :members: 77 | :undoc-members: 78 | 79 | Version mixin 80 | ------------- 81 | 82 | .. automodule:: python_pachyderm.experimental.mixin.version 83 | :members: 84 | :undoc-members: 85 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.rst: -------------------------------------------------------------------------------- 1 | Important 2 | ---------- 3 | 4 | .. automodule:: python_pachyderm.experimental.proto 5 | :members: 6 | :undoc-members: 7 | 8 | Admin proto 9 | ----------- 10 | 11 | .. automodule:: python_pachyderm.experimental.proto.v2.admin_v2 12 | :members: 13 | :undoc-members: 14 | :exclude-members: ApiBase, ApiStub 15 | 16 | Auth proto 17 | ---------- 18 | 19 | .. automodule:: python_pachyderm.experimental.proto.v2.auth_v2 20 | :members: 21 | :undoc-members: 22 | :exclude-members: ApiBase, ApiStub 23 | 24 | Debug proto 25 | ----------- 26 | 27 | .. automodule:: python_pachyderm.experimental.proto.v2.debug_v2 28 | :members: 29 | :undoc-members: 30 | :exclude-members: DebugBase, DebugStub 31 | 32 | Enterprise proto 33 | ---------------- 34 | 35 | .. automodule:: python_pachyderm.experimental.proto.v2.enterprise_v2 36 | :members: 37 | :undoc-members: 38 | :exclude-members: ApiBase, ApiStub 39 | 40 | Identity proto 41 | -------------- 42 | 43 | .. automodule:: python_pachyderm.experimental.proto.v2.identity_v2 44 | :members: 45 | :undoc-members: 46 | :exclude-members: ApiBase, ApiStub 47 | 48 | License proto 49 | ------------- 50 | 51 | .. automodule:: python_pachyderm.experimental.proto.v2.license_v2 52 | :members: 53 | :undoc-members: 54 | :exclude-members: ApiBase, ApiStub 55 | 56 | PFS proto 57 | --------- 58 | 59 | .. automodule:: python_pachyderm.experimental.proto.v2.pfs_v2 60 | :members: 61 | :undoc-members: 62 | :exclude-members: ApiBase, ApiStub, ActivateAuthRequest, ActivateAuthResponse 63 | 64 | PPS proto 65 | --------- 66 | 67 | .. automodule:: python_pachyderm.experimental.proto.v2.pps_v2 68 | :members: 69 | :undoc-members: 70 | :exclude-members: ApiBase, ApiStub, ActivateAuthRequest, ActivateAuthResponse 71 | 72 | Transaction proto 73 | ----------------- 74 | 75 | .. automodule:: python_pachyderm.experimental.proto.v2.transaction_v2 76 | :members: 77 | :undoc-members: 78 | :exclude-members: ApiBase, ApiStub 79 | 80 | Version proto 81 | ------------- 82 | 83 | .. automodule:: python_pachyderm.experimental.proto.v2.versionpb_v2 84 | :members: 85 | :undoc-members: 86 | :exclude-members: ApiBase, ApiStub 87 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.admin_v2.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.admin\_v2 package 2 | ========================================================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.admin_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.auth_v2.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.auth\_v2 package 2 | ======================================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.auth_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.debug_v2.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.debug\_v2 package 2 | ========================================================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.debug_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.enterprise_v2.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.enterprise\_v2 package 2 | ============================================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.enterprise_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.identity_v2.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.identity\_v2 package 2 | ============================================================ 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.identity_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.license_v2.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.license\_v2 package 2 | =========================================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.license_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.pfs_v2.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.pfs\_v2 package 2 | ======================================================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.pfs_v2 8 | :members: 9 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.pps_v2.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.pps\_v2 package 2 | ======================================================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.pps_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.proxy.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.proxy package 2 | ===================================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.proxy 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2 package 2 | =============================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | python_pachyderm.experimental.proto.v2.admin_v2 11 | python_pachyderm.experimental.proto.v2.auth_v2 12 | python_pachyderm.experimental.proto.v2.debug_v2 13 | python_pachyderm.experimental.proto.v2.enterprise_v2 14 | python_pachyderm.experimental.proto.v2.identity_v2 15 | python_pachyderm.experimental.proto.v2.license_v2 16 | python_pachyderm.experimental.proto.v2.pfs_v2 17 | python_pachyderm.experimental.proto.v2.pps_v2 18 | python_pachyderm.experimental.proto.v2.proxy 19 | python_pachyderm.experimental.proto.v2.transaction_v2 20 | python_pachyderm.experimental.proto.v2.versionpb_v2 21 | 22 | Module contents 23 | --------------- 24 | 25 | .. automodule:: python_pachyderm.experimental.proto.v2 26 | :members: 27 | :undoc-members: 28 | :show-inheritance: 29 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.transaction_v2.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.transaction\_v2 package 2 | =============================================================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.transaction_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.proto.v2.versionpb_v2.rst: -------------------------------------------------------------------------------- 1 | python\_pachyderm.experimental.proto.v2.versionpb\_v2 package 2 | ============================================================= 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: python_pachyderm.experimental.proto.v2.versionpb_v2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.experimental.rst: -------------------------------------------------------------------------------- 1 | Information 2 | ----------- 3 | .. automodule:: python_pachyderm.experimental 4 | :members: 5 | 6 | 7 | Experimental Mixins 8 | ------------------- 9 | .. toctree:: 10 | :maxdepth: 4 11 | 12 | python_pachyderm.experimental.mixin 13 | 14 | 15 | Experimental Protos 16 | ------------------- 17 | .. toctree:: 18 | :maxdepth: 4 19 | 20 | python_pachyderm.experimental.proto 21 | 22 | 23 | Experimental Client 24 | ------------------- 25 | .. automodule:: python_pachyderm.experimental.client 26 | :members: 27 | :show-inheritance: 28 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.mixin.rst: -------------------------------------------------------------------------------- 1 | Information 2 | ----------- 3 | 4 | .. automodule:: python_pachyderm.mixin 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | python\_pachyderm.mixin.admin 10 | ----------------------------- 11 | 12 | .. automodule:: python_pachyderm.mixin.admin 13 | :members: 14 | :undoc-members: 15 | 16 | python\_pachyderm.mixin.auth 17 | ---------------------------- 18 | 19 | .. automodule:: python_pachyderm.mixin.auth 20 | :members: 21 | :undoc-members: 22 | 23 | python\_pachyderm.mixin.debug 24 | ----------------------------- 25 | 26 | .. automodule:: python_pachyderm.mixin.debug 27 | :members: 28 | :undoc-members: 29 | 30 | python\_pachyderm.mixin.enterprise 31 | ---------------------------------- 32 | 33 | .. automodule:: python_pachyderm.mixin.enterprise 34 | :members: 35 | :undoc-members: 36 | 37 | python\_pachyderm.mixin.health 38 | ------------------------------ 39 | 40 | .. automodule:: python_pachyderm.mixin.health 41 | :members: 42 | :undoc-members: 43 | 44 | python\_pachyderm.mixin.identity 45 | -------------------------------- 46 | 47 | .. automodule:: python_pachyderm.mixin.identity 48 | :members: 49 | :undoc-members: 50 | 51 | python\_pachyderm.mixin.license 52 | ------------------------------- 53 | 54 | .. automodule:: python_pachyderm.mixin.license 55 | :members: 56 | :undoc-members: 57 | 58 | python\_pachyderm.mixin.pfs 59 | --------------------------- 60 | 61 | .. automodule:: python_pachyderm.mixin.pfs 62 | :members: 63 | :undoc-members: 64 | :exclude-members: FileTarstream 65 | 66 | python\_pachyderm.mixin.pps 67 | --------------------------- 68 | 69 | .. automodule:: python_pachyderm.mixin.pps 70 | :members: 71 | :undoc-members: 72 | 73 | python\_pachyderm.mixin.transaction 74 | ----------------------------------- 75 | 76 | .. automodule:: python_pachyderm.mixin.transaction 77 | :members: 78 | :undoc-members: 79 | 80 | python\_pachyderm.mixin.version 81 | ------------------------------- 82 | 83 | .. automodule:: python_pachyderm.mixin.version 84 | :members: 85 | :undoc-members: 86 | -------------------------------------------------------------------------------- /docs/source/python_pachyderm.rst: -------------------------------------------------------------------------------- 1 | Mixins 2 | ------ 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | python_pachyderm.mixin 8 | 9 | Client 10 | ------ 11 | 12 | .. automodule:: python_pachyderm.client 13 | :members: 14 | :show-inheritance: 15 | :exclude-members: BadClusterDeploymentID, ConfigError 16 | :special-members: __init__ 17 | 18 | PFS Helper 19 | ---------- 20 | 21 | .. automodule:: python_pachyderm.pfs 22 | :members: 23 | :show-inheritance: 24 | 25 | Util Helper 26 | ----------- 27 | 28 | .. automodule:: python_pachyderm.util 29 | :members: 30 | :undoc-members: 31 | 32 | Experimental Module 33 | ------------------- 34 | 35 | .. toctree:: 36 | :maxdepth: 4 37 | 38 | python_pachyderm.experimental 39 | -------------------------------------------------------------------------------- /etc/kube/check_ready.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$1" ]; then 4 | echo "Usage $0 label=value " 5 | exit 1 6 | fi 7 | 8 | namespace=default 9 | 10 | if [ -n "$2" ]; then 11 | namespace=$2 12 | fi 13 | 14 | results=$(kubectl get pods \ 15 | -l "$1" \ 16 | --namespace="$namespace" \ 17 | -o jsonpath='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' \ 18 | | tr ';' "\n") 19 | 20 | if [ -z "$results" ]; then 21 | echo "Empty result" 22 | echo "$results" 23 | exit 1 24 | fi 25 | 26 | 27 | readyPods=$(echo "$results" | tr ' ' "\n" | grep -c "Ready=True") 28 | allPods=$(echo "$results" | tr ' ' "\n" | grep -c "Ready=") 29 | 30 | if [ "$allPods" -eq 0 ]; then 31 | echo "No pods found yet" 32 | exit 1 33 | fi 34 | 35 | if [ "$readyPods" -ne "$allPods" ]; then 36 | echo "${readyPods}/${allPods} ready" 37 | exit 1 38 | fi 39 | 40 | echo "All pods are ready." 41 | exit 0 42 | -------------------------------------------------------------------------------- /etc/kube/start-minikube.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -Eex 4 | 5 | # Parse flags 6 | KUBE_VERSION="$( jq -r .kubernetes <"$(git rev-parse --show-toplevel)/version.json" )" 7 | while getopts ":v" opt; do 8 | case "${opt}" in 9 | v) 10 | KUBE_VERSION="v${OPTARG}" 11 | ;; 12 | \?) 13 | echo "Invalid argument: ${opt}" 14 | exit 1 15 | ;; 16 | esac 17 | done 18 | minikube version 19 | 20 | minikube_args=( 21 | "--vm-driver=none" 22 | "--kubernetes-version=${KUBE_VERSION}" 23 | ) 24 | 25 | 26 | # To make a 'sudo' run a binary in '~/cached-deps', you need to explicitly set 27 | # the path like so: 28 | # sudo env "PATH=$PATH" minikube foo 29 | sudo env "PATH=$PATH" "CHANGE_MINIKUBE_NONE_USER=true" \ 30 | minikube start "${minikube_args[@]}" 31 | 32 | # Try to connect for three minutes 33 | for _ in $(seq 36); do 34 | if kubectl version &>/dev/null; then 35 | exit 0 36 | fi 37 | sleep 5 38 | done 39 | 40 | # Give up--kubernetes isn't coming up 41 | minikube delete 42 | sleep 30 # Wait for minikube to go completely down 43 | exit 1 44 | -------------------------------------------------------------------------------- /etc/testing/circle/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euxo pipefail 4 | 5 | ARCH=amd64 6 | if [ "$(uname -m)" = "aarch64" ]; then ARCH=arm64; fi 7 | 8 | mkdir -p cached-deps 9 | 10 | # Install helm 11 | if [ ! -f cached-deps/helm ]; then 12 | HELM_VERSION=3.5.4 13 | curl -L https://get.helm.sh/helm-v${HELM_VERSION}-linux-${ARCH}.tar.gz \ 14 | | tar xzf - linux-${ARCH}/helm 15 | mv ./linux-${ARCH}/helm cached-deps/helm 16 | fi 17 | 18 | # Install kubectl 19 | # To get the latest kubectl version: 20 | # curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt 21 | if [ ! -f cached-deps/kubectl ] ; then 22 | KUBECTL_VERSION="v$(jq -r .kubernetes version.json)" 23 | curl -L -o kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \ 24 | chmod +x ./kubectl 25 | mv ./kubectl cached-deps/kubectl 26 | fi 27 | 28 | # Install minikube 29 | # To get the latest minikube version: 30 | # curl https://api.github.com/repos/kubernetes/minikube/releases | jq -r .[].tag_name | sort -V | tail -n1 31 | if [ ! -f cached-deps/minikube ] ; then 32 | MINIKUBE_VERSION="v$(jq -r .minikube version.json)" 33 | curl -L -o minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64 && \ 34 | chmod +x ./minikube 35 | mv ./minikube cached-deps/minikube 36 | fi 37 | 38 | export PACHYDERM_VERSION="$(jq -r .pachyderm version.json)" 39 | 40 | # Install Pachyderm 41 | curl -o /tmp/pachctl.deb -L https://github.com/pachyderm/pachyderm/releases/download/v${PACHYDERM_VERSION}/pachctl_${PACHYDERM_VERSION}_amd64.deb 42 | sudo dpkg -i /tmp/pachctl.deb 43 | -------------------------------------------------------------------------------- /etc/testing/circle/kube_debug.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "=== TEST FAILED OR TIMED OUT, DUMPING DEBUG INFO ===" 4 | 5 | export PATH=$(pwd):$(pwd)/cached-deps:$GOPATH/bin:$PATH 6 | 7 | # TODO: Extend this to show kubectl describe output for failed pods, this will 8 | # probably show why things are hanging. 9 | 10 | # SC2016 blocks variables in single-quoted strings, but these are 'eval'ed below 11 | # shellcheck disable=SC2016 12 | cmds=( 13 | 'rm ~/.pachyderm/config.json' 14 | 'pachctl version' 15 | 'pachctl list repo' 16 | 'pachctl list repo --raw | jq -r ".repo.name" | while read r; do 17 | echo "---"; 18 | pachctl inspect repo "${r}"; 19 | echo "Commits:"; 20 | pachctl list commit "${r}"; 21 | echo; 22 | done' 23 | 'pachctl list pipeline' 24 | 'pachctl list job' 25 | 'kubectl version' 26 | 'kubectl get all --all-namespaces' 27 | 'kubectl describe pod -l suite=pachyderm,app=pachd' 28 | 'kubectl describe pod -l suite=pachyderm,app=etcd' 29 | # Set --tail b/c by default 'kubectl logs' only outputs 10 lines if -l is set 30 | 'kubectl logs --tail=1500 -l suite=pachyderm,app=pachd' 31 | 'kubectl logs --tail=1500 -l suite=pachyderm,app=pachd --previous # if pachd restarted' 32 | 'sudo dmesg | tail -n 40' 33 | 'minikube logs | tail -n 100' 34 | 'top -b -n 1 | head -n 40' 35 | 'df -h' 36 | ) 37 | for c in "${cmds[@]}"; do 38 | echo "======================================================================" 39 | echo "${c}" 40 | echo "----------------------------------------------------------------------" 41 | eval "${c}" 42 | done 43 | echo "======================================================================" 44 | -------------------------------------------------------------------------------- /examples/jupyter/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7.3-slim-stretch 2 | ADD merge.py /merge.py 3 | -------------------------------------------------------------------------------- /examples/jupyter/README.md: -------------------------------------------------------------------------------- 1 | # Jupyter Notebook using Versioned Pachyderm Data 2 | 3 | In this example, we'll use a combination of Jupyter notebooks, Pandas, and Pachyderm to analyze Citi Bike sales data. Make sure you have these dependencies installed: 4 | 5 | * Pandas 6 | * Matplotlib 7 | * Jupyter 8 | 9 | Then point your Jupyter notebook to `investigate-unexpected-sales.ipynb` in this directory. 10 | 11 | Our challenge is to figure out why there was a sharp drop in sales on 7/30/16 and 7/31/16: 12 | 13 |  14 | 15 | This analysis will confirm that, on the days in questions, there was a 70%+ chance of rain, and this is likely the reason for poor bike sharing sales. Mystery solved! Here is the graph that we created: 16 | 17 |  18 | -------------------------------------------------------------------------------- /examples/jupyter/final_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/examples/jupyter/final_graph.png -------------------------------------------------------------------------------- /examples/jupyter/jupyter.json: -------------------------------------------------------------------------------- 1 | { 2 | "pipeline": { 3 | "name": "jupyter" 4 | }, 5 | "transform": { 6 | "image": "pachyderm/pachyderm_jupyter", 7 | "cmd": [ "sh" ], 8 | "stdin": [ 9 | "/opt/conda/bin/jupyter notebook" 10 | ] 11 | }, 12 | "input": { 13 | "cross": [ 14 | { 15 | "pfs": { 16 | "repo": "trips", 17 | "glob": "/" 18 | } 19 | }, 20 | { 21 | "pfs": { 22 | "repo": "weather", 23 | "glob": "/" 24 | } 25 | } 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /examples/jupyter/merge.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import json 4 | import datetime 5 | 6 | PRICE = 5 7 | 8 | 9 | def main(): 10 | try: 11 | weather_filenames = os.listdir("/pfs/weather") 12 | except FileNotFoundError: 13 | weather_filenames = [] 14 | 15 | with open("/pfs/out/data.csv", "w") as out_file: 16 | writer = csv.writer(out_file) 17 | 18 | for weather_filename in weather_filenames: 19 | dt = datetime.datetime.strptime(weather_filename, "%Y-%m-%d") 20 | trip_filepath = "/pfs/trips/{}-{}".format(dt.month, dt.strftime("%d-%y")) 21 | 22 | if os.path.exists(trip_filepath): 23 | with open( 24 | "/pfs/weather/{}".format(weather_filename), "r" 25 | ) as weather_file: 26 | with open(trip_filepath, "r") as trip_file: 27 | weather_json = json.load(weather_file) 28 | precip = weather_json["daily"]["data"][0]["precipProbability"] 29 | 30 | trip_csv = csv.reader(trip_file) 31 | next(trip_csv) # skip the header row 32 | trips = int(next(trip_csv)[1]) 33 | 34 | writer.writerow( 35 | [dt.strftime("%Y-%m-%d"), precip, trips, trips * PRICE] 36 | ) 37 | 38 | 39 | if __name__ == "__main__": 40 | main() 41 | -------------------------------------------------------------------------------- /examples/jupyter/sales.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/examples/jupyter/sales.png -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-01-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/1/16,39665,30074752,82871,56801553,179756,1316,139 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-02-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/2/16,35835,30110587,111151,56912704,179903,3596,314 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-03-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/3/16,35347,30145934,111116,57023820,180016,4095,169 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-04-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/4/16,34859,30180793,109745,57133565,180158,3826,155 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-05-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/5/16,43736,30224529,92107,57225672,180338,1357,160 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-06-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/6/16,51336,30275865,109240,57334912,180540,1288,153 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-07-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/7/16,48724,30324589,99975,57434887,180725,1255,113 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-08-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/8/16,44400,30368989,91632,57526519,180852,1258,152 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-09-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/9/16,38138,30407127,107359,57633878,180990,2844,145 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-10-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/10/16,41985,30449112,123282,57757160,181172,3280,137 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-11-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/11/16,52853,30501965,122260,57879420,181381,1440,211 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-12-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/12/16,56515,30558480,120414,57999834,181571,1472,192 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-13-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/13/16,55471,30613951,121029,58120863,181748,1510,146 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-14-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/14/16,50345,30664296,108541,58229404,181877,1326,126 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-15-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/15/16,48558,30712854,104701,58334105,182016,1709,173 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-16-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/16/16,38405,30751259,103429,58437534,182144,3281,122 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-17-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/17/16,41346,30792605,120635,58558169,182314,3440,153 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-18-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/18/16,43681,30836286,97216,58655385,182488,1062,154 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-19-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/19/16,54587,30890873,116806,58772191,182624,1498,139 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-20-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/20/16,57213,30948086,128799,58900990,182792,1601,158 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-21-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/21/16,54832,31002918,128476,59029466,182950,1553,161 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-22-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/22/16,49185,31052103,105666,59135132,183072,1701,176 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-23-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/23/16,37522,31089625,101810,59236942,183177,2932,138 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-24-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/24/16,39634,31129259,113680,59350622,183332,2966,114 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-25-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/25/16,35376,31164635,85145,59435767,183473,920,150 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-26-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/26/16,52904,31217539,108783,59544550,183595,1385,168 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-27-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/27/16,54148,31271687,115744,59660294,183731,1403,132 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-28-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/28/16,50134,31321821,98694,59758988,183846,1292,123 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-29-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/29/16,37693,31359514,81355,59840343,183937,1441,162 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-30-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/30/16,27780,31387294,72197,59912540,184039,2242,112 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/7-31-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 7/31/16,27651,31414945,70377,59982917,184157,1641,82 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-01-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/1/16,49395,31464340,101134,60084051,184325,1359,166 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-02-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/2/16,56744,31521084,116187,60200238,184558,1636,181 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-03-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/3/16,56992,31578076,118370,60318608,184766,1827,143 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-04-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/4/16,56587,31634663,118563,60437171,184960,1778,148 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-05-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/5/16,53286,31687949,116534,60553705,185186,2012,185 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-06-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/6/16,44209,31732158,125188,60678893,185436,4159,153 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-07-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/7/16,43631,31775789,116183,60795076,185682,3856,149 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-08-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/8/16,52251,31828040,115847,60910923,185956,1753,184 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-09-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/9/16,56101,31884141,114892,61025815,186223,1701,165 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-10-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/10/16,50100,31934241,101761,61127576,186401,1158,109 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-11-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/11/16,47420,31981661,93639,61221215,186596,1238,152 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-12-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/12/16,44729,32026390,90685,61311900,186771,1467,144 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-13-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/13/16,38903,32065293,100035,61411935,186990,3333,125 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-14-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/14/16,32961,32098254,74055,61485990,187173,2154,91 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-15-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/15/16,49056,32147310,107305,61593295,187394,1417,175 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-16-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/16/16,50411,32197721,103010,61696305,187601,1295,164 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-17-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/17/16,53983,32251704,113583,61809888,187820,1430,138 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-18-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/18/16,53209,32304913,115041,61924929,187998,1459,163 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-19-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/19/16,51658,32356571,110723,62035652,188186,1862,175 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-20-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/20/16,45808,32402379,138864,62174516,188499,4385,167 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-21-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/21/16,35041,32437420,96941,62271457,188725,2664,127 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-22-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/22/16,54201,32491621,118995,62390452,188996,1679,192 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-23-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/23/16,58665,32550286,127012,62517464,189297,1675,170 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-24-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/24/16,58134,32608420,127066,62644530,189577,1597,160 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-25-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/25/16,53480,32661900,111985,62756515,189856,1436,169 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-26-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/26/16,52026,32713926,112005,62868520,190156,1878,197 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-27-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/27/16,45673,32759599,120578,62989098,190480,3767,173 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-28-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/28/16,44639,32804238,121056,63110154,190844,3560,159 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-29-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/29/16,53279,32857517,115101,63225255,191255,1530,174 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-30-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/30/16,57392,32914909,121098,63346353,191713,1510,160 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/8-31-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 8/31/16,57200,32972109,118815,63465168,192380,1320,136 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-01-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/1/16,38174,33010283,77415,63542583,192592,714,104 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-02-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/2/16,52999,33063282,126790,63669373,192814,2314,252 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-03-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/3/16,41629,33104911,117476,63786849,193055,4009,268 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-04-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/4/16,46020,33150931,140347,63927196,193326,5467,185 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-05-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/5/16,45431,33196362,118022,64045218,193677,3778,190 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-06-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/6/16,52432,33248794,108743,64153961,194065,1018,178 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-07-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/7/16,60278,33309072,130391,64284352,194396,1328,199 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-08-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/8/16,61551,33370623,130259,64414611,194674,1511,179 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-09-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/9/16,56804,33427427,118213,64532824,194908,1582,178 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-10-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/10/16,48454,33475881,127252,64660076,195189,3549,229 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-11-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/11/16,52238,33528119,136353,64796429,195499,3572,216 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-12-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/12/16,61083,33589202,127447,64923876,195832,1620,225 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-13-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/13/16,64818,33654020,135366,65059242,196122,1462,198 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-14-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/14/16,58928,33712948,121511,65180753,196388,1369,157 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-15-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/15/16,67312,33780260,140436,65321189,196637,1612,240 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-16-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/16/16,65257,33845517,145454,65466643,196886,2047,225 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-17-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/17/16,59199,33904716,154037,65620680,197163,5117,210 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-18-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/18/16,51685,33956401,131788,65752468,197449,3191,159 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-19-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/19/16,34205,33990606,67180,65819648,197620,555,101 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-20-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/20/16,64881,34055487,135085,65954733,197880,1454,200 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-21-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/21/16,65453,34120940,142356,66097089,198118,1566,182 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-22-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/22/16,67762,34188702,146370,66243459,198363,1681,207 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-23-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/23/16,65499,34254201,142458,66385917,198577,2053,248 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-24-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/24/16,53258,34307459,137355,66523272,198827,3990,212 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-25-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/25/16,53563,34361022,143312,66666584,199198,3871,169 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-26-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/26/16,61120,34422142,128011,66794595,199480,1375,170 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-27-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/27/16,61334,34483476,122347,66916942,199739,1175,150 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-28-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/28/16,59456,34542932,120072,67037014,199989,931,107 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-29-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/29/16,55550,34598482,104364,67141378,200201,808,131 3 | -------------------------------------------------------------------------------- /examples/jupyter/trips_data/9-30-16: -------------------------------------------------------------------------------- 1 | Date,Trips over the past 24-hours (midnight to 11:59pm),Cumulative trips (since launch):,Miles traveled today (midnight to 11:59 pm),Miles traveled to date:,Total Annual Members (All Time),24-Hour Passes Purchased (midnight to 11:59 pm),3-Day Passes Purchased (midnight to 11:59 pm) 2 | 9/30/16,21368,34619850,45194,67186572,200447,175,20 3 | -------------------------------------------------------------------------------- /examples/opencv/README.md: -------------------------------------------------------------------------------- 1 | # Distributed Image Processing with OpenCV and Python Pachyderm 2 | 3 | This is a reproduction of Pachyderm's opencv example in python. A walkthrough is available in the Pachyderm *[docs](https://docs.pachyderm.io/en/latest/getting_started/beginner_tutorial.html)* 4 | 5 | This example showcases the python-pachyderm analogs of common `pachctl` commands, such as creating repos and pipelines or getting a file. 6 | 7 | **Prerequisites:** 8 | - A workspace on [Pachyderm Hub](https://docs.pachyderm.com/latest/hub/hub_getting_started/) (recommended) or Pachyderm running [locally](https://docs.pachyderm.com/latest/getting_started/local_installation/) 9 | - [python-pachyderm](https://pypi.org/project/python-pachyderm/) installed 10 | 11 | **To run:** 12 | ```shell 13 | $ python opencv.py 14 | ``` -------------------------------------------------------------------------------- /examples/opencv/edges/main.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | from matplotlib import pyplot as plt 4 | import os 5 | 6 | 7 | # make_edges reads an image from /pfs/images and outputs the result of running 8 | # edge detection on that image to /pfs/out. Note that /pfs/images and 9 | # /pfs/out are special directories that Pachyderm injects into the container. 10 | def make_edges(image): 11 | img = cv2.imread(image) 12 | tail = os.path.split(image)[1] 13 | edges = cv2.Canny(img, 100, 200) 14 | plt.imsave( 15 | os.path.join("/pfs/out", os.path.splitext(tail)[0] + ".png"), edges, cmap="gray" 16 | ) 17 | 18 | 19 | # walk /pfs/images and call make_edges on every file found 20 | for dirpath, dirs, files in os.walk("/pfs/images"): 21 | for file in files: 22 | make_edges(os.path.join(dirpath, file)) 23 | -------------------------------------------------------------------------------- /examples/opencv/edges/requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib>=3.1.2 2 | opencv-python==4.5.1.48 3 | -------------------------------------------------------------------------------- /examples/opencv/images/46Q8nDz.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/examples/opencv/images/46Q8nDz.jpg -------------------------------------------------------------------------------- /examples/opencv/images/8MN9Kg0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/examples/opencv/images/8MN9Kg0.jpg -------------------------------------------------------------------------------- /examples/opencv/images/g2QnNqa.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/examples/opencv/images/g2QnNqa.jpg -------------------------------------------------------------------------------- /examples/opencv/opencv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # This is a reproduction of pachyderm's opencv example in python. A 4 | # walkthrough is available in the pachyderm docs: 5 | # https://docs.pachyderm.io/en/latest/getting_started/beginner_tutorial.html 6 | # 7 | # It makes heavy use of python_pachyderm's higher-level utility functionality 8 | # (`create_python_pipeline`, `put_files`), as well as more run-of-the-mill 9 | # functionality (`create_repo`, `create_pipeline`). 10 | 11 | import os 12 | import shutil 13 | import tempfile 14 | import python_pachyderm 15 | from python_pachyderm.service import pps_proto 16 | 17 | 18 | def relpath(path): 19 | return os.path.join(os.path.dirname(os.path.abspath(__file__)), path) 20 | 21 | 22 | def main(): 23 | # Connects to a pachyderm cluster on the default host:port 24 | # (`localhost:30650`). This will work for certain environments (e.g. k8s 25 | # running on docker for mac), as well as when port forwarding is being 26 | # used. For other setups, you'll want one of the alternatives: 27 | # 1) To connect to pachyderm when this script is running inside the 28 | # cluster, use `python_pachyderm.Client.new_in_cluster()`. 29 | # 2) To connect to pachyderm via a pachd address, use 30 | # `python_pachyderm.Client.new_from_pachd_address`. 31 | # 3) To explicitly set the host and port, pass parameters into 32 | # `python_pachyderm.Client()`. 33 | client = python_pachyderm.Client() 34 | 35 | # Create a repo called images 36 | client.create_repo("images") 37 | 38 | # Create the edges pipeline (and the edges repo automatically). This 39 | # pipeline runs when data is committed to the images repo, as indicated 40 | # by the input field. 41 | client.create_pipeline( 42 | "edges", 43 | transform=pps_proto.Transform( 44 | cmd=["python3", "/edges.py"], 45 | image="pachyderm/opencv", 46 | ), 47 | input=pps_proto.Input(pfs=pps_proto.PFSInput(repo="images", glob="/*")), 48 | ) 49 | 50 | # Create the montage pipeline (and the montage repo automatically). This 51 | # pipeline runs when data is committed to either the images repo or edges 52 | # repo, as indicated by the input field. 53 | client.create_pipeline( 54 | "montage", 55 | transform=pps_proto.Transform( 56 | cmd=["sh"], 57 | image="v4tech/imagemagick", 58 | stdin=[ 59 | "montage -shadow -background SkyBlue -geometry 300x300+2+2 $(find /pfs ! -name .env -type f | sort) /pfs/out/montage.png" 60 | ], 61 | ), 62 | input=pps_proto.Input( 63 | cross=[ 64 | pps_proto.Input(pfs=pps_proto.PFSInput(glob="/", repo="images")), 65 | pps_proto.Input(pfs=pps_proto.PFSInput(glob="/", repo="edges")), 66 | ] 67 | ), 68 | ) 69 | 70 | with client.commit("images", "master") as commit: 71 | # Add some images, recursively inserting content from the images 72 | # directory. Alternatively, you could use `client.put_file_url` or 73 | # `client_put_file_bytes`. 74 | python_pachyderm.put_files(client, relpath("images"), commit, "/") 75 | 76 | # Wait for the commit (and its downstream commits) to finish 77 | client.wait_commit(("montage", "master")) 78 | 79 | # Get the montage 80 | source_file = client.get_file(("montage", "master"), "/montage.png") 81 | with tempfile.NamedTemporaryFile(suffix="montage.png", delete=False) as dest_file: 82 | shutil.copyfileobj(source_file, dest_file) 83 | print("montage written to {}".format(dest_file.name)) 84 | 85 | 86 | if __name__ == "__main__": 87 | main() 88 | -------------------------------------------------------------------------------- /examples/spout/README.md: -------------------------------------------------------------------------------- 1 | > INFO: python-pachyderm 7.x introduces profound architectual changes to the product. As a result, our examples are kept in two separate branches: 2 | > - Branch Master: Examples using python-pachyderm 7.x - https://github.com/pachyderm/python-pachyderm/tree/master/examples 3 | > - Branch v6.x: Examples using python-pachyderm 6.x - https://github.com/pachyderm/python-pachyderm/tree/v6.x/examples 4 | 5 | # Spout Pipelines (Python) 6 | 7 | This example is a reproduction of the Spouts101 example from the Pachyderm repo. This example uses the python-pachyderm analogs for creating pipelines (`spout.py`), which was done using `pachctl` commands in the Spouts101 example. For more information on Spouts, a full walkthrough of the original example, or the pipelines' user code, go *[here](https://github.com/pachyderm/pachyderm/tree/master/examples/spouts/spout101)*. 8 | 9 | **Prerequisites:** 10 | - A workspace on [Pachyderm Hub](https://docs.pachyderm.com/latest/hub/hub_getting_started/) (recommended) or Pachyderm running [locally](https://docs.pachyderm.com/latest/getting_started/local_installation/) 11 | - [python-pachyderm](https://pypi.org/project/python-pachyderm/) installed 12 | 13 | **To run:** 14 | ```shell 15 | $ python spout.py 16 | ``` -------------------------------------------------------------------------------- /examples/spout/spout.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import python_pachyderm 5 | from python_pachyderm.service import pps_proto 6 | 7 | 8 | def relpath(path): 9 | return os.path.join(os.path.dirname(os.path.abspath(__file__)), path) 10 | 11 | 12 | def main(): 13 | client = python_pachyderm.Client() 14 | 15 | client.create_pipeline( 16 | pipeline_name="spout", 17 | transform=pps_proto.Transform( 18 | cmd=["python3", "consumer/main.py"], 19 | image="pachyderm/example-spout101:2.0.0-beta.5", 20 | ), 21 | spout=pps_proto.Spout(), 22 | description="A spout pipeline that emulates the reception of data from an external source", 23 | ) 24 | 25 | client.create_pipeline( 26 | pipeline_name="processor", 27 | transform=pps_proto.Transform( 28 | cmd=["python3", "processor/main.py"], 29 | image="pachyderm/example-spout101:2.0.0-beta.5", 30 | ), 31 | input=pps_proto.Input( 32 | pfs=pps_proto.PFSInput(repo="spout", branch="master", glob="/*") 33 | ), 34 | description="A pipeline that sorts 1KB vs 2KB files", 35 | ) 36 | 37 | client.create_pipeline( 38 | pipeline_name="reducer", 39 | transform=pps_proto.Transform( 40 | cmd=["bash"], 41 | stdin=[ 42 | "set -x", 43 | "FILES=/pfs/processor/*/*", 44 | "for f in $FILES", 45 | "do", 46 | "directory=`dirname $f`", 47 | "out=`basename $directory`", 48 | "cat $f >> /pfs/out/${out}.txt", 49 | "done", 50 | ], 51 | ), 52 | input=pps_proto.Input( 53 | pfs=pps_proto.PFSInput(repo="processor", branch="master", glob="/*") 54 | ), 55 | description="A pipeline that reduces 1K/ and 2K/ directories", 56 | ) 57 | 58 | 59 | if __name__ == "__main__": 60 | main() 61 | -------------------------------------------------------------------------------- /poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true 3 | 4 | [repositories.testpypi] 5 | url = "https://test.pypi.org/legacy/" 6 | -------------------------------------------------------------------------------- /proto/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6-slim 2 | LABEL maintainer="jdoliner@pachyderm.io" 3 | 4 | RUN pip3 install grpcio==1.38.0 grpcio-tools==1.38.0 5 | RUN apt-get update && apt-get install -y gcc 6 | RUN pip3 install "betterproto[compiler]"==2.0.0b3 7 | 8 | ADD run /bin 9 | ENTRYPOINT ["/bin/run"] 10 | WORKDIR /work 11 | -------------------------------------------------------------------------------- /proto/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # will extract from stdin and to ./proto/pachyderm/src 5 | tar xf /dev/stdin 6 | 7 | # VERSION matches the MajorVersion of pachyderm 8 | OUTDIR="python_pachyderm/proto/v${VERSION}" 9 | 10 | mkdir -p ${OUTDIR} 11 | mv ./proto/pachyderm/src/* ${OUTDIR} 12 | 13 | # Remove protobuf extensions, that are Go specific. 14 | for i in $(find ${OUTDIR} -name "*.proto"); do 15 | # remove gogoproto 16 | sed -i 's/import.*gogo.proto.*\;//' "${i}" 17 | sed -i 's/\[.*gogoproto.*\]//' "${i}" 18 | sed -i 's/.*gogoproto.*//' "${i}" 19 | 20 | # remove the protoextensions/log.proto 21 | sed -i 's/import.*protoextensions\/log.proto.*\;//' ${i} 22 | sed -i 's/\[.*log.*\]//' ${i} 23 | done 24 | 25 | # fix imports to be relative to OUTDIR 26 | for i in $(find ${OUTDIR} -name "*.proto"); do 27 | perl -pi -e "s/import \"((?!google).*)\"/import \"python_pachyderm\/proto\/v${VERSION}\/\$1\"/" ${i} 28 | done 29 | 30 | # generate protobuf and gRPC code from proto files 31 | # should result in {}_pb2.py and {}_pb2_grpc.py per proto 32 | find ${OUTDIR} -name '*.proto' | xargs python3 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. 33 | 34 | # Clean up 35 | find ${OUTDIR} -name '*.proto' | xargs rm 36 | find ${OUTDIR} -type d -exec touch {}/__init__.py \; 37 | 38 | tar cf - ${OUTDIR} 39 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "python-pachyderm" 3 | version = "7.6.0" 4 | description = "Python Pachyderm Client" 5 | authors = ["Pachyderm Integrations "] 6 | include = ["version.json"] 7 | license = "Apache 2.0" 8 | documentation = "https://python-pachyderm.readthedocs.io/en/stable/" 9 | readme = 'README.md' 10 | repository = "https://github.com/pachyderm/python-pachyderm" 11 | keywords = ["pachyderm"] 12 | classifiers = [ 13 | "Intended Audience :: Developers", 14 | "License :: OSI Approved :: Apache Software License", 15 | "Programming Language :: Python :: 3", 16 | "Programming Language :: Python :: 3.8", 17 | "Programming Language :: Python :: 3.9", 18 | "Programming Language :: Python :: 3.10", 19 | "Programming Language :: Pythohn :: 3.11", 20 | "Programming Language :: Python :: Implementation :: CPython", 21 | ] 22 | 23 | 24 | [tool.poetry.dependencies] 25 | python = ">=3.8,<4.0" 26 | grpcio = ">=1.54.2" 27 | grpcio-health-checking = ">=1.38.0" 28 | grpc-interceptor = "^0.13.0" 29 | importlib-metadata = { version = ">1.5.1", python = "<3.8" } 30 | protobuf = ">=3.17.1,<4.0.0" 31 | python-dotenv = ">=1.0.0" 32 | 33 | [tool.poetry.dev-dependencies] 34 | black = ">=22.6.0" 35 | certifi = ">=2019.11.28" 36 | flake8 = ">=3.9.1" 37 | myst-parser = "0.15.2" 38 | numpydoc = ">=1.1.0" 39 | pre-commit = ">=2.12.1" 40 | pytest = ">=5.3.4" 41 | pytest-mock = ">=3.6.1" 42 | pytest-timeout = ">=1.4.2" 43 | Sphinx = "4.3.0" 44 | sphinx-rtd-theme = "1.0.0" 45 | tox = ">=3.23.1" 46 | 47 | [tool.black] 48 | force-exclude = ''' 49 | /( 50 | proto 51 | | src/python_pachyderm/proto 52 | )/ 53 | ''' 54 | 55 | [build-system] 56 | requires = ["poetry-core>=1.0.0"] 57 | build-backend = "poetry.core.masonry.api" 58 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | # make flake8 play nicely with black 3 | extend-ignore = E203, E266, E501, W503, F403, F401 4 | # 88 conforms with black's default line-length 5 | max-line-length = 88 6 | max-doc-length = 80 7 | extend-exclude = 8 | .*/*, 9 | venv/*, 10 | _build/*, 11 | build-out/*, 12 | build/*, 13 | dist/*, 14 | docs/*, 15 | proto/*, 16 | src/python_pachyderm/proto/* 17 | src/python_pachyderm/experimental/proto/* 18 | get-poetry.py 19 | 20 | [aliases] 21 | test = pytest 22 | -------------------------------------------------------------------------------- /src/python_pachyderm/__init__.py: -------------------------------------------------------------------------------- 1 | import string as _string 2 | import importlib as _importlib 3 | import enum as _enum 4 | from google.protobuf.internal.enum_type_wrapper import ( 5 | EnumTypeWrapper as _EnumTypeWrapper, 6 | ) 7 | 8 | from .mixin.pfs import PFSFile, ModifyFileClient 9 | from .client import Client, ConfigError, BadClusterDeploymentID 10 | from .datum_batching import batch_all_datums 11 | from .util import ( 12 | put_files, 13 | parse_json_pipeline_spec, 14 | parse_dict_pipeline_spec, 15 | ) 16 | from grpc import RpcError 17 | 18 | # Python version compatibility. 19 | try: 20 | # >= 3.8 21 | import importlib.metadata as metadata 22 | except ImportError: 23 | # < 3.8 24 | import importlib_metadata as metadata # type: ignore 25 | 26 | 27 | __pdoc__ = {"proto": False} 28 | 29 | __all__ = [ 30 | "Client", 31 | "RpcError", 32 | "put_files", 33 | "PFSFile", 34 | "ModifyFileClient", 35 | "parse_json_pipeline_spec", 36 | "parse_dict_pipeline_spec", 37 | "ConfigError", 38 | "BadClusterDeploymentID", 39 | "batch_all_datums", 40 | ] 41 | 42 | __version__ = "" 43 | try: 44 | __version__ = metadata.version(__name__) # type: ignore 45 | except FileNotFoundError: 46 | pass 47 | 48 | 49 | def _import_protos(path): 50 | """ 51 | Imports items selectively from the auto-generated proto package. 52 | 53 | Importing is done dynamically so we can selectively blacklist items. We 54 | also dynamically define enums that build on top of the auto-generated 55 | protobuf enums, to create a more pythonic API. 56 | 57 | More broadly, the dark magic in here allows us to maintain parity with 58 | Pachyderm protobufs when they change, without having to maintain a manual 59 | mapping of protobuf to python_pachyderm values. 60 | """ 61 | 62 | g = globals() 63 | module = _importlib.import_module(path) 64 | uppercase_letters = set(_string.ascii_uppercase) 65 | lowercase_letters = set(_string.ascii_lowercase) 66 | 67 | def import_item(g, module, key): 68 | value = getattr(module, key) 69 | 70 | if isinstance(value, _EnumTypeWrapper): 71 | # Dynamically define an enum class that is exported 72 | g[key] = _enum.Enum(key, {k: v for (k, v) in value.items()}) 73 | else: 74 | # Export the value 75 | g[key] = value 76 | 77 | __all__.append(key) 78 | 79 | def should_import(key): 80 | return key[0] in uppercase_letters and any( 81 | c in lowercase_letters for c in key[1:] 82 | ) 83 | 84 | for key in dir(module): 85 | if should_import(key): 86 | import_item(g, module, key) 87 | elif key.startswith("google_dot_protobuf_dot_"): 88 | sub_module = getattr(module, key) 89 | for key in dir(sub_module): 90 | if should_import(key): 91 | import_item(g, sub_module, key) 92 | 93 | 94 | # _import_protos("python_pachyderm.proto.v2.pfs.pfs_pb2") 95 | # _import_protos("python_pachyderm.proto.v2.pps.pps_pb2") 96 | # _import_protos("python_pachyderm.proto.v2.version.versionpb.version_pb2") 97 | # _import_protos("python_pachyderm.proto.v2.transaction.transaction_pb2") 98 | # _import_protos("python_pachyderm.proto.v2.admin.admin_pb2") 99 | # _import_protos("python_pachyderm.proto.v2.auth.auth_pb2") 100 | # _import_protos("python_pachyderm.proto.v2.enterprise.enterprise_pb2") 101 | # _import_protos("python_pachyderm.proto.v2.license.license_pb2") 102 | # _import_protos("python_pachyderm.proto.v2.identity.identity_pb2") 103 | -------------------------------------------------------------------------------- /src/python_pachyderm/datum_batching.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from typing import Callable 3 | 4 | from . import Client 5 | 6 | PIPELINE_FUNC = Callable[..., None] 7 | 8 | 9 | def batch_all_datums(user_code: PIPELINE_FUNC) -> PIPELINE_FUNC: 10 | """A decorator that will repeatedly call the wrapped function until 11 | all datums have been processed. Before calling the wrapped function, 12 | this decorator will call the NextDatum endpoint within the worker 13 | and set any environment variables specified by the worker. 14 | 15 | Any expections raised during the execution of the wrapped function 16 | will be reported back to the worker. See the pachyderm documentation 17 | for more information on how the datum batching feature works. 18 | 19 | Note: This can only be used within a Pachyderm worker. 20 | 21 | Examples 22 | -------- 23 | >>> from python_pachyderm import batch_all_datums 24 | >>> 25 | >>> @batch_all_datums 26 | >>> def pipeline(): 27 | >>> # process datums 28 | >>> pass 29 | >>> 30 | >>> if __name__ == '__main__': 31 | >>> # Perform an expensive computation here before 32 | >>> # entering your datum processing function 33 | >>> # i.e. initializing a model. 34 | >>> pipeline() 35 | """ 36 | 37 | @wraps(user_code) 38 | def wrapper(*args, **kwargs) -> None: 39 | worker = Client().worker 40 | while True: 41 | with worker.batch_datum(): 42 | user_code(*args, **kwargs) 43 | 44 | return wrapper 45 | -------------------------------------------------------------------------------- /src/python_pachyderm/errors.py: -------------------------------------------------------------------------------- 1 | """ Errors that can be raised by this library. """ 2 | from typing import Union 3 | 4 | from grpc import RpcError 5 | from grpc._channel import _InactiveRpcError 6 | 7 | 8 | class AuthServiceNotActivated(ConnectionError): 9 | """If the auth service is not activated but is required.""" 10 | 11 | @classmethod 12 | def try_from(cls, error: RpcError) -> Union["AuthServiceNotActivated", RpcError]: 13 | if isinstance(error, _InactiveRpcError): 14 | details = error.details() 15 | if "the auth service is not activated" in details: 16 | return cls(details) 17 | return error 18 | 19 | 20 | class InvalidTransactionOperation(RuntimeError): 21 | """Error triggered when an invalid operation (i.e. file write) 22 | is called when inside a transaction. 23 | """ 24 | 25 | def __init__(self): 26 | super().__init__( 27 | "File operations are not permitted within a pachyderm transaction." 28 | ) 29 | -------------------------------------------------------------------------------- /src/python_pachyderm/interceptor.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | from typing import Any, Callable, List, Optional, Tuple 3 | 4 | import grpc 5 | from grpc_interceptor import ClientCallDetails, ClientInterceptor 6 | 7 | MetadataType = List[Tuple[str, str]] 8 | 9 | 10 | class MetadataClientInterceptor(ClientInterceptor): 11 | def __init__(self, metadata: MetadataType): 12 | self.metadata = metadata 13 | 14 | def intercept( 15 | self, method: Callable, request: Any, call_details: ClientCallDetails 16 | ): 17 | call_details_metadata = list(call_details.metadata or []) 18 | call_details_metadata.extend(self.metadata) 19 | new_details = ClientCallDetails( 20 | compression=call_details.compression, 21 | credentials=call_details.credentials, 22 | metadata=call_details_metadata, 23 | method=call_details.method, 24 | timeout=call_details.timeout, 25 | wait_for_ready=call_details.wait_for_ready, 26 | ) 27 | 28 | future = method(request, new_details) 29 | future.add_done_callback(_check_connection_error) 30 | return future 31 | 32 | 33 | def _check_connection_error(grpc_future: grpc.Future): 34 | """Callback function that checks if a gRPC.Future experienced a 35 | ConnectionError and attempt to sanitize the error message for the user. 36 | """ 37 | error: Optional[grpc.Call] = grpc_future.exception() 38 | if error is not None: 39 | unable_to_connect = "failed to connect to all addresses" in error.details() 40 | if error.code() == grpc.StatusCode.UNAVAILABLE and unable_to_connect: 41 | error_message = "Could not connect to pachyderm instance\n" 42 | if "PACHD_PEER_SERVICE_HOST" in environ: 43 | error_message += ( 44 | "\tPACHD_PEER_SERVICE_HOST is detected. " 45 | "Please use Client.new_in_cluster() when using" 46 | " python_pachyderm within the a pipeline. " 47 | ) 48 | raise ConnectionError(error_message) from error 49 | raise error 50 | -------------------------------------------------------------------------------- /src/python_pachyderm/mixin/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exposes a mixin for each pachyderm service. These mixins should not be used 3 | directly; instead, you should use ``python_pachyderm.Client()``. The mixins 4 | exist exclusively in order to provide better code organization (because we have 5 | several mixins, rather than one giant :class:`.Client` class.) 6 | """ 7 | -------------------------------------------------------------------------------- /src/python_pachyderm/mixin/admin.py: -------------------------------------------------------------------------------- 1 | import grpc 2 | from google.protobuf import empty_pb2 3 | 4 | from python_pachyderm.proto.v2.admin import admin_pb2, admin_pb2_grpc 5 | 6 | 7 | class AdminMixin: 8 | """A mixin for admin-related functionality.""" 9 | 10 | _channel: grpc.Channel 11 | 12 | def __init__(self): 13 | self.__stub = admin_pb2_grpc.APIStub(self._channel) 14 | super().__init__() 15 | 16 | # TODO: This method should auto-populate it's message with information about 17 | # the version of this package that is making the call. This is to allow 18 | # the cluster to emit warnings about incompatible versions. 19 | # This is not currently feasible to implement until we have better coupling 20 | # with the versions of the core product. 21 | def inspect_cluster(self) -> admin_pb2.ClusterInfo: 22 | """Inspects a cluster. 23 | 24 | Returns 25 | ------- 26 | admin_pb2.ClusterInfo 27 | A protobuf object with info on the cluster. 28 | """ 29 | message = empty_pb2.Empty() 30 | return self.__stub.InspectCluster(message) 31 | -------------------------------------------------------------------------------- /src/python_pachyderm/mixin/enterprise.py: -------------------------------------------------------------------------------- 1 | import grpc 2 | 3 | from python_pachyderm.proto.v2.enterprise import enterprise_pb2, enterprise_pb2_grpc 4 | 5 | 6 | class EnterpriseMixin: 7 | """A mixin for enterprise-related functionality.""" 8 | 9 | _channel: grpc.Channel 10 | 11 | def __init__(self): 12 | self.__stub = enterprise_pb2_grpc.APIStub(self._channel) 13 | super().__init__() 14 | 15 | def activate_enterprise(self, license_server: str, id: str, secret: str) -> None: 16 | """Activates enterprise by registering with a license server. 17 | 18 | Parameters 19 | ---------- 20 | license_server : str 21 | The Pachyderm Enterprise Server to register with. 22 | id : str 23 | The unique ID for this cluster. 24 | secret : str 25 | The secret for registering this cluster. 26 | """ 27 | message = enterprise_pb2.ActivateRequest( 28 | license_server=license_server, 29 | id=id, 30 | secret=secret, 31 | ) 32 | self.__stub.Activate(message) 33 | 34 | def get_enterprise_state(self) -> enterprise_pb2.GetStateResponse: 35 | """Gets the current enterprise state of the cluster. 36 | 37 | Returns 38 | ------- 39 | enterprise_pb2.GetStateResponse 40 | A protobuf object that returns a state enum, info on the token, 41 | and an empty activation code. 42 | """ 43 | message = enterprise_pb2.GetStateRequest() 44 | return self.__stub.GetState(message) 45 | 46 | def deactivate_enterprise(self) -> None: 47 | """Deactivates enterprise.""" 48 | message = enterprise_pb2.DeactivateRequest() 49 | self.__stub.Deactivate(message) 50 | 51 | def get_activation_code(self) -> enterprise_pb2.GetActivationCodeResponse: 52 | """Returns the enterprise code used to activate Pachyderm Enterprise in 53 | this cluster. 54 | 55 | Returns 56 | ------- 57 | enterprise_pb2.GetActivationCodeResponse 58 | A protobuf object that returns a state enum, info on the token, 59 | and the activation code. 60 | """ 61 | message = enterprise_pb2.GetActivationCodeRequest() 62 | return self.__stub.GetActivationCode(message) 63 | 64 | def pause_enterprise(self) -> None: 65 | """Pauses the cluster.""" 66 | message = enterprise_pb2.PauseRequest() 67 | self.__stub.Pause(message) 68 | 69 | def unpause_enterprise(self) -> None: 70 | """Unpauses the cluster.""" 71 | message = enterprise_pb2.UnpauseRequest() 72 | self.__stub.Unpause(message) 73 | 74 | def get_pause_status(self) -> enterprise_pb2.PauseStatusResponse: 75 | """Gets the pause status of the cluster. 76 | 77 | Returns 78 | ------- 79 | enterprise_pb2.PauseStatusResponse 80 | A protobuf object that returns a status enum. 81 | """ 82 | message = enterprise_pb2.PauseStatusRequest() 83 | return self.__stub.PauseStatus(message) 84 | -------------------------------------------------------------------------------- /src/python_pachyderm/mixin/health.py: -------------------------------------------------------------------------------- 1 | import grpc 2 | from grpc_health.v1 import health_pb2, health_pb2_grpc 3 | 4 | 5 | class HealthMixin: 6 | """A mixin for health-related functionality.""" 7 | 8 | _channel: grpc.Channel 9 | 10 | def __init__(self): 11 | self.__stub = health_pb2_grpc.HealthStub(self._channel) 12 | super().__init__() 13 | 14 | def health_check(self) -> health_pb2.HealthCheckResponse: 15 | """Returns a health check indicating if the server can handle 16 | RPCs. 17 | 18 | Returns 19 | ------- 20 | health_pb2.HealthCheckResponse 21 | A protobuf object with a status enum indicating server health. 22 | """ 23 | message = health_pb2.HealthCheckRequest() 24 | return self.__stub.Check(message) 25 | -------------------------------------------------------------------------------- /src/python_pachyderm/mixin/version.py: -------------------------------------------------------------------------------- 1 | import grpc 2 | from google.protobuf import empty_pb2 3 | 4 | from python_pachyderm.proto.v2.version.versionpb import version_pb2, version_pb2_grpc 5 | 6 | 7 | class VersionMixin: 8 | """A mixin for version-related functionality.""" 9 | 10 | _channel: grpc.Channel 11 | 12 | def __init__(self): 13 | self.__stub = version_pb2_grpc.APIStub(self._channel) 14 | super().__init__() 15 | 16 | def get_remote_version(self) -> version_pb2.Version: 17 | """Gets version of Pachyderm server. 18 | 19 | Returns 20 | ------- 21 | version_pb2.Version 22 | A protobuf object with info on the pachd version. 23 | """ 24 | message = empty_pb2.Empty() 25 | return self.__stub.GetVersion(message) 26 | -------------------------------------------------------------------------------- /src/python_pachyderm/mixin/worker.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from typing import ContextManager, Optional 3 | 4 | import grpc 5 | from dotenv import load_dotenv 6 | 7 | from python_pachyderm.proto.v2.worker import worker_pb2, worker_pb2_grpc 8 | 9 | 10 | class WorkerMixin: 11 | """A mixin for worker binary-related functionality.""" 12 | 13 | _dotenv_path = "/pfs/.env" 14 | __error: Optional[str] = None # used by batch_datum 15 | 16 | def __init__(self, channel: grpc.Channel): 17 | """ 18 | Note: This API stub must have an open grpc channel work the worker 19 | binary. The ``Client`` object should automatically do this for 20 | the user. 21 | """ 22 | self.__stub = worker_pb2_grpc.WorkerStub(channel) 23 | super().__init__() 24 | 25 | def next_datum(self, *, error: str = "") -> worker_pb2.NextDatumResponse: 26 | """Calls the NextDatum endpoint within the worker to step forward during 27 | datum batching. 28 | 29 | Parameters 30 | ---------- 31 | error : str, optional 32 | An error message to report to the worker binary indicating the 33 | current datum could not be processed. 34 | """ 35 | message = worker_pb2.NextDatumRequest(error=error) 36 | return self.__stub.NextDatum(message) 37 | 38 | @contextmanager 39 | def batch_datum(self) -> ContextManager: 40 | """A ContextManager that, when entered, calls the NextDatum 41 | endpoint within the worker to step forward during datum batching. 42 | This context manager will also prepare the environment for the user 43 | and report errors that occur back to the worker. 44 | 45 | This context manager expects to be called within an infinite while 46 | loop -- see the examples section. This context can only be entered 47 | from within a Pachyderm worker and the worker will terminate your 48 | code when all datums have been processed. 49 | 50 | Note: The API stub must have an open gRPC channel with the worker 51 | for NextDatum to function correctly. The ``Client`` object 52 | should automatically do this for the user. 53 | 54 | Examples 55 | -------- 56 | >>> from python_pachyderm import Client 57 | >>> 58 | >>> worker = Client().worker 59 | >>> 60 | >>> # Perform an expensive computation here before 61 | >>> # you being processing your datums 62 | >>> # i.e. initializing a model. 63 | >>> 64 | >>> while True: 65 | >>> with worker.batch_datum(): 66 | >>> # process datums 67 | >>> pass 68 | """ 69 | self.next_datum(error=self.__error or "") 70 | load_dotenv(self._dotenv_path, override=True) 71 | 72 | self.__error = None 73 | try: 74 | yield 75 | except Exception as error: 76 | self.__error = repr(error) 77 | # TODO: Probably want better logging here than a print statement. 78 | print(f"{self.__error}\nReporting above error to worker.") 79 | -------------------------------------------------------------------------------- /src/python_pachyderm/pfs.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import NamedTuple, Union 3 | 4 | from python_pachyderm.proto.v2.pfs import pfs_pb2 5 | 6 | # copied from pachyderm/pachyderm 7 | valid_branch_re = re.compile(r"^[a-zA-Z0-9_-]+$") 8 | uuid_re = re.compile(r"[0-9a-f]{12}4[0-9a-f]{19}") 9 | 10 | 11 | class Commit(NamedTuple): 12 | """A namedtuple subclass to specify a Commit.""" 13 | 14 | repo: str 15 | branch: str = None 16 | id: str = None 17 | repo_type: str = "user" 18 | project: str = "default" 19 | 20 | def to_pb(self) -> pfs_pb2.Commit: 21 | """Converts itself into a ``pfs_pb2.Commit``.""" 22 | return pfs_pb2.Commit( 23 | id=self.id, 24 | branch=pfs_pb2.Branch( 25 | repo=pfs_pb2.Repo( 26 | name=self.repo, 27 | type=self.repo_type, 28 | project=pfs_pb2.Project(name=self.project), 29 | ), 30 | name=self.branch, 31 | ), 32 | ) 33 | 34 | @staticmethod 35 | def from_pb(commit: pfs_pb2.Commit) -> "Commit": 36 | """Converts a ``pfs_pb2.Commit`` object into a :class:`.Commit` 37 | object. 38 | """ 39 | return Commit( 40 | repo=commit.branch.repo.name, 41 | branch=commit.branch.name, 42 | id=commit.id, 43 | repo_type=commit.branch.repo.type, 44 | project=commit.branch.repo.project.name, 45 | ) 46 | 47 | 48 | # TODO: How to reconcile tuple commit format with projects. 49 | SubcommitType = Union[tuple, dict, Commit, pfs_pb2.Commit] 50 | """Composite type for a subcommit, a commit at the repo-level. 51 | 52 | Examples 53 | -------- 54 | Tuple: 55 | 56 | >>> sc = ("foo", "master") 57 | >>> sc2 = ("foo", "467c580611234cdb8cc9758c7aa96087") 58 | 59 | Dict: 60 | 61 | >>> sc = {repo: "foo", branch: "master", repo_type: "spec"} 62 | 63 | Commit: 64 | 65 | >>> from python_pachyderm.pfs import Commit 66 | >>> sc = Commit(repo="foo", branch="master") 67 | 68 | pfs_pb2.Commit: 69 | 70 | >>> from python_pachyderm.service import pfs_proto 71 | >>> sc = pfs_pb2.Commit( 72 | ... branch=pfs_pb2.Branch( 73 | ... repo=pfs_pb2.Repo(name="foo", type="user"), 74 | ... name="master", 75 | ... ) 76 | ... ) 77 | """ 78 | 79 | 80 | def commit_from( 81 | commit: SubcommitType = None, 82 | ) -> pfs_pb2.Commit: 83 | """A commit can be identified by (repo, branch, commit_id, repo_type) 84 | 85 | Helper function to convert objects that represent a Commit query into a 86 | protobuf Commit object. 87 | 88 | Parameters 89 | ---------- 90 | commit : SubcommitType, optional 91 | The commit representation to convert to a protobuf commit object. 92 | 93 | Returns 94 | ------- 95 | pfs_pb2.Commit 96 | A protobuf object that represents a commit. 97 | """ 98 | if isinstance(commit, pfs_pb2.Commit): 99 | return commit 100 | if isinstance(commit, Commit): 101 | return commit.to_pb() 102 | if isinstance(commit, tuple): 103 | repo, branch, commit_id, repo_type = None, None, None, "user" 104 | if len(commit) == 2: 105 | repo, branch_or_commit = commit 106 | if uuid_re.match(branch_or_commit) or not valid_branch_re.match( 107 | branch_or_commit 108 | ): 109 | commit_id = branch_or_commit 110 | else: 111 | branch = branch_or_commit 112 | elif len(commit) == 3: 113 | repo, branch, commit_id = commit 114 | else: 115 | repo, branch, commit_id, repo_type = commit 116 | return Commit( 117 | repo=repo, branch=branch, id=commit_id, repo_type=repo_type 118 | ).to_pb() 119 | if isinstance(commit, dict): 120 | return Commit(**commit).to_pb() 121 | if commit is None: 122 | return None 123 | 124 | raise TypeError("Please provide a tuple, dict, or Commit object") 125 | -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/admin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/admin/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/admin/admin_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | 5 | from python_pachyderm.proto.v2.admin import admin_pb2 as python__pachyderm_dot_proto_dot_v2_dot_admin_dot_admin__pb2 6 | 7 | 8 | class APIStub(object): 9 | """Missing associated documentation comment in .proto file.""" 10 | 11 | def __init__(self, channel): 12 | """Constructor. 13 | 14 | Args: 15 | channel: A grpc.Channel. 16 | """ 17 | self.InspectCluster = channel.unary_unary( 18 | '/admin_v2.API/InspectCluster', 19 | request_serializer=python__pachyderm_dot_proto_dot_v2_dot_admin_dot_admin__pb2.InspectClusterRequest.SerializeToString, 20 | response_deserializer=python__pachyderm_dot_proto_dot_v2_dot_admin_dot_admin__pb2.ClusterInfo.FromString, 21 | ) 22 | 23 | 24 | class APIServicer(object): 25 | """Missing associated documentation comment in .proto file.""" 26 | 27 | def InspectCluster(self, request, context): 28 | """Missing associated documentation comment in .proto file.""" 29 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 30 | context.set_details('Method not implemented!') 31 | raise NotImplementedError('Method not implemented!') 32 | 33 | 34 | def add_APIServicer_to_server(servicer, server): 35 | rpc_method_handlers = { 36 | 'InspectCluster': grpc.unary_unary_rpc_method_handler( 37 | servicer.InspectCluster, 38 | request_deserializer=python__pachyderm_dot_proto_dot_v2_dot_admin_dot_admin__pb2.InspectClusterRequest.FromString, 39 | response_serializer=python__pachyderm_dot_proto_dot_v2_dot_admin_dot_admin__pb2.ClusterInfo.SerializeToString, 40 | ), 41 | } 42 | generic_handler = grpc.method_handlers_generic_handler( 43 | 'admin_v2.API', rpc_method_handlers) 44 | server.add_generic_rpc_handlers((generic_handler,)) 45 | 46 | 47 | # This class is part of an EXPERIMENTAL API. 48 | class API(object): 49 | """Missing associated documentation comment in .proto file.""" 50 | 51 | @staticmethod 52 | def InspectCluster(request, 53 | target, 54 | options=(), 55 | channel_credentials=None, 56 | call_credentials=None, 57 | insecure=False, 58 | compression=None, 59 | wait_for_ready=None, 60 | timeout=None, 61 | metadata=None): 62 | return grpc.experimental.unary_unary(request, target, '/admin_v2.API/InspectCluster', 63 | python__pachyderm_dot_proto_dot_v2_dot_admin_dot_admin__pb2.InspectClusterRequest.SerializeToString, 64 | python__pachyderm_dot_proto_dot_v2_dot_admin_dot_admin__pb2.ClusterInfo.FromString, 65 | options, channel_credentials, 66 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 67 | -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/auth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/auth/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/debug/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/debug/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/enterprise/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/enterprise/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/identity/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/identity/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/license/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/license/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/pfs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/pfs/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/pps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/pps/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/proxy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/proxy/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/proxy/proxy_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | 5 | from python_pachyderm.proto.v2.proxy import proxy_pb2 as python__pachyderm_dot_proto_dot_v2_dot_proxy_dot_proxy__pb2 6 | 7 | 8 | class APIStub(object): 9 | """Missing associated documentation comment in .proto file.""" 10 | 11 | def __init__(self, channel): 12 | """Constructor. 13 | 14 | Args: 15 | channel: A grpc.Channel. 16 | """ 17 | self.Listen = channel.unary_stream( 18 | '/proxy.API/Listen', 19 | request_serializer=python__pachyderm_dot_proto_dot_v2_dot_proxy_dot_proxy__pb2.ListenRequest.SerializeToString, 20 | response_deserializer=python__pachyderm_dot_proto_dot_v2_dot_proxy_dot_proxy__pb2.ListenResponse.FromString, 21 | ) 22 | 23 | 24 | class APIServicer(object): 25 | """Missing associated documentation comment in .proto file.""" 26 | 27 | def Listen(self, request, context): 28 | """Listen streams database events. 29 | It signals that it is internally set up by sending an initial empty ListenResponse. 30 | """ 31 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 32 | context.set_details('Method not implemented!') 33 | raise NotImplementedError('Method not implemented!') 34 | 35 | 36 | def add_APIServicer_to_server(servicer, server): 37 | rpc_method_handlers = { 38 | 'Listen': grpc.unary_stream_rpc_method_handler( 39 | servicer.Listen, 40 | request_deserializer=python__pachyderm_dot_proto_dot_v2_dot_proxy_dot_proxy__pb2.ListenRequest.FromString, 41 | response_serializer=python__pachyderm_dot_proto_dot_v2_dot_proxy_dot_proxy__pb2.ListenResponse.SerializeToString, 42 | ), 43 | } 44 | generic_handler = grpc.method_handlers_generic_handler( 45 | 'proxy.API', rpc_method_handlers) 46 | server.add_generic_rpc_handlers((generic_handler,)) 47 | 48 | 49 | # This class is part of an EXPERIMENTAL API. 50 | class API(object): 51 | """Missing associated documentation comment in .proto file.""" 52 | 53 | @staticmethod 54 | def Listen(request, 55 | target, 56 | options=(), 57 | channel_credentials=None, 58 | call_credentials=None, 59 | insecure=False, 60 | compression=None, 61 | wait_for_ready=None, 62 | timeout=None, 63 | metadata=None): 64 | return grpc.experimental.unary_stream(request, target, '/proxy.API/Listen', 65 | python__pachyderm_dot_proto_dot_v2_dot_proxy_dot_proxy__pb2.ListenRequest.SerializeToString, 66 | python__pachyderm_dot_proto_dot_v2_dot_proxy_dot_proxy__pb2.ListenResponse.FromString, 67 | options, channel_credentials, 68 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 69 | -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/task/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/task/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/task/task_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | 5 | -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/transaction/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/transaction/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/version/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/version/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/version/versionpb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/version/versionpb/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/version/versionpb/version_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | 5 | from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 6 | from python_pachyderm.proto.v2.version.versionpb import version_pb2 as python__pachyderm_dot_proto_dot_v2_dot_version_dot_versionpb_dot_version__pb2 7 | 8 | 9 | class APIStub(object): 10 | """Missing associated documentation comment in .proto file.""" 11 | 12 | def __init__(self, channel): 13 | """Constructor. 14 | 15 | Args: 16 | channel: A grpc.Channel. 17 | """ 18 | self.GetVersion = channel.unary_unary( 19 | '/versionpb_v2.API/GetVersion', 20 | request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, 21 | response_deserializer=python__pachyderm_dot_proto_dot_v2_dot_version_dot_versionpb_dot_version__pb2.Version.FromString, 22 | ) 23 | 24 | 25 | class APIServicer(object): 26 | """Missing associated documentation comment in .proto file.""" 27 | 28 | def GetVersion(self, request, context): 29 | """Missing associated documentation comment in .proto file.""" 30 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 31 | context.set_details('Method not implemented!') 32 | raise NotImplementedError('Method not implemented!') 33 | 34 | 35 | def add_APIServicer_to_server(servicer, server): 36 | rpc_method_handlers = { 37 | 'GetVersion': grpc.unary_unary_rpc_method_handler( 38 | servicer.GetVersion, 39 | request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, 40 | response_serializer=python__pachyderm_dot_proto_dot_v2_dot_version_dot_versionpb_dot_version__pb2.Version.SerializeToString, 41 | ), 42 | } 43 | generic_handler = grpc.method_handlers_generic_handler( 44 | 'versionpb_v2.API', rpc_method_handlers) 45 | server.add_generic_rpc_handlers((generic_handler,)) 46 | 47 | 48 | # This class is part of an EXPERIMENTAL API. 49 | class API(object): 50 | """Missing associated documentation comment in .proto file.""" 51 | 52 | @staticmethod 53 | def GetVersion(request, 54 | target, 55 | options=(), 56 | channel_credentials=None, 57 | call_credentials=None, 58 | insecure=False, 59 | compression=None, 60 | wait_for_ready=None, 61 | timeout=None, 62 | metadata=None): 63 | return grpc.experimental.unary_unary(request, target, '/versionpb_v2.API/GetVersion', 64 | google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, 65 | python__pachyderm_dot_proto_dot_v2_dot_version_dot_versionpb_dot_version__pb2.Version.FromString, 66 | options, channel_credentials, 67 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 68 | -------------------------------------------------------------------------------- /src/python_pachyderm/proto/v2/worker/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pachyderm/python-pachyderm/dbba5b585a3d8d266f13a1f299ba06e597d1ab0e/src/python_pachyderm/proto/v2/worker/__init__.py -------------------------------------------------------------------------------- /src/python_pachyderm/service.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | from grpc_health.v1 import health_pb2 as health_proto 4 | from grpc_health.v1 import health_pb2_grpc as health_grpc 5 | from python_pachyderm.proto.v2.admin import admin_pb2 as admin_proto 6 | from python_pachyderm.proto.v2.admin import admin_pb2_grpc as admin_grpc 7 | from python_pachyderm.proto.v2.auth import auth_pb2 as auth_proto 8 | from python_pachyderm.proto.v2.auth import auth_pb2_grpc as auth_grpc 9 | from python_pachyderm.proto.v2.debug import debug_pb2 as debug_proto 10 | from python_pachyderm.proto.v2.debug import debug_pb2_grpc as debug_grpc 11 | from python_pachyderm.proto.v2.enterprise import enterprise_pb2 as enterprise_proto 12 | from python_pachyderm.proto.v2.enterprise import enterprise_pb2_grpc as enterprise_grpc 13 | from python_pachyderm.proto.v2.identity import identity_pb2 as identity_proto 14 | from python_pachyderm.proto.v2.identity import identity_pb2_grpc as identity_grpc 15 | from python_pachyderm.proto.v2.license import license_pb2 as license_proto 16 | from python_pachyderm.proto.v2.license import license_pb2_grpc as license_grpc 17 | from python_pachyderm.proto.v2.pfs import pfs_pb2 as pfs_proto 18 | from python_pachyderm.proto.v2.pfs import pfs_pb2_grpc as pfs_grpc 19 | from python_pachyderm.proto.v2.pps import pps_pb2 as pps_proto 20 | from python_pachyderm.proto.v2.pps import pps_pb2_grpc as pps_grpc 21 | from python_pachyderm.proto.v2.transaction import transaction_pb2 as transaction_proto 22 | from python_pachyderm.proto.v2.transaction import ( 23 | transaction_pb2_grpc as transaction_grpc, 24 | ) 25 | from python_pachyderm.proto.v2.version.versionpb import version_pb2 as version_proto 26 | from python_pachyderm.proto.v2.version.versionpb import version_pb2_grpc as version_grpc 27 | 28 | MB = 1024**2 29 | MAX_RECEIVE_MESSAGE_SIZE = 20 * MB 30 | PRIMARY_USER_AGENT = "python-pachyderm" 31 | # SECONDARY_USER_AGENT = "v0.0.0" # TODO: populate this programmatically 32 | GRPC_CHANNEL_OPTIONS = [ 33 | ("grpc.max_receive_message_length", MAX_RECEIVE_MESSAGE_SIZE), 34 | ("grpc.primary_user_agent", PRIMARY_USER_AGENT), 35 | # ("grpc.secondary_user_agent", SECONDARY_USER_AGENT), 36 | ] 37 | 38 | 39 | class Service(Enum): 40 | ADMIN = 0 41 | AUTH = 1 42 | DEBUG = 2 43 | HEALTH = 3 44 | ENTERPRISE = 4 45 | PFS = 5 46 | PPS = 6 47 | TRANSACTION = 7 48 | VERSION = 8 49 | LICENSE = 9 50 | IDENTITY = 10 51 | 52 | @property 53 | def grpc_module(self): 54 | return GRPC_MODULES[self] 55 | 56 | @property 57 | def stub(self): 58 | grpc_module = self.grpc_module 59 | 60 | for key in dir(grpc_module): 61 | if key.endswith("Stub"): 62 | return getattr(self.grpc_module, key) 63 | 64 | @property 65 | def servicer(self): 66 | grpc_module = self.grpc_module 67 | 68 | for key in dir(grpc_module): 69 | if key.endswith("Servicer"): 70 | return getattr(self.grpc_module, key) 71 | 72 | @property 73 | def proto_module(self): 74 | return PROTO_MODULES[self] 75 | 76 | @property 77 | def options(self): 78 | return [ 79 | ("grpc.max_receive_message_length", MAX_RECEIVE_MESSAGE_SIZE), 80 | ] 81 | 82 | 83 | GRPC_MODULES = { 84 | Service.ADMIN: admin_grpc, 85 | Service.AUTH: auth_grpc, 86 | Service.DEBUG: debug_grpc, 87 | Service.ENTERPRISE: enterprise_grpc, 88 | Service.HEALTH: health_grpc, 89 | Service.IDENTITY: identity_grpc, 90 | Service.LICENSE: license_grpc, 91 | Service.PFS: pfs_grpc, 92 | Service.PPS: pps_grpc, 93 | Service.TRANSACTION: transaction_grpc, 94 | Service.VERSION: version_grpc, 95 | } 96 | 97 | PROTO_MODULES = { 98 | Service.ADMIN: admin_proto, 99 | Service.AUTH: auth_proto, 100 | Service.DEBUG: debug_proto, 101 | Service.ENTERPRISE: enterprise_proto, 102 | Service.HEALTH: health_proto, 103 | Service.IDENTITY: identity_proto, 104 | Service.LICENSE: license_proto, 105 | Service.PFS: pfs_proto, 106 | Service.PPS: pps_proto, 107 | Service.TRANSACTION: transaction_proto, 108 | Service.VERSION: version_proto, 109 | } 110 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Unit test package for python-pachyderm.""" 2 | -------------------------------------------------------------------------------- /tests/health.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Tests for health-related functionality.""" 4 | 5 | import python_pachyderm 6 | 7 | 8 | def test_health(): 9 | python_pachyderm.Client().health() 10 | -------------------------------------------------------------------------------- /tests/test_admin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Tests admin-related functionality""" 4 | 5 | import python_pachyderm 6 | from python_pachyderm.service import admin_proto 7 | 8 | 9 | def test_inspect_cluster(): 10 | client = python_pachyderm.Client() 11 | res = client.inspect_cluster() 12 | assert isinstance(res, admin_proto.ClusterInfo) 13 | -------------------------------------------------------------------------------- /tests/test_auth.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Tests admin-related functionality""" 4 | 5 | import os 6 | 7 | import pytest 8 | 9 | import python_pachyderm 10 | from python_pachyderm.errors import AuthServiceNotActivated 11 | from python_pachyderm.service import auth_proto, identity_proto 12 | from tests import util 13 | 14 | 15 | @pytest.fixture 16 | def client(): 17 | pc = python_pachyderm.Client() 18 | pc.activate_license(os.environ["PACH_PYTHON_ENTERPRISE_CODE"]) 19 | pc.add_cluster("localhost", "localhost:1650", secret="secret") 20 | pc.activate_enterprise("localhost:1650", "localhost", "secret") 21 | 22 | pc.auth_token = "iamroot" 23 | pc.activate_auth(pc.auth_token) 24 | pc.set_identity_server_config( 25 | config=identity_proto.IdentityServerConfig(issuer="http://localhost:1658") 26 | ) 27 | yield pc 28 | # not redundant because auth_token could be overriden by tests 29 | pc.auth_token = "iamroot" 30 | pc.delete_all() 31 | pc.deactivate_enterprise() 32 | 33 | 34 | @util.skip_if_no_enterprise() 35 | def test_auth_configuration(client): 36 | client.get_auth_configuration() 37 | client.set_auth_configuration( 38 | auth_proto.OIDCConfig( 39 | issuer="http://localhost:1658", 40 | client_id="client", 41 | client_secret="secret", 42 | redirect_uri="http://test.example.com", 43 | ) 44 | ) 45 | 46 | 47 | @util.skip_if_no_enterprise() 48 | def test_cluster_role_bindings(client): 49 | cluster_resource = auth_proto.Resource(type=auth_proto.CLUSTER) 50 | binding = client.get_role_binding(cluster_resource) 51 | assert binding["pach:root"].roles["clusterAdmin"] 52 | client.modify_role_binding( 53 | cluster_resource, "robot:someuser", roles=["clusterAdmin"] 54 | ) 55 | 56 | binding = client.get_role_binding(cluster_resource) 57 | assert binding["robot:someuser"].roles["clusterAdmin"] 58 | 59 | 60 | @util.skip_if_no_enterprise() 61 | def test_authorize(client): 62 | client.authorize( 63 | auth_proto.Resource(type=auth_proto.REPO, name="foobar"), 64 | [auth_proto.Permission.REPO_READ], 65 | ) 66 | 67 | 68 | @util.skip_if_no_enterprise() 69 | def test_who_am_i(client): 70 | assert client.who_am_i().username == "pach:root" 71 | 72 | 73 | @util.skip_if_no_enterprise() 74 | def test_get_roles_for_permission(client): 75 | # Checks built-in roles 76 | roles = client.get_roles_for_permission(auth_proto.Permission.REPO_READ) 77 | for r in roles: 78 | assert auth_proto.Permission.REPO_READ in r.permissions 79 | 80 | roles = client.get_roles_for_permission( 81 | auth_proto.Permission.CLUSTER_GET_PACHD_LOGS 82 | ) 83 | for r in roles: 84 | assert auth_proto.Permission.CLUSTER_GET_PACHD_LOGS in r.permissions 85 | 86 | 87 | @util.skip_if_no_enterprise() 88 | def test_robot_token(client): 89 | auth_token = client.get_robot_token("robot:root", ttl=30) 90 | client.auth_token = auth_token 91 | assert client.who_am_i().username == "robot:root" 92 | client.revoke_auth_token(auth_token) 93 | with pytest.raises(python_pachyderm.RpcError): 94 | client.who_am_i() 95 | 96 | 97 | @util.skip_if_no_enterprise() 98 | def test_groups(client): 99 | assert client.get_groups() == [] 100 | client.set_groups_for_user("pach:root", ["foogroup"]) 101 | assert client.get_groups() == ["foogroup"] 102 | assert client.get_users("foogroup") == ["pach:root"] 103 | client.modify_members("foogroup", remove=["pach:root"]) 104 | assert client.get_groups() == [] 105 | assert client.get_users("foogroup") == [] 106 | 107 | 108 | def test_deactivate_error(): 109 | """Test that calling AuthMixin.deactivate_auth returns a custom error 110 | if auth has not been activated.""" 111 | client = python_pachyderm.Client() 112 | with pytest.raises(AuthServiceNotActivated): 113 | client.deactivate_auth() 114 | -------------------------------------------------------------------------------- /tests/test_debug.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Tests debug-related functionality""" 4 | 5 | import python_pachyderm 6 | from google.protobuf import duration_pb2 7 | 8 | 9 | def test_dump_v2(): 10 | client = python_pachyderm.Client() 11 | response = next(client.dump()) 12 | assert isinstance(response.content.content, bytes) 13 | assert response.progress.progress > 0 14 | 15 | 16 | def test_profile_cpu(): 17 | client = python_pachyderm.Client() 18 | for b in client.profile_cpu(duration_pb2.Duration(seconds=1)): 19 | assert isinstance(b, bytes) 20 | assert len(b) > 0 21 | 22 | 23 | def test_binary(): 24 | client = python_pachyderm.Client() 25 | for b in client.binary(): 26 | assert isinstance(b, bytes) 27 | assert len(b) > 0 28 | -------------------------------------------------------------------------------- /tests/test_enterprise.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Tests admin-related functionality""" 4 | 5 | import os 6 | import pytest 7 | 8 | import python_pachyderm 9 | from python_pachyderm.service import enterprise_proto, identity_proto 10 | from tests import util 11 | 12 | 13 | @pytest.fixture 14 | def client(): 15 | pc = python_pachyderm.Client() 16 | pc.delete_all_license() 17 | 18 | pc.activate_license(os.environ["PACH_PYTHON_ENTERPRISE_CODE"]) 19 | pc.add_cluster("localhost", "localhost:1650", secret="secret") 20 | pc.update_cluster("localhost", "localhost:1650", "localhost:16650") 21 | pc.activate_enterprise("localhost:1650", "localhost", "secret") 22 | 23 | pc.auth_token = "iamroot" 24 | pc.activate_auth(pc.auth_token) 25 | pc.set_identity_server_config( 26 | config=identity_proto.IdentityServerConfig(issuer="http://localhost:1658") 27 | ) 28 | 29 | yield pc 30 | 31 | pc.delete_cluster("localhost") 32 | pc.deactivate_enterprise() 33 | pc.delete_all() 34 | 35 | 36 | @util.skip_if_no_enterprise() 37 | def test_enterprise(client): 38 | assert len(client.list_clusters()) == len(client.list_user_clusters()) 39 | assert client.get_enterprise_state().state == enterprise_proto.State.ACTIVE 40 | assert ( 41 | client.get_activation_code().activation_code 42 | == os.environ["PACH_PYTHON_ENTERPRISE_CODE"] 43 | ) 44 | assert ( 45 | client.get_pause_status().status 46 | == enterprise_proto.PauseStatusResponse.PauseStatus.UNPAUSED 47 | ) 48 | -------------------------------------------------------------------------------- /tests/test_identity.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Tests identity-related functionality""" 4 | 5 | import os 6 | 7 | import pytest 8 | 9 | import python_pachyderm 10 | from python_pachyderm.errors import AuthServiceNotActivated 11 | from python_pachyderm.service import identity_proto 12 | from tests import util 13 | 14 | 15 | @pytest.fixture 16 | def client(): 17 | pc = python_pachyderm.Client() 18 | pc.activate_license(os.environ["PACH_PYTHON_ENTERPRISE_CODE"]) 19 | pc.add_cluster("localhost", "localhost:1650", secret="secret") 20 | pc.activate_enterprise("localhost:1650", "localhost", "secret") 21 | 22 | pc.auth_token = "iamroot" 23 | pc.activate_auth(pc.auth_token) 24 | pc.set_identity_server_config( 25 | config=identity_proto.IdentityServerConfig(issuer="http://localhost:1658") 26 | ) 27 | yield pc 28 | # not redundant because auth_token could be overriden by tests 29 | pc.auth_token = "iamroot" 30 | pc.delete_all() 31 | pc.deactivate_enterprise() 32 | 33 | 34 | @util.skip_if_no_enterprise() 35 | def test_identity_server_config(client): 36 | isc = client.get_identity_server_config() 37 | assert isc.issuer == "http://localhost:1658" 38 | 39 | 40 | @util.skip_if_no_enterprise() 41 | def test_oidc_client(client): 42 | oidc1 = client.create_oidc_client( 43 | identity_proto.OIDCClient(id="oidc1", name="pach1", secret="secret1") 44 | ) 45 | client.create_oidc_client( 46 | identity_proto.OIDCClient(id="oidc2", name="pach2", secret="secret2") 47 | ) 48 | 49 | assert len(client.list_oidc_clients()) == 2 50 | assert client.get_oidc_client(oidc1.id).name == "pach1" 51 | assert client.get_oidc_client("oidc2").name == "pach2" 52 | 53 | client.update_oidc_client( 54 | identity_proto.OIDCClient(id="oidc1", name="pach3", secret="secret1") 55 | ) 56 | oidc_updated = client.get_oidc_client(oidc1.id) 57 | assert oidc_updated.name == "pach3" 58 | 59 | client.delete_oidc_client(oidc1.id) 60 | assert len(client.list_oidc_clients()) == 1 61 | 62 | 63 | def test_delete_all_error(): 64 | """Test that calling IdentityMixin.delete_all_identity returns a custom 65 | error if auth has not been activated.""" 66 | client = python_pachyderm.Client() 67 | with pytest.raises(AuthServiceNotActivated): 68 | client.delete_all_identity() 69 | -------------------------------------------------------------------------------- /tests/test_spout.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Spout tests""" 4 | import time 5 | import pytest 6 | import python_pachyderm 7 | from python_pachyderm.service import pps_proto, pfs_proto 8 | 9 | 10 | def test_create_spout(): 11 | client = python_pachyderm.Client() 12 | client.delete_all() 13 | 14 | client.create_pipeline( 15 | pipeline_name="pipeline-create-spout", 16 | transform=pps_proto.Transform( 17 | cmd=["sh"], 18 | image="alpine", 19 | ), 20 | spout=pps_proto.Spout(), 21 | ) 22 | 23 | assert len(list(client.list_pipeline())) == 1 24 | 25 | 26 | @pytest.mark.timeout(45) 27 | def test_spout_commit(): 28 | client = python_pachyderm.Client() 29 | client.delete_all() 30 | 31 | client.create_pipeline( 32 | pipeline_name="pipeline-spout-commit", 33 | transform=pps_proto.Transform( 34 | cmd=["bash"], 35 | stdin=[ 36 | "echo 'commit time' >> file.txt", 37 | "pachctl put file pipeline-spout-commit@master:/file.txt -f file.txt", 38 | ], 39 | ), 40 | spout=pps_proto.Spout(), 41 | ) 42 | 43 | c = client.subscribe_commit( 44 | repo_name="pipeline-spout-commit", 45 | branch="master", 46 | state=pfs_proto.FINISHED, 47 | origin_kind=pfs_proto.USER, 48 | ) 49 | next(c) 50 | 51 | commit_infos = list(client.list_commit("pipeline-spout-commit")) 52 | assert len(commit_infos) == 1 53 | -------------------------------------------------------------------------------- /tests/test_version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Tests for versioning-related functionality.""" 4 | 5 | import python_pachyderm 6 | from python_pachyderm.proto.v2.version.versionpb import version_pb2 7 | 8 | 9 | def test_remote_version(): 10 | version_pb = python_pachyderm.Client().get_remote_version() 11 | assert isinstance(version_pb, version_pb2.Version) 12 | -------------------------------------------------------------------------------- /tests/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import string 4 | import random 5 | 6 | import pytest 7 | 8 | import python_pachyderm 9 | from python_pachyderm.service import pps_proto 10 | 11 | _test_pachyderm_version = None 12 | 13 | 14 | def test_pachyderm_version(): 15 | global _test_pachyderm_version 16 | 17 | if _test_pachyderm_version is None: 18 | value = os.environ.get("PACHYDERM_VERSION") 19 | 20 | if value is None: 21 | client = python_pachyderm.Client() 22 | value = client.get_remote_version() 23 | _test_pachyderm_version = (value.major, value.minor, value.micro) 24 | else: 25 | _test_pachyderm_version = tuple(int(i) for i in value.split(".")) 26 | 27 | return _test_pachyderm_version 28 | 29 | 30 | def skip_if_below_pachyderm_version(major, minor, revision): 31 | test = test_pachyderm_version() < (major, minor, revision) 32 | reason = "requires pachyderm {}.{}.{} or higher".format(major, minor, revision) 33 | return pytest.mark.skipif(test, reason=reason) 34 | 35 | 36 | def skip_if_no_enterprise(): 37 | test = os.environ.get("PACH_PYTHON_ENTERPRISE_CODE") is None 38 | return pytest.mark.skipif(test, reason="enterprise code not available") 39 | 40 | 41 | def random_string(n): 42 | return "".join( 43 | random.choice(string.ascii_lowercase + string.digits) for _ in range(n) 44 | ) 45 | 46 | 47 | def test_repo_name(test_name, prefix=None, suffix=None): 48 | prefix = "" if prefix is None else "{}-".format(prefix) 49 | suffix = suffix or random_string(6) 50 | return "{}{}-{}".format(prefix, test_name, suffix) 51 | 52 | 53 | def create_test_repo(client, test_name, project_name=None, prefix=None, suffix=None): 54 | repo_name = test_repo_name(test_name, prefix=prefix, suffix=suffix) 55 | client.create_repo( 56 | repo_name, 57 | project_name=project_name, 58 | description="python_pachyderm test repo for {}".format(test_name), 59 | ) 60 | return repo_name 61 | 62 | 63 | def create_test_pipeline(client: python_pachyderm.Client, test_name): 64 | repo_name_suffix = random_string(3) 65 | project_name = f"{test_name}-{repo_name_suffix}" 66 | client.create_project(project_name) 67 | input_repo_name = create_test_repo( 68 | client, 69 | test_name, 70 | project_name=project_name, 71 | prefix="input", 72 | suffix=repo_name_suffix, 73 | ) 74 | pipeline_repo_name = test_repo_name( 75 | test_name, prefix="pipeline", suffix=repo_name_suffix 76 | ) 77 | 78 | client.create_pipeline( 79 | pipeline_repo_name, 80 | project_name=project_name, 81 | transform=pps_proto.Transform( 82 | cmd=["sh"], 83 | image="alpine", 84 | stdin=["cp /pfs/{}/*.dat /pfs/out/".format(input_repo_name)], 85 | ), 86 | input=pps_proto.Input(pfs=pps_proto.PFSInput(glob="/*", repo=input_repo_name)), 87 | ) 88 | 89 | with client.commit(input_repo_name, "master", project_name=project_name) as commit: 90 | client.put_file_bytes(commit, "file.dat", b"DATA") 91 | 92 | return (commit, input_repo_name, pipeline_repo_name, project_name) 93 | 94 | 95 | def get_cluster_deployment_id(): 96 | client = python_pachyderm.Client() 97 | cluster_info = client.inspect_cluster() 98 | return cluster_info.deployment_id 99 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py37, py310, lint, examples 3 | isolated_build = True 4 | 5 | [testenv] 6 | setenv = 7 | PYTHONPATH = {toxinidir} 8 | HOME = /home/circleci 9 | extras = 10 | test 11 | allowlist_externals = 12 | poetry 13 | pytest 14 | commands = 15 | poetry install -v 16 | poetry run pytest tests --basetemp={envtmpdir} 17 | passenv = PACHYDERM_VERSION PACH_PYTHON_ENTERPRISE_CODE 18 | 19 | [testenv:lint] 20 | allowlist_externals=make 21 | deps = 22 | black 23 | flake8 24 | commands = make lint 25 | 26 | [testenv:examples] 27 | commands=python ./examples/opencv/opencv.py 28 | -------------------------------------------------------------------------------- /version.json: -------------------------------------------------------------------------------- 1 | { 2 | "pachyderm": "2.7.0", 3 | "kubernetes": "1.22.8", 4 | "minikube": "1.30.1" 5 | } 6 | --------------------------------------------------------------------------------
Docs
72 | Please activate JavaScript to enable the search functionality. 73 |