├── .github └── workflows │ └── pypi-cd.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── License ├── MANIFEST.in ├── README.md ├── README.rst ├── benchmarks ├── getprofile.sh ├── perf_storage_api_arrow.py ├── perf_tabletunnel.py └── perf_types.py ├── bin ├── copyright.py └── string_fixer.py ├── conf └── test.conf.template ├── cupid ├── __init__.py ├── _version.py ├── config.py ├── errors.py ├── io │ ├── __init__.py │ ├── kvstore.py │ └── table │ │ ├── __init__.py │ │ ├── core.py │ │ ├── record.py │ │ └── tests │ │ ├── __init__.py │ │ └── test_table.py ├── proto │ ├── __init__.py │ ├── cupid_process_service.proto │ ├── cupid_process_service_pb2.py │ ├── cupid_subprocess_service.proto │ ├── cupid_subprocess_service_pb2.py │ ├── cupid_task_service.proto │ ├── cupid_task_service_pb2.py │ ├── cupidtaskparam.proto │ ├── cupidtaskparam_pb2.py │ ├── kv_store_service.proto │ └── kv_store_service_pb2.py ├── rpc.py ├── runtime │ ├── __init__.py │ ├── ctypes_libs.py │ ├── runtime.py │ └── runtime_test.py ├── session.py ├── tests │ ├── __init__.py │ └── core.py └── utils.py ├── docs ├── Makefile ├── make.bat ├── requirements-doc.txt └── source │ ├── _static │ ├── PyODPS.png │ ├── d2-node-en.png │ ├── d2-node-zh.png │ ├── df-age-hist.png │ ├── df-op-merge-visualize.svg │ ├── df-plot-iris-collection.png │ ├── df-plot-iris-hist.png │ ├── df-plot-iris-sequence.png │ ├── df-plot-iris-sum.png │ ├── df-steps-visualize.svg │ ├── df-value-count-plot.png │ ├── dfview-data-grid.png │ ├── dfview-pie-aggsel.png │ ├── dfview-scatter-opts.png │ ├── dfview-scatter.png │ ├── exec_notify.png │ ├── ml_retry_btn.png │ ├── progress_example.png │ ├── pyodps_output_decision_tree.svg │ ├── pyodps_roc_output.png │ ├── task_details_dialog.png │ ├── theme_override-nrtd.css │ ├── theme_override-nrtd.js │ ├── theme_override.css │ └── theme_override.js │ ├── api-df.rst │ ├── api-entry.rst │ ├── api-models.rst │ ├── api-tunnel.rst │ ├── api-types.rst │ ├── api.rst │ ├── base-functions.rst │ ├── base-instances.rst │ ├── base-models.rst │ ├── base-projects.rst │ ├── base-resources.rst │ ├── base-schemas.rst │ ├── base-sql.rst │ ├── base-tables.rst │ ├── base-types.rst │ ├── base.rst │ ├── conf.py │ ├── db-dbapi.rst │ ├── db-sqlalchemy.rst │ ├── db.rst │ ├── df-agg.rst │ ├── df-basic.rst │ ├── df-debug-instruction.rst │ ├── df-element.rst │ ├── df-merge.rst │ ├── df-plot.rst │ ├── df-quickstart.rst │ ├── df-sort-distinct-apply.rst │ ├── df-window.rst │ ├── df.rst │ ├── faq-ext.rst │ ├── faq.rst │ ├── index.rst │ ├── installation-ext.rst │ ├── interactive.rst │ ├── locale │ └── en │ │ └── LC_MESSAGES │ │ ├── api-df.po │ │ ├── api-entry.po │ │ ├── api-models.po │ │ ├── api-tunnel.po │ │ ├── api-types.po │ │ ├── api.po │ │ ├── base-functions.po │ │ ├── base-instances.po │ │ ├── base-models.po │ │ ├── base-projects.po │ │ ├── base-resources.po │ │ ├── base-schemas.po │ │ ├── base-sql.po │ │ ├── base-tables.po │ │ ├── base-types.po │ │ ├── base.po │ │ ├── db-dbapi.po │ │ ├── db-sqlalchemy.po │ │ ├── db.po │ │ ├── df-agg.po │ │ ├── df-basic.po │ │ ├── df-debug-instruction.po │ │ ├── df-element.po │ │ ├── df-merge.po │ │ ├── df-plot.po │ │ ├── df-quickstart.po │ │ ├── df-sort-distinct-apply.po │ │ ├── df-window.po │ │ ├── df.po │ │ ├── faq-ext.po │ │ ├── faq.po │ │ ├── index.po │ │ ├── installation-ext.po │ │ ├── interactive.po │ │ ├── options.po │ │ ├── platform-d2.po │ │ ├── platform-migrate-ext.po │ │ ├── platform.po │ │ ├── pyodps-pack-minikube.po │ │ └── pyodps-pack.po │ ├── norm_zh.py │ ├── options.rst │ ├── platform-d2.rst │ ├── platform.rst │ ├── pyodps-pack-minikube.rst │ └── pyodps-pack.rst ├── examples └── storage_api │ ├── __init__.py │ ├── commit_write_session.py │ ├── create_read_session.py │ ├── create_write_session.py │ ├── get_read_session.py │ ├── get_write_session.py │ ├── read_rows.py │ ├── util.py │ └── write_rows.py ├── hooks └── pre-commit ├── misc ├── algorithms.xsd ├── avg.input ├── data ├── explode.input ├── instance.xsd ├── license_head.pl ├── plus.input ├── python ├── stdint.h └── test_udf.py ├── notebooks ├── nb_init.py └── quick_example.ipynb ├── odps ├── __init__.py ├── _version.py ├── accounts.py ├── apis │ ├── __init__.py │ └── storage_api │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── storage_api.py │ │ └── tests │ │ ├── __init__.py │ │ ├── data_item.conf │ │ ├── record_batch_generate.py │ │ ├── test_storage_api.py │ │ ├── test_storage_api_arrow.py │ │ └── util.py ├── compat.py ├── config.py ├── conftest.py ├── console.py ├── core.py ├── counters.py ├── crc.py ├── dag.py ├── dbapi.py ├── df │ ├── __init__.py │ ├── backends │ │ ├── __init__.py │ │ ├── analyzer.py │ │ ├── context.py │ │ ├── core.py │ │ ├── engine.py │ │ ├── errors.py │ │ ├── formatter.py │ │ ├── frame.py │ │ ├── odpssql │ │ │ ├── __init__.py │ │ │ ├── analyzer.py │ │ │ ├── codegen.py │ │ │ ├── compiler.py │ │ │ ├── context.py │ │ │ ├── engine.py │ │ │ ├── models.py │ │ │ ├── rewriter.py │ │ │ ├── tests │ │ │ │ ├── __init__.py │ │ │ │ ├── test_codegen.py │ │ │ │ ├── test_compiler.py │ │ │ │ └── test_engine.py │ │ │ ├── tunnel.py │ │ │ └── types.py │ │ ├── optimize │ │ │ ├── __init__.py │ │ │ ├── columnpruning.py │ │ │ ├── core.py │ │ │ ├── predicatepushdown.py │ │ │ ├── tests │ │ │ │ ├── __init__.py │ │ │ │ ├── test_columnpruning.py │ │ │ │ └── test_predicatepushdown.py │ │ │ └── utils.py │ │ ├── pd │ │ │ ├── __init__.py │ │ │ ├── analyzer.py │ │ │ ├── compiler.py │ │ │ ├── engine.py │ │ │ ├── tests │ │ │ │ ├── __init__.py │ │ │ │ ├── test_compiler.py │ │ │ │ └── test_engine.py │ │ │ └── types.py │ │ ├── rewriter.py │ │ ├── seahawks │ │ │ ├── __init__.py │ │ │ ├── compiler.py │ │ │ ├── engine.py │ │ │ ├── ext.py │ │ │ ├── models.py │ │ │ └── tests │ │ │ │ ├── __init__.py │ │ │ │ └── test_engine.py │ │ ├── selecter.py │ │ ├── sqlalchemy │ │ │ ├── __init__.py │ │ │ ├── analyzer.py │ │ │ ├── compiler.py │ │ │ ├── engine.py │ │ │ ├── ext.py │ │ │ ├── rewriter.py │ │ │ ├── tests │ │ │ │ ├── __init__.py │ │ │ │ └── test_engine.py │ │ │ ├── types.py │ │ │ └── utils.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── core.py │ │ │ ├── test_formatter.py │ │ │ ├── test_mixed_compiler.py │ │ │ ├── test_mixed_engine.py │ │ │ ├── test_selecter.py │ │ │ └── test_with_schema.py │ │ └── utils.py │ ├── core.py │ ├── delay.py │ ├── engines.py │ ├── examples.py │ ├── expr │ │ ├── __init__.py │ │ ├── arithmetic.py │ │ ├── collections.py │ │ ├── composites.py │ │ ├── core.py │ │ ├── datetimes.py │ │ ├── dynamic.py │ │ ├── element.py │ │ ├── errors.py │ │ ├── expressions.py │ │ ├── formatter.py │ │ ├── groupby.py │ │ ├── math.py │ │ ├── merge.py │ │ ├── query.py │ │ ├── reduction.py │ │ ├── strings.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── conftest.py │ │ │ ├── core.py │ │ │ ├── test_arithmetic.py │ │ │ ├── test_collections.py │ │ │ ├── test_composites.py │ │ │ ├── test_core.py │ │ │ ├── test_datetimes.py │ │ │ ├── test_dynamic.py │ │ │ ├── test_element.py │ │ │ ├── test_expressions.py │ │ │ ├── test_formatter.py │ │ │ ├── test_groupby.py │ │ │ ├── test_math.py │ │ │ ├── test_merge.py │ │ │ ├── test_query.py │ │ │ ├── test_reduction.py │ │ │ ├── test_strings.py │ │ │ ├── test_utils.py │ │ │ └── test_window.py │ │ ├── utils.py │ │ └── window.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_dataframe.py │ │ ├── test_delay.py │ │ ├── test_examples.py │ │ ├── test_ui.py │ │ └── test_utils.py │ ├── tools │ │ ├── __init__.py │ │ ├── lib │ │ │ ├── __init__.py │ │ │ ├── bloomfilter.py │ │ │ ├── hll.py │ │ │ └── tests │ │ │ │ ├── __init__.py │ │ │ │ ├── test_bloomfilter.py │ │ │ │ └── test_hll.py │ │ ├── libtools.py │ │ └── plotting.py │ ├── types.py │ ├── ui.py │ └── utils.py ├── distcache.py ├── errors.py ├── examples │ ├── __init__.py │ └── tables.py ├── expressions │ ├── __init__.py │ ├── core.py │ ├── functions.py │ └── tests │ │ ├── __init__.py │ │ └── test_expressions.py ├── inter.py ├── ipython │ ├── __init__.py │ ├── completer.py │ ├── magics.py │ └── tests │ │ ├── __init__.py │ │ ├── test_completer.py │ │ └── test_magics.py ├── lib │ ├── __init__.py │ ├── cgi_compat.py │ ├── cloudpickle.py │ ├── enum.py │ ├── ext_types.py │ ├── futures │ │ ├── __init__.py │ │ ├── _base.py │ │ ├── process.py │ │ └── thread.py │ ├── importer.py │ ├── lib_utils.py │ ├── monotonic.py │ ├── six.py │ ├── tblib │ │ ├── __init__.py │ │ ├── cpython.py │ │ ├── decorators.py │ │ └── pickling_support.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_cloudpickle.py │ │ ├── test_ext_types.py │ │ ├── test_importer.py │ │ └── test_xnamedtuple.py │ ├── tzlocal │ │ ├── LICENSE.txt │ │ ├── __init__.py │ │ ├── unix.py │ │ ├── utils.py │ │ ├── win32.py │ │ └── windows_tz.py │ ├── version.py │ └── xnamedtuple.py ├── mars_extension │ ├── __init__.py │ ├── legacy │ │ ├── __init__.py │ │ ├── actors │ │ │ ├── __init__.py │ │ │ └── session.py │ │ ├── core.py │ │ ├── dataframe │ │ │ ├── __init__.py │ │ │ ├── datasource.py │ │ │ └── datastore.py │ │ ├── deploy │ │ │ ├── __init__.py │ │ │ ├── app.py │ │ │ ├── client.py │ │ │ ├── core.py │ │ │ ├── gscoordinator.py │ │ │ ├── notebook.py │ │ │ ├── scheduler.py │ │ │ ├── utils.py │ │ │ ├── web.py │ │ │ └── worker.py │ │ ├── filesystem.py │ │ ├── io │ │ │ ├── __init__.py │ │ │ ├── core.py │ │ │ ├── local.py │ │ │ └── oss.py │ │ ├── oss.py │ │ ├── run_script.py │ │ ├── tensor │ │ │ ├── __init__.py │ │ │ ├── datasource.py │ │ │ ├── datastore.py │ │ │ └── tests │ │ │ │ ├── __init__.py │ │ │ │ ├── test_datasource.py │ │ │ │ └── test_datastore.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_filesystem.py │ │ │ ├── test_mars_cluster.py │ │ │ └── test_mars_conversion.py │ │ └── web │ │ │ ├── __init__.py │ │ │ └── apihandlers.py │ ├── oscar │ │ ├── __init__.py │ │ ├── bearer_token_init.py │ │ ├── core.py │ │ ├── cupid_service.py │ │ ├── dataframe │ │ │ ├── __init__.py │ │ │ ├── datasource.py │ │ │ └── datastore.py │ │ ├── deploy │ │ │ ├── __init__.py │ │ │ ├── app.py │ │ │ ├── client.py │ │ │ ├── config.yml │ │ │ ├── core.py │ │ │ ├── gscoordinator.py │ │ │ ├── logging.conf │ │ │ ├── notebook.py │ │ │ ├── supervisor.py │ │ │ └── worker.py │ │ ├── filesystem.py │ │ ├── run_script.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_filesystem.py │ │ │ ├── test_mars_cluster.py │ │ │ └── test_tunnel_io.py │ │ └── web.py │ ├── tests │ │ ├── __init__.py │ │ └── test_utils.py │ └── utils.py ├── ml │ ├── __init__.py │ ├── algolib │ │ ├── __init__.py │ │ ├── base_algo.py │ │ ├── loader.py │ │ ├── objects.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── test_algo_build.py │ ├── classifiers │ │ ├── __init__.py │ │ ├── _customize.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ ├── test_classifiers.py │ │ │ └── test_sparse_classifiers.py │ ├── clustering │ │ ├── __init__.py │ │ ├── _customize.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── test_clustering.py │ ├── cross_validation.py │ ├── engine.py │ ├── enums.py │ ├── expr │ │ ├── __init__.py │ │ ├── core.py │ │ ├── exporters.py │ │ ├── mixin.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── pmml.py │ │ │ └── tests │ │ │ │ ├── __init__.py │ │ │ │ └── test_pmml.py │ │ ├── op.py │ │ ├── specialized.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ ├── test_mixin.py │ │ │ └── test_op.py │ ├── feature │ │ ├── __init__.py │ │ ├── _customize.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── test_feature.py │ ├── metrics │ │ ├── __init__.py │ │ ├── _customize.py │ │ ├── classification.py │ │ ├── clustering.py │ │ ├── regression.py │ │ ├── scorer.py │ │ └── utils.py │ ├── models.py │ ├── network │ │ ├── __init__.py │ │ ├── _customize.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── test_networking.py │ ├── pipeline │ │ ├── __init__.py │ │ ├── core.py │ │ ├── steps.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── test_pipeline.py │ ├── preprocess │ │ ├── __init__.py │ │ ├── _customize.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── test_preprocess.py │ ├── recommend │ │ ├── __init__.py │ │ ├── _customize.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── test_recommend.py │ ├── regression │ │ ├── __init__.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── test_regression.py │ ├── runners.py │ ├── statistics │ │ ├── __init__.py │ │ ├── _customize.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── test_statistics.py │ ├── tensor │ │ └── __init__.py │ ├── tests │ │ ├── __init__.py │ │ ├── base.py │ │ ├── test_cv.py │ │ ├── test_dataframe.py │ │ ├── test_execute.py │ │ ├── test_models.py │ │ ├── test_partitions.py │ │ ├── test_persist.py │ │ └── test_runner.py │ ├── text │ │ ├── __init__.py │ │ ├── _customize.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── test_text_algo.py │ ├── timeseries │ │ ├── __init__.py │ │ └── _customize.py │ └── utils.py ├── models │ ├── __init__.py │ ├── cache.py │ ├── cluster_info.py │ ├── core.py │ ├── function.py │ ├── functions.py │ ├── instance.py │ ├── instances.py │ ├── job.py │ ├── ml │ │ ├── __init__.py │ │ ├── offlinemodel.py │ │ └── offlinemodels.py │ ├── partition.py │ ├── partitions.py │ ├── project.py │ ├── projects.py │ ├── quota.py │ ├── quotas.py │ ├── readers.py │ ├── record.py │ ├── resource.py │ ├── resourcefile.py │ ├── resources.py │ ├── schema.py │ ├── schemas.py │ ├── security │ │ ├── __init__.py │ │ ├── config.py │ │ ├── roles.py │ │ └── users.py │ ├── session │ │ ├── __init__.py │ │ ├── core.py │ │ ├── v1.py │ │ └── v2.py │ ├── storage_tier.py │ ├── table.py │ ├── tableio.py │ ├── tables.py │ ├── tasks │ │ ├── __init__.py │ │ ├── copy.py │ │ ├── core.py │ │ ├── maxframe.py │ │ ├── merge.py │ │ ├── misc.py │ │ ├── sql.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ ├── test_merge.py │ │ │ └── test_tasks.py │ ├── tenant.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_functions.py │ │ ├── test_instances.py │ │ ├── test_offline_models.py │ │ ├── test_partitions.py │ │ ├── test_projects.py │ │ ├── test_quotas.py │ │ ├── test_resources.py │ │ ├── test_schemas.py │ │ ├── test_security.py │ │ ├── test_session.py │ │ ├── test_storage_tier.py │ │ ├── test_tableio.py │ │ ├── test_tables.py │ │ ├── test_tenant.py │ │ ├── test_volumes.py │ │ └── test_xflows.py │ ├── volume_ext.py │ ├── volume_fs.py │ ├── volume_parted.py │ ├── volumes.py │ ├── worker.py │ ├── xflow.py │ └── xflows.py ├── readers.py ├── rest.py ├── serializers.py ├── sqlalchemy_odps.py ├── src │ ├── __init__.py │ ├── crc32.c │ ├── crc32c_c.pxd │ ├── crc32c_c.pyx │ ├── stringstream.pxd │ ├── timegm.c │ ├── types_c.pxd │ ├── types_c.pyx │ ├── utils_c.pxd │ └── utils_c.pyx ├── static │ ├── algorithms │ │ ├── classifier.xml │ │ ├── clustering.xml │ │ ├── feature.xml │ │ ├── metrics.xml │ │ ├── network.xml │ │ ├── preprocess.xml │ │ ├── recommend.xml │ │ ├── regression.xml │ │ ├── statistics.xml │ │ └── text.xml │ ├── data │ │ ├── dow_jones.txt │ │ ├── ionosphere.txt │ │ ├── iris.txt │ │ └── splited_words.txt │ └── ui │ │ ├── Gruntfile.js │ │ ├── package.json │ │ ├── src │ │ ├── chosen-sprite.png │ │ ├── chosen-sprite@2x.png │ │ ├── chosen.css │ │ ├── chosen.js │ │ ├── common.js │ │ ├── df-view.js │ │ ├── echarts.min.js │ │ ├── fonts │ │ │ ├── custom-font.css │ │ │ ├── custom-font.eot │ │ │ ├── custom-font.svg │ │ │ ├── custom-font.ttf │ │ │ └── custom-font.woff │ │ ├── html-notify.js │ │ ├── ml-retry.js │ │ ├── progress.js │ │ ├── styles.css │ │ └── westeros.js │ │ └── target │ │ ├── chosen-sprite.png │ │ ├── chosen-sprite@2x.png │ │ ├── chosen.css │ │ ├── fonts │ │ ├── custom-font.css │ │ ├── custom-font.eot │ │ ├── custom-font.svg │ │ ├── custom-font.ttf │ │ └── custom-font.woff │ │ ├── main.js │ │ └── styles.css ├── superset_odps.py ├── tempobj.py ├── tests │ ├── __init__.py │ ├── core.py │ ├── dictconfig.py │ ├── test_accounts.py │ ├── test_config.py │ ├── test_crc.py │ ├── test_dag.py │ ├── test_dbapi.py │ ├── test_errors.py │ ├── test_inter.py │ ├── test_serializers.py │ ├── test_sqlalchemy_odps.py │ ├── test_superset_odps.py │ ├── test_tempobjs.py │ ├── test_types.py │ ├── test_unixsocket.py │ └── test_utils.py ├── tunnel │ ├── __init__.py │ ├── base.py │ ├── checksum.py │ ├── checksum_c.pxd │ ├── checksum_c.pyx │ ├── errors.py │ ├── hasher.py │ ├── hasher_c.pxd │ ├── hasher_c.pyx │ ├── instancetunnel.py │ ├── io │ │ ├── __init__.py │ │ ├── reader.py │ │ ├── reader_c.pxd │ │ ├── reader_c.pyx │ │ ├── stream.py │ │ ├── types.py │ │ ├── writer.py │ │ ├── writer_c.pxd │ │ └── writer_c.pyx │ ├── pb │ │ ├── __init__.py │ │ ├── decoder.py │ │ ├── decoder_c.pxd │ │ ├── decoder_c.pyx │ │ ├── encoder.py │ │ ├── encoder_c.pxd │ │ ├── encoder_c.pyx │ │ ├── errors.py │ │ ├── input_stream.py │ │ ├── output_stream.py │ │ ├── util_c.pxi │ │ └── wire_format.py │ ├── tabletunnel.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_arrow_tabletunnel.py │ │ ├── test_hasher.py │ │ ├── test_instancetunnel.py │ │ ├── test_pb.py │ │ ├── test_streamio.py │ │ ├── test_tabletunnel.py │ │ └── test_volumetunnel.py │ ├── volumetunnel.py │ └── wireconstants.py ├── types.py ├── udf │ ├── __init__.py │ ├── runtime.py │ ├── tests │ │ ├── __init__.py │ │ ├── pyou │ │ ├── test_executioncontext.py │ │ ├── test_resource.py │ │ ├── test_runners.py │ │ ├── test_simple_run.py │ │ ├── test_types_py2.py │ │ ├── test_usercounter.py │ │ └── udf_examples.py │ └── tools │ │ ├── __init__.py │ │ ├── runners.py │ │ └── utils.py ├── ui │ ├── __init__.py │ ├── common.py │ ├── progress.py │ └── tests │ │ ├── __init__.py │ │ ├── base.py │ │ └── test_ui.py └── utils.py ├── odps_scripts ├── __init__.py ├── pyodps_pack.py └── pyou.py ├── pyproject.toml ├── requirements-full.txt ├── requirements-wheel.txt ├── requirements.txt ├── setup.cfg └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # sqlite3 db files 7 | *.db 8 | 9 | # Packages 10 | *.egg 11 | *.egg-info 12 | dist 13 | build 14 | eggs 15 | parts 16 | var 17 | sdist 18 | develop-eggs 19 | .installed.cfg 20 | lib64 21 | __pycache__ 22 | 23 | # Installer logs 24 | pip-log.txt 25 | 26 | # xlib logs 27 | xlibsyslog 28 | 29 | # node-js 30 | node_modules 31 | jspm_packages 32 | package-lock.json 33 | 34 | npm-debug.log* 35 | .npm 36 | .node_repl_history 37 | .grunt 38 | 39 | # Unit test / coverage reports 40 | .coverage 41 | htmlcov 42 | .tox 43 | nosetests.xml 44 | .cache 45 | .pytest* 46 | 47 | # IDEs 48 | .idea 49 | .vscode 50 | *.iml 51 | 52 | # virtualenv 53 | env 54 | 55 | # FIXME: Find a better place for test.conf 56 | odps/tests/test.conf 57 | 58 | .DS_Store 59 | 60 | # notebook 61 | .ipynb_checkpoints 62 | test*.ipynb 63 | 64 | # cython generated 65 | odps/src/*/*_c.c* 66 | odps/src/*_c.c* 67 | odps/tunnel/pb/*_c.c* 68 | odps/tunnel/io/*_c.c* 69 | odps/tunnel/pdio/*_c.c* 70 | odps/tunnel/*_c.c* 71 | 72 | # benchmark 73 | benchmarks/*.out 74 | benchmarks/*.png 75 | 76 | # generated reports 77 | misc/algo_diff.txt 78 | 79 | # test pyodpswrapper 80 | bin/test_pyodpswrapper.py 81 | 82 | # tmp files 83 | *.tmp 84 | *.swp 85 | 86 | # documentation 87 | docs/**/*.mo 88 | /odps/lab_extension/lib/ 89 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/source/conf.py 17 | 18 | # We recommend specifying your dependencies to enable reproducible builds: 19 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 20 | python: 21 | install: 22 | - requirements: docs/requirements-doc.txt 23 | 24 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include odps/static *.* 2 | prune odps/static/ui/node_modules 3 | include requirements.txt 4 | global-include odps/**/*.yml 5 | include odps/mars_extension/oscar/deploy/logging.conf 6 | global-include *.pyx 7 | global-include *.pxd 8 | include odps/src/crc32.c 9 | include odps/src/utils.h 10 | include misc/stdint.h 11 | global-exclude *_c.c* 12 | -------------------------------------------------------------------------------- /benchmarks/getprofile.sh: -------------------------------------------------------------------------------- 1 | rm profile.png 2 | gprof2dot -f pstats profile.out | dot -Tpng -o profile.png -------------------------------------------------------------------------------- /conf/test.conf.template: -------------------------------------------------------------------------------- 1 | [odps] 2 | access_id= 3 | secret_access_key= 4 | project= 5 | endpoint= 6 | 7 | [cupid] 8 | proxy_endpoint= 9 | 10 | [predict] 11 | endpoint= 12 | 13 | [test] 14 | logging_level=INFO 15 | secondary_user= 16 | py37_executable= 17 | py310_executable= 18 | -------------------------------------------------------------------------------- /cupid/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .session import CupidSession 16 | from .config import options # no op 17 | from .runtime import context, ContainerStatus, WorkItemProgress 18 | 19 | from ._version import version_info, __version__ 20 | -------------------------------------------------------------------------------- /cupid/_version.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from odps._version import __version__, version_info 16 | -------------------------------------------------------------------------------- /cupid/io/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .table import TableSplit, CupidTableDownloadSession, CupidTableUploadSession 16 | from .kvstore import CupidKVStore 17 | -------------------------------------------------------------------------------- /cupid/io/table/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .core import CupidTableUploadSession, CupidTableDownloadSession, TableSplit 16 | from .record import CupidRecordReader, CupidRecordWriter 17 | -------------------------------------------------------------------------------- /cupid/io/table/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /cupid/proto/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/cupid/proto/__init__.py -------------------------------------------------------------------------------- /cupid/proto/cupid_process_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package apsara.odps.cupid.protocol; 4 | 5 | option cc_generic_services = true; 6 | option py_generic_services = true; 7 | option java_generic_services = true; 8 | option java_outer_classname = "CupidProcessServiceProto"; 9 | 10 | message EnvEntry 11 | { 12 | optional string name = 1; 13 | optional string value = 2; 14 | } 15 | 16 | message ChildEnv 17 | { 18 | repeated EnvEntry entries = 1; 19 | } 20 | 21 | service ProcessService 22 | { 23 | rpc Prepare(ChildEnv) returns (ChildEnv); 24 | } 25 | -------------------------------------------------------------------------------- /cupid/proto/kv_store_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package apsara.odps.cupid.protocol; 3 | 4 | option cc_generic_services = true; 5 | option java_generic_services = true; 6 | option py_generic_services = true; 7 | 8 | service KVStoreService 9 | { 10 | rpc Put(PutRequest) returns (PutResponse); 11 | 12 | rpc Get(GetRequest) returns (GetResponse); 13 | } 14 | 15 | message PutRequest 16 | { 17 | optional string key = 1; 18 | optional bytes value = 2; 19 | } 20 | 21 | message PutResponse 22 | { 23 | } 24 | 25 | message GetRequest 26 | { 27 | optional string value = 1; 28 | } 29 | 30 | message GetResponse 31 | { 32 | optional bytes value = 1; 33 | } 34 | -------------------------------------------------------------------------------- /cupid/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .runtime import context, RuntimeContext, ContainerStatus, WorkItemProgress 16 | -------------------------------------------------------------------------------- /cupid/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /cupid/tests/core.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /docs/requirements-doc.txt: -------------------------------------------------------------------------------- 1 | jupyter>=1.0.0 2 | ipython>=4.0.0 3 | numpy>=1.6.0 4 | pandas>=0.17.0 5 | matplotlib>=1.4 6 | pytz>=2017.2 7 | pymars>=0.5.4 8 | protobuf>=3.6 9 | sphinxcontrib-jquery>=4.0 10 | sphinx_rtd_theme>=0.5.0 11 | 12 | -------------------------------------------------------------------------------- /docs/source/_static/PyODPS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/PyODPS.png -------------------------------------------------------------------------------- /docs/source/_static/d2-node-en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/d2-node-en.png -------------------------------------------------------------------------------- /docs/source/_static/d2-node-zh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/d2-node-zh.png -------------------------------------------------------------------------------- /docs/source/_static/df-age-hist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/df-age-hist.png -------------------------------------------------------------------------------- /docs/source/_static/df-plot-iris-collection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/df-plot-iris-collection.png -------------------------------------------------------------------------------- /docs/source/_static/df-plot-iris-hist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/df-plot-iris-hist.png -------------------------------------------------------------------------------- /docs/source/_static/df-plot-iris-sequence.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/df-plot-iris-sequence.png -------------------------------------------------------------------------------- /docs/source/_static/df-plot-iris-sum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/df-plot-iris-sum.png -------------------------------------------------------------------------------- /docs/source/_static/df-value-count-plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/df-value-count-plot.png -------------------------------------------------------------------------------- /docs/source/_static/dfview-data-grid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/dfview-data-grid.png -------------------------------------------------------------------------------- /docs/source/_static/dfview-pie-aggsel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/dfview-pie-aggsel.png -------------------------------------------------------------------------------- /docs/source/_static/dfview-scatter-opts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/dfview-scatter-opts.png -------------------------------------------------------------------------------- /docs/source/_static/dfview-scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/dfview-scatter.png -------------------------------------------------------------------------------- /docs/source/_static/exec_notify.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/exec_notify.png -------------------------------------------------------------------------------- /docs/source/_static/ml_retry_btn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/ml_retry_btn.png -------------------------------------------------------------------------------- /docs/source/_static/progress_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/progress_example.png -------------------------------------------------------------------------------- /docs/source/_static/pyodps_roc_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/pyodps_roc_output.png -------------------------------------------------------------------------------- /docs/source/_static/task_details_dialog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/docs/source/_static/task_details_dialog.png -------------------------------------------------------------------------------- /docs/source/_static/theme_override-nrtd.css: -------------------------------------------------------------------------------- 1 | .wy-nav-content { 2 | max-width: 1050px 3 | } 4 | 5 | .wy-table-responsive table td, .wy-table-responsive table th { 6 | white-space: normal !important; 7 | } 8 | 9 | .wy-nav-side { 10 | padding-bottom: 3em; 11 | } 12 | -------------------------------------------------------------------------------- /docs/source/_static/theme_override.css: -------------------------------------------------------------------------------- 1 | .rst-content p.rubric-h2 { 2 | font-size: 150%; 3 | margin-top: 0.85em; 4 | margin-bottom: 0.85em; 5 | } 6 | .rst-content h6.rubric-default { 7 | margin-bottom: 12px; 8 | } 9 | -------------------------------------------------------------------------------- /docs/source/_static/theme_override.js: -------------------------------------------------------------------------------- 1 | $(function() { 2 | var div_replacer = function(r, hn) { 3 | var id_str = '', sub_class='rubric-sub'; 4 | if (!hn) { 5 | hn = 6; 6 | sub_class = 'rubric-sub rubric-default'; 7 | } 8 | if ($(r).attr('id')) id_str = 'id="' + $(r).attr('id') + '" '; 9 | $(r).replaceWith('
' 10 | + $(r).html() + '
'); 11 | }; 12 | for (var hn = 1; hn <= 6; hn++) { 13 | $('.rubric-h' + hn).each(function (i, r) { div_replacer(r, hn); }); 14 | } 15 | $('p.rubric').each(function (i, r) { div_replacer(r); }); 16 | $('.rubric').each(function (i, r) { 17 | var rubric_id = 'rubric' + (i + 1); 18 | if ($(r).attr('id')) { 19 | rubric_id = $(r).attr('id'); 20 | } else { 21 | $(r).attr('id', rubric_id); 22 | } 23 | $(r).find('.rubric-sub').append(''); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /docs/source/api-df.rst: -------------------------------------------------------------------------------- 1 | .. _api_df: 2 | 3 | DataFrame 4 | ========= 5 | 6 | .. autoclass:: odps.df.DataFrame 7 | :members: 8 | 9 | .. autoclass:: odps.df.CollectionExpr 10 | :members: 11 | :inherited-members: 12 | 13 | .. autoclass:: odps.df.SequenceExpr 14 | :members: 15 | :inherited-members: 16 | 17 | .. autoclass:: odps.df.expr.expressions.Int64SequenceExpr 18 | :members: 19 | 20 | .. autoclass:: odps.df.expr.expressions.StringSequenceExpr 21 | :members: 22 | 23 | .. autoclass:: odps.df.Scalar 24 | :members: 25 | :inherited-members: 26 | 27 | .. autofunction:: odps.df.NullScalar 28 | 29 | .. autoclass:: odps.df.RandomScalar 30 | :members: 31 | :inherited-members: 32 | 33 | .. autoclass:: odps.df.expr.groupby.GroupBy 34 | :members: 35 | :inherited-members: 36 | 37 | .. autoclass:: odps.df.expr.groupby.SequenceGroupBy 38 | :members: 39 | :inherited-members: 40 | 41 | .. autoclass:: odps.df.expr.groupby.Int64SequenceGroupBy 42 | :members: 43 | :inherited-members: 44 | -------------------------------------------------------------------------------- /docs/source/api-entry.rst: -------------------------------------------------------------------------------- 1 | .. _api_entry: 2 | 3 | MaxCompute entry 4 | ================= 5 | 6 | .. autoclass:: odps.ODPS 7 | :members: 8 | :exclude-members: attach_session, create_session, default_session 9 | -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | ************* 4 | API Reference 5 | ************* 6 | 7 | Here we give automatically generated API documentation of PyODPS. 8 | 9 | .. toctree:: 10 | :maxdepth: 1 11 | 12 | api-entry 13 | api-types 14 | api-models 15 | api-tunnel 16 | api-df 17 | -------------------------------------------------------------------------------- /docs/source/base-functions.rst: -------------------------------------------------------------------------------- 1 | .. _functions: 2 | 3 | 函数 4 | ======== 5 | 6 | ODPS用户可以编写自定义 `函数 `_ 用在ODPS SQL中。 7 | 8 | 基本操作 9 | --------- 10 | 11 | 可以调用 ODPS 入口对象的 :meth:`~odps.ODPS.list_functions` 来获取项目空间下的所有函数， 12 | :meth:`~odps.ODPS.exist_function` 能判断是否存在函数，\ :meth:`~odps.ODPS.get_function` 13 | 获取函数对象。 14 | 15 | 创建函数 16 | --------- 17 | 18 | .. code-block:: python 19 | 20 | >>> # 引用当前 project 中的资源 21 | >>> resource = o.get_resource('my_udf.py') 22 | >>> function = o.create_function('test_function', class_type='my_udf.Test', resources=[resource]) 23 | >>> # 引用其他 project 中的资源 24 | >>> resource2 = o.get_resource('my_udf.py', project='another_project') 25 | >>> function2 = o.create_function('test_function2', class_type='my_udf.Test', resources=[resource2]) 26 | 27 | 删除函数 28 | --------- 29 | 30 | .. code-block:: python 31 | 32 | >>> o.delete_function('test_function') 33 | >>> function.drop() # Function对象存在时直接调用drop 34 | 35 | 更新函数 36 | --------- 37 | 38 | 只需对函数调用 ``update`` 方法即可。 39 | 40 | .. code-block:: python 41 | 42 | >>> function = o.get_function('test_function') 43 | >>> new_resource = o.get_resource('my_udf2.py') 44 | >>> function.class_type = 'my_udf2.Test' 45 | >>> function.resources = [new_resource, ] 46 | >>> function.update() # 更新函数 47 | 48 | -------------------------------------------------------------------------------- /docs/source/base-projects.rst: -------------------------------------------------------------------------------- 1 | .. _projects: 2 | 3 | 项目空间 4 | ========= 5 | 6 | `项目空间 `_ 是ODPS的基本组织单元， 7 | 有点类似于Database的概念。 8 | 9 | 我们通过 ODPS 入口对象的 :meth:`~odps.ODPS.get_project` 来取到某个项目空间。 10 | 11 | .. code-block:: python 12 | 13 | project = o.get_project('my_project') # 取到某个项目 14 | project = o.get_project() # 取到默认项目 15 | 16 | 如果不提供参数，则取到默认项目空间。 17 | 18 | :meth:`~odps.ODPS.exist_project` 方法能检验某个项目空间是否存在。 19 | -------------------------------------------------------------------------------- /docs/source/base.rst: -------------------------------------------------------------------------------- 1 | .. _base-index: 2 | 3 | ============== 4 | 基本类型及操作 5 | ============== 6 | 7 | PyODPS 提供直接针对 ODPS 对象的基本操作接口，可通过符合 Python 习惯的编程方式操作 ODPS。 8 | 9 | 我们会对这几部分来分别展开说明。 10 | 11 | .. toctree:: 12 | :maxdepth: 1 13 | 14 | base-types 15 | base-projects 16 | base-tables 17 | base-sql 18 | base-instances 19 | base-resources 20 | base-functions 21 | base-volume-int 22 | base-schemas 23 | base-models 24 | -------------------------------------------------------------------------------- /docs/source/db.rst: -------------------------------------------------------------------------------- 1 | .. _db-index: 2 | 3 | ================== 4 | 数据库接口 5 | ================== 6 | 7 | PyODPS 提供了 DBAPI 和 SQLAlchemy 支持，可通过 DBAPI 和 SQLAlchemy 操作 MaxCompute，\ 8 | 进而与其他第三方包或者环境互操作。下面的文档介绍了如何通过 PyODPS 使用 DBAPI 和 SQLAlchemy 9 | 操作 MaxCompute。 10 | 11 | .. toctree:: 12 | :maxdepth: 1 13 | 14 | db-dbapi 15 | db-sqlalchemy 16 | -------------------------------------------------------------------------------- /docs/source/df.rst: -------------------------------------------------------------------------------- 1 | .. _df: 2 | 3 | ********** 4 | DataFrame 5 | ********** 6 | 7 | 8 | PyODPS 提供了 DataFrame API，它提供了类似 pandas 的接口，但是能充分利用 ODPS 的计算能力； 9 | 同时能在本地使用同样的接口，用 pandas 进行计算。 10 | 11 | .. note:: 12 | 13 | PyODPS DataFrame 未来将停止维护。对于新项目，建议使用 `MaxFrame 14 | `_\ 。 15 | 16 | PyODPS DataFrame 尽管看起来和 pandas 形似，但并不是 pandas。pandas 的功能，例如完整的 17 | Series 支持、Index 支持、按行读取数据、多 DataFrame 按 iloc 横向合并等，PyODPS DataFrame 18 | 并不支持。因而使用前请参考文档确定你的写法是否被支持。 19 | 20 | 21 | 22 | .. toctree:: 23 | :maxdepth: 1 24 | 25 | df-quickstart 26 | df-basic 27 | df-element 28 | df-agg 29 | df-sort-distinct-apply 30 | df-merge 31 | df-window 32 | df-plot 33 | df-debug-instruction 34 | -------------------------------------------------------------------------------- /docs/source/faq-ext.rst: -------------------------------------------------------------------------------- 1 | .. rubric:: 安装失败 / 出现问题 2 | 3 | 请参考 `PyODPS 安装常见问题解决 `_ 。 4 | 5 | .. rubric:: 提示 Project not found 6 | 7 | Endpoint配置不对，详细配置参考 8 | `MaxCompute 开通 Region 和服务连接对照表 `_ 。 9 | 此外还需要注意 ODPS 入口对象参数位置是否填写正确。 10 | 11 | .. rubric:: 如何手动指定 Tunnel Endpoint 12 | :name: faq_tunnel_endpoint 13 | 14 | 可以使用下面的方法创建带有 Tunnel Endpoint 的 ODPS 入口（参数值请自行替换，不包含星号）： 15 | 16 | .. code-block:: python 17 | 18 | import os 19 | from odps import ODPS 20 | # 保证 ALIBABA_CLOUD_ACCESS_KEY_ID 环境变量设置为用户 Access Key ID， 21 | # ALIBABA_CLOUD_ACCESS_KEY_SECRET 环境变量设置为用户 Access Key Secret， 22 | # 不建议直接使用 Access Key ID / Access Key Secret 字符串 23 | o = ODPS( 24 | os.getenv('ALIBABA_CLOUD_ACCESS_KEY_ID'), 25 | os.getenv('ALIBABA_CLOUD_ACCESS_KEY_SECRET'), 26 | project='**your-project**', 27 | endpoint='**your-endpoint**', 28 | tunnel_endpoint='**your-tunnel-endpoint**', 29 | ) 30 | -------------------------------------------------------------------------------- /docs/source/installation-ext.rst: -------------------------------------------------------------------------------- 1 | .. _install: 2 | 3 | ************** 4 | 安装指南 5 | ************** 6 | 7 | 如果能访问外网，推荐使用 pip 安装。较新版本的 Python 通常自带 pip。如果你的 Python 不包含 pip，可以参考 8 | `地址 `_ 安装，推荐使用 `阿里云镜像 `_ 9 | 加快下载速度。 10 | 11 | 接着升级 pip 和 setuptools 的版本： 12 | 13 | .. code-block:: sh 14 | 15 | pip install -U pip setuptools 16 | 17 | 此后可以安装 PyODPS： 18 | 19 | .. code-block:: sh 20 | 21 | pip install pyodps 22 | 23 | 如果安装时出现 `urllib3 v2.0 only supports OpenSSL 1.1.1+` 的报错，需要先安装一个兼容旧版 24 | OpenSSL 的 urllib3 版本 25 | 26 | .. code-block:: sh 27 | 28 | pip install urllib3\<2.0 29 | 30 | 此后再安装 PyODPS。 31 | 32 | 检查安装完成： 33 | 34 | .. code-block:: sh 35 | 36 | python -c "from odps import ODPS" 37 | 38 | 如果使用的python不是系统默认的python版本，安装完pip则可以： 39 | 40 | .. code-block:: sh 41 | 42 | /home/tops/bin/python2.7 -m pip install -U pip setuptools 43 | 44 | 其余步骤类似。 45 | -------------------------------------------------------------------------------- /docs/source/locale/en/LC_MESSAGES/api.po: -------------------------------------------------------------------------------- 1 | # SOME DESCRIPTIVE TITLE. 2 | # Copyright (C) 2014-2018, The Alibaba Group Holding Ltd. 3 | # This file is distributed under the same license as the PyODPS package. 4 | # FIRST AUTHOR , 2018. 5 | # 6 | #, fuzzy 7 | msgid "" 8 | msgstr "" 9 | "Project-Id-Version: PyODPS 0.7.16\n" 10 | "Report-Msgid-Bugs-To: \n" 11 | "POT-Creation-Date: 2018-04-19 17:37+0800\n" 12 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 13 | "Last-Translator: FULL NAME \n" 14 | "Language-Team: LANGUAGE \n" 15 | "MIME-Version: 1.0\n" 16 | "Content-Type: text/plain; charset=utf-8\n" 17 | "Content-Transfer-Encoding: 8bit\n" 18 | "Generated-By: Babel 2.12.1\n" 19 | 20 | #: ../../source/api.rst:5 21 | msgid "API Reference" 22 | msgstr "" 23 | 24 | #: ../../source/api.rst:7 25 | msgid "Here we give automatically generated API documentation of PyODPS." 26 | msgstr "" 27 | 28 | -------------------------------------------------------------------------------- /docs/source/locale/en/LC_MESSAGES/base.po: -------------------------------------------------------------------------------- 1 | # SOME DESCRIPTIVE TITLE. 2 | # Copyright (C) 2014-2018, The Alibaba Group Holding Ltd. 3 | # This file is distributed under the same license as the PyODPS package. 4 | # FIRST AUTHOR , 2018. 5 | # 6 | #, fuzzy 7 | msgid "" 8 | msgstr "" 9 | "Project-Id-Version: PyODPS 0.7.16\n" 10 | "Report-Msgid-Bugs-To: \n" 11 | "POT-Creation-Date: 2025-01-15 13:24+0800\n" 12 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 13 | "Last-Translator: FULL NAME \n" 14 | "Language-Team: LANGUAGE \n" 15 | "MIME-Version: 1.0\n" 16 | "Content-Type: text/plain; charset=utf-8\n" 17 | "Content-Transfer-Encoding: 8bit\n" 18 | "Generated-By: Babel 2.16.0\n" 19 | 20 | #: ../../source/base.rst:5 21 | msgid "基本类型及操作" 22 | msgstr "Basic types and operations" 23 | 24 | #: ../../source/base.rst:7 25 | msgid "PyODPS 提供直接针对 ODPS 对象的基本操作接口，可通过符合 Python 习惯的编程方式操作 ODPS。" 26 | msgstr "" 27 | "Python on MaxCompute (PyODPS) provides basic operations for ODPS objects." 28 | " You can follow the Python syntax and use the MaxCompute operations." 29 | 30 | #: ../../source/base.rst:9 31 | msgid "我们会对这几部分来分别展开说明。" 32 | msgstr "The following sections describe basic operations for ODPS objects." 33 | 34 | -------------------------------------------------------------------------------- /docs/source/locale/en/LC_MESSAGES/db.po: -------------------------------------------------------------------------------- 1 | # SOME DESCRIPTIVE TITLE. 2 | # Copyright (C) 2014-2018, The Alibaba Group Holding Ltd. 3 | # This file is distributed under the same license as the PyODPS package. 4 | # FIRST AUTHOR , 2025. 5 | # 6 | #, fuzzy 7 | msgid "" 8 | msgstr "" 9 | "Project-Id-Version: PyODPS 0.12.3\n" 10 | "Report-Msgid-Bugs-To: \n" 11 | "POT-Creation-Date: 2025-01-15 17:52+0800\n" 12 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 13 | "Last-Translator: FULL NAME \n" 14 | "Language: en\n" 15 | "Language-Team: en \n" 16 | "Plural-Forms: nplurals=2; plural=(n != 1);\n" 17 | "MIME-Version: 1.0\n" 18 | "Content-Type: text/plain; charset=utf-8\n" 19 | "Content-Transfer-Encoding: 8bit\n" 20 | "Generated-By: Babel 2.16.0\n" 21 | 22 | #: ../../source/db.rst:5 23 | msgid "数据库接口" 24 | msgstr "Database interfaces" 25 | 26 | #: ../../source/db.rst:7 27 | msgid "" 28 | "PyODPS 提供了 DBAPI 和 SQLAlchemy 支持，可通过 DBAPI 和 SQLAlchemy 操作 " 29 | "MaxCompute，\\ 进而与其他第三方包或者环境互操作。下面的文档介绍了如何通过" 30 | " PyODPS 使用 DBAPI 和 SQLAlchemy 操作 MaxCompute。" 31 | msgstr "" 32 | "PyODPS provides DBAPI and SQLAlchemy support, which allows you to operate" 33 | " MaxCompute through DBAPI and SQLAlchemy. The following document " 34 | "describes how to use DBAPI and SQLAlchemy to operate MaxCompute with " 35 | "PyODPS." 36 | 37 | -------------------------------------------------------------------------------- /docs/source/locale/en/LC_MESSAGES/platform.po: -------------------------------------------------------------------------------- 1 | # SOME DESCRIPTIVE TITLE. 2 | # Copyright (C) 2014-2018, The Alibaba Group Holding Ltd. 3 | # This file is distributed under the same license as the PyODPS package. 4 | # FIRST AUTHOR , 2018. 5 | # 6 | #, fuzzy 7 | msgid "" 8 | msgstr "" 9 | "Project-Id-Version: PyODPS 0.7.16\n" 10 | "Report-Msgid-Bugs-To: \n" 11 | "POT-Creation-Date: 2018-04-26 11:34+0800\n" 12 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 13 | "Last-Translator: FULL NAME \n" 14 | "Language-Team: LANGUAGE \n" 15 | "MIME-Version: 1.0\n" 16 | "Content-Type: text/plain; charset=utf-8\n" 17 | "Content-Transfer-Encoding: 8bit\n" 18 | "Generated-By: Babel 2.12.1\n" 19 | 20 | #: ../../source/platform.rst:5 21 | msgid "工具平台使用指南" 22 | msgstr "Platform instructions" 23 | 24 | #: ../../source/platform.rst:7 25 | msgid "" 26 | "PyODPS 可在 DataWorks 等数据开发平台中作为节点调用。这些平台提供了 PyODPS" 27 | " 运行环境，**不需要** 手动创建 ODPS 入口对象，免除了手动配置的麻烦，而且" 28 | "还提供了调度执行的能力。对于想从平台迁移到自行部署 PyODPS 环境的用户，" 29 | "下面也提供了迁移注意事项。" 30 | msgstr "" 31 | "Python on MaxCompute (PyODPS) can be used as a node on data development " 32 | "platforms such as DataWorks. These platforms provide the PyODPS running " 33 | "environment and allow scheduling and execution. Therefore, you **do not " 34 | "need** to manually create the ODPS object. To migrate from these " 35 | "platforms to local PyODPS, see the following migration instructions:" 36 | 37 | -------------------------------------------------------------------------------- /docs/source/platform.rst: -------------------------------------------------------------------------------- 1 | .. _platform: 2 | 3 | *********************** 4 | 工具平台使用指南 5 | *********************** 6 | 7 | PyODPS 可在 DataWorks 等数据开发平台中作为节点调用。这些平台提供了 PyODPS 运行环境，**不需要** 8 | 手动创建 ODPS 入口对象，免除了手动配置的麻烦，而且还提供了调度执行的能力。对于想从平台迁移到自行部署 PyODPS 9 | 环境的用户，下面也提供了迁移注意事项。 10 | 11 | .. toctree:: 12 | :maxdepth: 1 13 | 14 | platform-d2 15 | platform-pai-studio-int 16 | -------------------------------------------------------------------------------- /docs/source/pyodps-pack-minikube.rst: -------------------------------------------------------------------------------- 1 | .. _pack_minikube: 2 | 3 | 使用 Minikube 作为 Docker 环境 4 | _______________________________ 5 | `Minikube `_ 是一种常用的 Docker Desktop 替代环境。与 Docker Desktop 或者 6 | Rancher Desktop 直接通过图形界面启动不同，Minikube 需要通过命令行启动并手动配置环境。 7 | 8 | 依照 `这篇文档 `_ 完成安装 Minikube 后，启动 Minikube： 9 | 10 | .. code-block:: bash 11 | 12 | minikube start 13 | 14 | 此后，需要手动设置 Minikube 需要的环境变量。MacOS 用户可以使用 15 | 16 | .. code-block:: bash 17 | 18 | eval $(minikube -p minikube docker-env) 19 | 20 | 此后，即可在当前 Shell 会话中使用 pyodps-pack 进行打包。如果启动新的 Shell 会话，你可能需要重新配置环境变量。 21 | 22 | 对于 Windows 用户，可能需要使用 HyperV 作为默认 VM 驱动，参见 `这篇文档 `_： 23 | 24 | .. code-block:: batch 25 | 26 | minikube start --driver=hyperv 27 | 28 | 此后为当前 Shell 配置环境变量。如果你使用的是 CMD，可以使用下面的命令： 29 | 30 | .. code-block:: batch 31 | 32 | @FOR /f "tokens=*" %i IN ('minikube -p minikube docker-env --shell cmd') DO @%i 33 | 34 | 如果你使用的是 Powershell，可以使用下面的命令： 35 | 36 | .. code-block:: powershell 37 | 38 | & minikube -p minikube docker-env --shell powershell | Invoke-Expression 39 | 40 | 关于如何使用 Minikube 的进一步信息请参考 `Minikube 文档 `_ 。 41 | -------------------------------------------------------------------------------- /examples/storage_api/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from odps.apis.storage_api.tests.util import * 16 | -------------------------------------------------------------------------------- /examples/storage_api/commit_write_session.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import sys 17 | 18 | from util import * 19 | 20 | from odps.apis.storage_api import * 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | def commit_write_session(session_id, commit_msg): 26 | client = get_arrow_client() 27 | req = SessionRequest(session_id=session_id) 28 | resp = client.commit_write_session(req, [commit_msg]) 29 | 30 | if resp.status == Status.WAIT: 31 | resp = client.get_write_session(req) 32 | 33 | if resp.session_status != SessionStatus.COMMITTED: 34 | logger.info("Fail to commit write session") 35 | return 36 | 37 | logger.info("Commit write session success") 38 | 39 | 40 | if __name__ == "__main__": 41 | logging.basicConfig( 42 | format="%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s", 43 | level=logging.INFO, 44 | ) 45 | if len(sys.argv) != 3: 46 | raise ValueError("Please provide session id and commit message\n") 47 | 48 | session_id = sys.argv[1] 49 | commit_msg = sys.argv[2] 50 | commit_write_session(session_id, commit_msg) 51 | -------------------------------------------------------------------------------- /examples/storage_api/create_write_session.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | 17 | from util import * 18 | 19 | from odps.apis.storage_api import * 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | def create_write_session(): 25 | client = get_arrow_client() 26 | req = TableBatchWriteRequest(partition_spec="pt=test_write_1") 27 | resp = client.create_write_session(req) 28 | 29 | if resp.status != Status.OK: 30 | logger.info("Create write session failed") 31 | return 32 | 33 | logger.info(resp.session_id) 34 | 35 | 36 | if __name__ == "__main__": 37 | logging.basicConfig( 38 | format="%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s", 39 | level=logging.INFO, 40 | ) 41 | create_write_session() 42 | -------------------------------------------------------------------------------- /examples/storage_api/get_read_session.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import sys 17 | 18 | from util import * 19 | 20 | from odps.apis.storage_api import * 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | def check_session_status(session_id): 26 | client = get_arrow_client() 27 | req = SessionRequest(session_id=session_id) 28 | resp = client.get_read_session(req) 29 | 30 | if resp.status != Status.OK: 31 | logger.info("Get read session failed") 32 | return 33 | 34 | if resp.session_status == SessionStatus.NORMAL: 35 | logger.info("Read session id: " + resp.session_id) 36 | else: 37 | logger.info("Session status is not expected") 38 | 39 | 40 | if __name__ == "__main__": 41 | logging.basicConfig( 42 | format="%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s", 43 | level=logging.INFO, 44 | ) 45 | if len(sys.argv) != 2: 46 | raise ValueError("Please provide session id") 47 | 48 | session_id = sys.argv[1] 49 | check_session_status(session_id) 50 | -------------------------------------------------------------------------------- /examples/storage_api/get_write_session.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import sys 17 | 18 | from util import * 19 | 20 | from odps.apis.storage_api import * 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | def check_session_status(session_id): 26 | client = get_arrow_client() 27 | req = SessionRequest(session_id=session_id) 28 | resp = client.get_write_session(req) 29 | 30 | if resp.status != Status.OK: 31 | logger.info("Get write session failed") 32 | return 33 | 34 | if resp.session_status == SessionStatus.NORMAL: 35 | logger.info("Write session id: " + resp.session_id) 36 | else: 37 | logger.info("Session status is not expected") 38 | 39 | 40 | if __name__ == "__main__": 41 | logging.basicConfig( 42 | format="%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s", 43 | level=logging.INFO, 44 | ) 45 | if len(sys.argv) != 2: 46 | raise ValueError("Please provide session id") 47 | 48 | session_id = sys.argv[1] 49 | check_session_status(session_id) 50 | -------------------------------------------------------------------------------- /examples/storage_api/util.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | from odps import ODPS 18 | from odps.apis.storage_api import * 19 | 20 | o = ODPS( 21 | os.getenv("CLOUD_ACCESS_KEY_ID"), 22 | os.getenv("CLOUD_ACCESS_KEY_SECRET"), 23 | project="your-default-project", 24 | endpoint="your-end-point", 25 | ) 26 | 27 | table = "" 28 | quota_name = "" 29 | 30 | 31 | def get_arrow_client(): 32 | odps_table = o.get_table(table) 33 | client = StorageApiArrowClient(odps=o, table=odps_table, quota_name=quota_name) 34 | 35 | return client 36 | -------------------------------------------------------------------------------- /hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | STAGE_FILES=$(git diff --cached --name-only --diff-filter=ACM) 4 | 5 | echo 'check sensitive information ...' 6 | FAIL=0 7 | for FILE in $STAGE_FILES 8 | do 9 | grep --color -Hni -E "(ssh-rsa|authorized_keys|id_dsa|ssh-keygen)" $FILE && FAIL=1 10 | grep --color -Hni -E "(private key|secret|signature|accessid|access_id|access_key|accesskey|access_|password)(.*?)(\=|\:)(\s*)(\'|\")[^\$^%][^)]+(\'|\")[^)]*$" $FILE && FAIL=1 11 | grep --color -Hni -E "jdbc\:odps\:.*?accessId\=[^\.]+)" $FILE && FAIL=1 12 | done 13 | 14 | if [ ${FAIL} == 0 ]; then 15 | echo 'check sensitive information ... passed' 16 | exit 0 17 | else 18 | echo 'check sensitive information ... failed' 19 | exit 1 20 | fi 21 | -------------------------------------------------------------------------------- /misc/avg.input: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | 3 4 | 4 5 | 5 6 | -------------------------------------------------------------------------------- /misc/data: -------------------------------------------------------------------------------- 1 | hello 2 | world 3 | -------------------------------------------------------------------------------- /misc/explode.input: -------------------------------------------------------------------------------- 1 | a|b|c 2 | d|e|f 3 | NULL 4 | -------------------------------------------------------------------------------- /misc/license_head.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # this script add license head to python source file 4 | 5 | use strict; 6 | use warnings; 7 | 8 | my $license = <<"EOF"; 9 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 10 | # 11 | # Licensed under the Apache License, Version 2.0 (the "License"); 12 | # you may not use this file except in compliance with the License. 13 | # You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, software 18 | # distributed under the License is distributed on an "AS IS" BASIS, 19 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 20 | # See the License for the specific language governing permissions and 21 | # limitations under the License. 22 | 23 | EOF 24 | 25 | my $tmp = "tmp.$$"; 26 | for my $input ( @ARGV) { 27 | open IN, "<", $input or warn "failed to open $input\n"; 28 | open OUT, ">", $tmp or die "failed to open $tmp\n"; 29 | my $line = ; 30 | if ($line =~ m/^# -\*- coding:/) { 31 | print OUT $line; 32 | print OUT $license; 33 | } elsif ($line =~ m/^#!\/usr/) { 34 | print OUT $line; 35 | $line = ; 36 | if ($line =~ m/^# -\*- coding:/) { 37 | print OUT $line; 38 | print OUT $license; 39 | } else { 40 | print OUT $license; 41 | print OUT $line; 42 | } 43 | } else { 44 | print OUT $license; 45 | print OUT $line; 46 | } 47 | while () { 48 | print OUT $_; 49 | } 50 | close OUT; 51 | close IN; 52 | 0 == system("mv $tmp $input") or die "failed to overwrite $input\n"; 53 | } 54 | -------------------------------------------------------------------------------- /misc/plus.input: -------------------------------------------------------------------------------- 1 | 11 2 | 32 3 | -------------------------------------------------------------------------------- /misc/python: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export PYTHONPATH=.:misc 4 | 5 | python $@ 6 | -------------------------------------------------------------------------------- /notebooks/nb_init.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """ 16 | Notebook initializing code 17 | You can edit your ipython_config.py to detect and add this file to 18 | exec_lines to help create pre-defined objects. 19 | """ 20 | import sys 21 | 22 | from six.moves.configparser import ConfigParser 23 | 24 | sys.path.append('../') 25 | 26 | config = ConfigParser() 27 | config.read('../odps/tests/test.conf') 28 | 29 | access_id = config.get('odps', 'access_id') 30 | secret_access_key = config.get('odps', 'secret_access_key') 31 | project = config.get('odps', 'project') 32 | endpoint = config.get('odps', 'endpoint') 33 | -------------------------------------------------------------------------------- /odps/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | from ._version import __version__ 18 | 19 | __all__ = ["ODPS", "DataFrame", "options"] 20 | 21 | if sys.version_info[0] == 2 and sys.version_info[:2] < (2, 7): 22 | raise Exception("pyodps supports python 2.7+ (including python 3+).") 23 | 24 | from .config import options 25 | from .core import ODPS 26 | from .df import DataFrame, NullScalar, RandomScalar, Scalar 27 | from .inter import enter, list_rooms, setup, teardown 28 | from .lib import Monthdelta 29 | from .utils import show_versions 30 | from .utils import write_log as log 31 | 32 | try: 33 | from .ipython import load_ipython_extension 34 | except ImportError: 35 | pass 36 | try: 37 | from sqlalchemy.dialects import registry 38 | 39 | registry.register("odps", "odps.sqlalchemy_odps", "ODPSDialect") 40 | except ImportError: 41 | pass 42 | 43 | 44 | def install_plugins(): 45 | try: 46 | from .ml import install_mixin, install_plugin 47 | except (ImportError, SyntaxError): 48 | pass 49 | 50 | 51 | install_plugins() 52 | del install_plugins 53 | -------------------------------------------------------------------------------- /odps/_version.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | version_info = (0, 12, 4) 16 | _num_index = max(idx if isinstance(v, int) else 0 for idx, v in enumerate(version_info)) 17 | __version__ = ".".join(map(str, version_info[: _num_index + 1])) + "".join( 18 | version_info[_num_index + 1 :] 19 | ) 20 | -------------------------------------------------------------------------------- /odps/apis/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /odps/apis/storage_api/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | if sys.version_info[0] == 3: 18 | from .storage_api import * 19 | -------------------------------------------------------------------------------- /odps/apis/storage_api/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | import pytest 18 | 19 | from ...config import options 20 | from ...tests.core import tn 21 | 22 | if sys.version_info[0] >= 3: 23 | from .storage_api import StorageApiArrowClient 24 | else: 25 | StorageApiArrowClient = None 26 | 27 | 28 | _test_table_id = 0 29 | 30 | 31 | @pytest.fixture 32 | def storage_api_client(odps): 33 | global _test_table_id 34 | 35 | options.enable_schema = True 36 | 37 | test_table_name = tn("test_halo_common_table_" + str(_test_table_id)) 38 | _test_table_id += 1 39 | odps.delete_table(test_table_name, if_exists=True) 40 | table = odps.create_table( 41 | test_table_name, 42 | ("a BIGINT, b BIGINT, c BIGINT, d BIGINT", "pt string"), 43 | if_not_exists=True, 44 | ) 45 | try: 46 | yield StorageApiArrowClient(odps, table) 47 | finally: 48 | table.drop(async_=True) 49 | options.enable_schema = False 50 | -------------------------------------------------------------------------------- /odps/apis/storage_api/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | import pytest 18 | 19 | try: 20 | import pyarrow as pa 21 | except ImportError: 22 | pa = None 23 | pytestmark = pytest.mark.skip("Need pyarrow to run this test") 24 | 25 | if sys.version_info[0] == 2: 26 | pytestmark = pytest.mark.skip("Need python3.5+ to run this test") 27 | -------------------------------------------------------------------------------- /odps/apis/storage_api/tests/data_item.conf: -------------------------------------------------------------------------------- 1 | { 2 | "with_partition": { 3 | "val": 5, 4 | "batch_size": 4096, 5 | "batch_count": 300, 6 | "partition": "pt=test_write_1" 7 | }, 8 | "without_partition": { 9 | "val": 5, 10 | "batch_size": 4096, 11 | "batch_count": 300 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /odps/df/backends/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/backends/context.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | class ExecuteContext(object): 19 | def __init__(self): 20 | self._expr_id_cached_data = dict() 21 | 22 | def cache(self, expr, data): 23 | if data is not None: 24 | self._expr_id_cached_data[expr._id] = data 25 | 26 | def is_cached(self, expr): 27 | return expr._id in self._expr_id_cached_data 28 | 29 | def get_cached(self, expr): 30 | return self._expr_id_cached_data[expr._id] 31 | 32 | def uncache(self, expr): 33 | if expr._id in self._expr_id_cached_data: 34 | del self._expr_id_cached_data[expr._id] 35 | 36 | context = ExecuteContext() 37 | -------------------------------------------------------------------------------- /odps/df/backends/errors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | class CompileError(Exception): 19 | pass 20 | 21 | 22 | class NoBackendFound(Exception): 23 | pass 24 | 25 | 26 | class DagDependencyError(Exception): 27 | pass 28 | -------------------------------------------------------------------------------- /odps/df/backends/odpssql/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/backends/odpssql/models.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | class MemCacheReference(object): 19 | __slots__ = 'expr_id', 'ref_name' 20 | 21 | def __init__(self, expr_id, ref_name): 22 | self.expr_id = expr_id 23 | self.ref_name = ref_name -------------------------------------------------------------------------------- /odps/df/backends/odpssql/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/backends/optimize/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .core import Optimizer 18 | -------------------------------------------------------------------------------- /odps/df/backends/optimize/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/backends/pd/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/backends/pd/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/backends/pd/tests/test_compiler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .....tests.core import pandas_case 18 | from ....expr.expressions import * 19 | from .... import types 20 | from ...context import ExecuteContext 21 | from ..compiler import PandasCompiler 22 | 23 | 24 | @pandas_case 25 | def test_pandas_compilation(): 26 | import pandas as pd 27 | import numpy as np 28 | 29 | df = pd.DataFrame(np.arange(9).reshape(3, 3), columns=list('abc')) 30 | 31 | schema = TableSchema.from_lists(list('abc'), [types.int8] * 3) 32 | expr = CollectionExpr(_source_data=df, _schema=schema) 33 | 34 | expr = expr['a', 'b'] 35 | _ = ExecuteContext() 36 | 37 | compiler = PandasCompiler(expr.to_dag()) 38 | dag = compiler.compile(expr) 39 | 40 | assert len(dag._graph) == 4 41 | topos = dag.topological_sort() 42 | assert isinstance(topos[0][0], CollectionExpr) 43 | assert isinstance(topos[1][0], Column) 44 | assert isinstance(topos[2][0], Column) 45 | assert isinstance(topos[3][0], ProjectCollectionExpr) 46 | -------------------------------------------------------------------------------- /odps/df/backends/seahawks/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/backends/seahawks/engine.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from ..sqlalchemy.engine import SQLAlchemyEngine 18 | from ..odpssql.types import df_schema_to_odps_schema 19 | from .compiler import SeahawksCompiler 20 | from .models import SeahawksTable 21 | 22 | 23 | class SeahawksEngine(SQLAlchemyEngine): 24 | 25 | def __init__(self, odps): 26 | super(SeahawksEngine, self).__init__(odps=odps) 27 | 28 | def _compile(self, expr, dag): 29 | compiler = SeahawksCompiler(dag, self._odps) 30 | return compiler.compile(expr) 31 | 32 | def _status_ui(self, ui): 33 | ui.status('Try to execute by seahawks...') 34 | 35 | @classmethod 36 | def _create_table(cls, table_name, sa, expr): 37 | from .ext import CreateTempTableAs 38 | 39 | return CreateTempTableAs(table_name, sa, expr) 40 | 41 | def _get_table(self, table_name, expr_dag, bind=None): 42 | return SeahawksTable(name=table_name, 43 | schema=expr_dag.root.schema) 44 | -------------------------------------------------------------------------------- /odps/df/backends/seahawks/ext.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | from sqlalchemy.ext.compiler import compiles 19 | 20 | from ..sqlalchemy.ext import SACreateTempTableAs 21 | 22 | 23 | class CreateTempTableAs(SACreateTempTableAs): 24 | 25 | def __init__(self, name, query, expr): 26 | super(CreateTempTableAs, self).__init__(name, query) 27 | self.expr = expr 28 | 29 | 30 | @compiles(CreateTempTableAs) 31 | def create_temp_table_as(element, compiler, **kw): 32 | return 'CREATE TEMP TABLE %s WITH(appendonly=true, orientation=heap, bucketnum=1) ' \ 33 | 'AS %s DISTRIBUTED BY (%s)' % ( 34 | element.name, 35 | compiler.process(element.query), 36 | ', '.join(element.expr.schema.names) # we just distribute by all columns 37 | ) 38 | -------------------------------------------------------------------------------- /odps/df/backends/seahawks/models.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | from odps.models import Table 19 | 20 | 21 | class MockProject(object): 22 | def __init__(self): 23 | self.name = 'mocked_project' 24 | 25 | 26 | class SeahawksTable(Table): 27 | def __init__(self, **kwargs): 28 | super(SeahawksTable, self).__init__(**kwargs) 29 | 30 | self._loaded = True 31 | 32 | @property 33 | def project(self): 34 | return MockProject() 35 | 36 | def reload(self): 37 | return 38 | -------------------------------------------------------------------------------- /odps/df/backends/seahawks/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/backends/sqlalchemy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/backends/sqlalchemy/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/backends/sqlalchemy/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | def get_sa_backend(table): 19 | return table.metadata.bind.name 20 | 21 | 22 | def is_sa_backend_postgres(table): 23 | return get_sa_backend(table) == 'postgresql' -------------------------------------------------------------------------------- /odps/df/backends/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """ 17 | Created on 2015-12-24 18 | 19 | @author: xuye.qin 20 | """ 21 | -------------------------------------------------------------------------------- /odps/df/engines.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .backends.engine import get_default_engine -------------------------------------------------------------------------------- /odps/df/expr/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/expr/errors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | class ExpressionError(Exception): 19 | pass -------------------------------------------------------------------------------- /odps/df/expr/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/expr/tests/conftest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | 18 | from ....config import options 19 | 20 | 21 | @pytest.fixture(autouse=True) 22 | def no_test_interactive(): 23 | old_interactive = options.interactive 24 | try: 25 | options.interactive = False 26 | yield 27 | finally: 28 | options.interactive = old_interactive 29 | -------------------------------------------------------------------------------- /odps/df/expr/tests/core.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ....models import Table 16 | 17 | 18 | class MockProject(object): 19 | def __init__(self): 20 | self.name = 'mocked_project' 21 | 22 | 23 | class MockTable(Table): 24 | __slots__ = "_mock_project", 25 | 26 | def __init__(self, **kwargs): 27 | from ....core import ODPS 28 | 29 | super(MockTable, self).__init__(**kwargs) 30 | 31 | self._loaded = True 32 | self._mock_project = MockProject() 33 | if getattr(self, "_client", None) is not None: 34 | client = self._client 35 | odps_entry = ODPS( 36 | account=client._account, project=client.project, 37 | endpoint=client.endpoint, schema=client.schema, 38 | ) 39 | self._mock_project.odps = odps_entry 40 | 41 | @property 42 | def project(self): 43 | return self._mock_project 44 | 45 | -------------------------------------------------------------------------------- /odps/df/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/tests/test_examples.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .. import examples 17 | 18 | 19 | def test_examples(odps, tunnel): 20 | df = examples.create_iris(odps, tunnel=tunnel) 21 | assert len(df.columns) == 5 22 | 23 | df = examples.create_ionosphere(odps, tunnel=tunnel) 24 | assert len(df.columns) == 35 25 | -------------------------------------------------------------------------------- /odps/df/tools/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .plotting import * 18 | from .libtools import * 19 | -------------------------------------------------------------------------------- /odps/df/tools/lib/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | from .bloomfilter import BloomFilter 19 | from .hll import HyperLogLog 20 | -------------------------------------------------------------------------------- /odps/df/tools/lib/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/df/tools/lib/tests/test_hll.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import pytest 18 | 19 | from .....compat import irange as xrange 20 | from .. import HyperLogLog 21 | 22 | 23 | def test_hll(): 24 | hll = HyperLogLog(0.05) 25 | buf = hll.buffer() 26 | 27 | for i in xrange(10000): 28 | hll(buf, str(i)) 29 | 30 | assert pytest.approx(hll.getvalue(buf) / float(10000), 0.1) == 1 31 | 32 | for i in xrange(100000, 200000): 33 | hll(buf, str(i)) 34 | 35 | assert pytest.approx(hll.getvalue(buf) / 110000, 0.2) == 1 36 | 37 | buf2 = hll.buffer() 38 | 39 | for i in xrange(10000): 40 | hll(buf2, str(i)) 41 | 42 | hll.merge(buf, buf2) 43 | 44 | assert pytest.approx(hll.getvalue(buf) / 110000, 0.2) == 1 45 | -------------------------------------------------------------------------------- /odps/distcache.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | """ 17 | This module provides stubs for resource references in SQL UDFs. 18 | Actual implementations are provided by MaxCompute SQL Engine 19 | which are NOT available within PyODPS code base. DO NOT USE 20 | these methods in your local environments or environments other 21 | than MaxCompute SQL UDFs as they will NEVER be functional. 22 | """ 23 | 24 | 25 | def get_cache_file(name): 26 | return open(name) 27 | 28 | 29 | def get_cache_table(name): 30 | pass 31 | 32 | 33 | def get_cache_archive(name, relative_path="."): 34 | pass 35 | 36 | 37 | def get_cache_tabledesc(name): 38 | pass 39 | 40 | 41 | def get_cache_tableinfo(name): 42 | pass 43 | -------------------------------------------------------------------------------- /odps/examples/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .tables import * 17 | -------------------------------------------------------------------------------- /odps/expressions/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .core import parse 16 | -------------------------------------------------------------------------------- /odps/expressions/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /odps/ipython/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from . import completer, magics 18 | 19 | 20 | def load_ipython_extension(ipython): 21 | for mod in (magics, completer): 22 | if hasattr(magics, "load_ipython_extension"): 23 | mod.load_ipython_extension(ipython) 24 | -------------------------------------------------------------------------------- /odps/ipython/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/lib/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # make it compatible to imports on PyODPS between 0.11.4 and 0.11.5. 18 | import requests # noqa 19 | 20 | from .ext_types import Monthdelta 21 | -------------------------------------------------------------------------------- /odps/lib/futures/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2009 Brian Quinlan. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | 4 | """Execute computations asynchronously using threads or processes.""" 5 | 6 | __author__ = "Brian Quinlan (brian@sweetapp.com)" 7 | 8 | from ._base import ( 9 | FIRST_COMPLETED, 10 | FIRST_EXCEPTION, 11 | ALL_COMPLETED, 12 | CancelledError, 13 | TimeoutError, 14 | Future, 15 | Executor, 16 | wait, 17 | as_completed, 18 | ) 19 | from .thread import ThreadPoolExecutor 20 | 21 | try: 22 | from .process import ProcessPoolExecutor 23 | except ImportError: 24 | # some platforms don't have multiprocessing 25 | pass 26 | -------------------------------------------------------------------------------- /odps/lib/tblib/decorators.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from functools import wraps 3 | 4 | from ..six import PY3, raise_from, reraise 5 | from . import Traceback 6 | 7 | 8 | class Error(object): 9 | def __init__(self, exc_type, exc_value, traceback): 10 | self.exc_type = exc_type 11 | self.exc_value = exc_value 12 | self.__traceback = Traceback(traceback) 13 | 14 | @property 15 | def traceback(self): 16 | return self.__traceback.as_traceback() 17 | 18 | def reraise(self): 19 | if PY3: 20 | raise_from(self.exc_value.with_traceback(self.traceback), None) 21 | else: 22 | reraise(self.exc_type, self.exc_value, self.traceback) 23 | 24 | 25 | def return_error(func, exc_type=Exception): 26 | @wraps(func) 27 | def return_exceptions_wrapper(*args, **kwargs): 28 | try: 29 | return func(*args, **kwargs) 30 | except exc_type: 31 | return Error(*sys.exc_info()) 32 | 33 | return return_exceptions_wrapper 34 | 35 | 36 | returns_error = ( 37 | return_errors 38 | ) = returns_errors = return_error # cause I make too many typos 39 | 40 | 41 | @return_error 42 | def apply_with_return_error(args): 43 | """ 44 | args is a tuple where the first argument is a callable. 45 | 46 | eg:: 47 | 48 | apply_with_return_error((func, 1, 2, 3)) - this will call func(1, 2, 3) 49 | 50 | """ 51 | return args[0](*args[1:]) 52 | -------------------------------------------------------------------------------- /odps/lib/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/lib/tzlocal/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2011-2017 Lennart Regebro 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /odps/lib/tzlocal/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | if sys.platform == 'win32': 3 | from .win32 import get_localzone, reload_localzone 4 | else: 5 | from .unix import get_localzone, reload_localzone 6 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/actors/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .session import CupidSessionManagerActor 16 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/dataframe/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .datasource import read_odps_table, DataFrameReadTable, ReadODPSTableHeadRule 18 | from .datastore import write_odps_table, DataFrameWriteTable 19 | 20 | from mars.optimizes.runtime.dataframe import DataFrameRuntimeOptimizer 21 | 22 | DataFrameRuntimeOptimizer.register_rule(ReadODPSTableHeadRule) 23 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/deploy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/deploy/utils.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import json 18 | import time 19 | 20 | from ....utils import to_str 21 | 22 | 23 | def wait_all_schedulers_ready(kv_store, scheduler_keys): 24 | schedulers = scheduler_keys.split(",") 25 | while True: 26 | scheduler_endpoints = [] 27 | for scheduler_key in schedulers: 28 | json_val = to_str(kv_store.get(scheduler_key)) 29 | if json_val: 30 | config = json.loads(to_str(json_val)) 31 | scheduler_endpoints.append(to_str(config["endpoint"])) 32 | if len(scheduler_endpoints) == len(schedulers): 33 | break 34 | time.sleep(1) 35 | return scheduler_endpoints 36 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/deploy/worker.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | 19 | from mars.deploy.kubernetes.worker import K8SWorkerApplication 20 | 21 | from .core import CupidServiceMixin 22 | 23 | logger = logging.getLogger(__name__) 24 | 25 | 26 | class CupidWorkerApplication(CupidServiceMixin, K8SWorkerApplication): 27 | pass 28 | 29 | 30 | main = CupidWorkerApplication() 31 | 32 | if __name__ == "__main__": 33 | main() 34 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/io/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .core import open, glob 16 | 17 | from . import local, oss 18 | 19 | del local, oss 20 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/io/core.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ....compat import urlparse 16 | 17 | _file_systems = dict() 18 | 19 | 20 | def _get_scheme(path): 21 | result = urlparse(path) 22 | return result.scheme if result.scheme else "file" 23 | 24 | 25 | def open(path, mode, **kwargs): 26 | scheme = _get_scheme(path) 27 | fs = _file_systems[scheme](**kwargs) 28 | return fs.open(path, mode) 29 | 30 | 31 | def glob(path, **kwargs): 32 | scheme = _get_scheme(path) 33 | fs = _file_systems[scheme](**kwargs) 34 | return fs.glob(path) 35 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/io/local.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import glob 17 | from ....compat import urlparse 18 | 19 | 20 | class LocalFileSystem(object): 21 | sep = os.sep 22 | 23 | def __init__(self, **__): 24 | self._work_dir = os.getcwd() 25 | 26 | def _normalize_path(self, path): 27 | parse_result = urlparse(path) 28 | path = parse_result.path 29 | if parse_result.netloc: 30 | path = os.path.join(parse_result.netloc, parse_result.path) 31 | 32 | if not os.path.isabs(path): 33 | return os.path.join(self._work_dir, path) 34 | return os.path.normpath(path) 35 | 36 | def glob(self, path): 37 | for p in glob.glob(self._normalize_path(path)): 38 | yield "file://" + p 39 | 40 | def open(self, path, mode="rb"): 41 | return open(self._normalize_path(path), mode) 42 | 43 | 44 | from . import core 45 | 46 | core._file_systems["file"] = LocalFileSystem 47 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/tensor/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import os 18 | import pkgutil 19 | import sys 20 | 21 | try: 22 | from .datasource import read_coo 23 | from .datastore import write_coo 24 | except ImportError as e: 25 | if "CI_MODE" in os.environ and pkgutil.find_loader("mars") is not None: 26 | if ( 27 | sys.version_info[0] > 2 28 | ): # todo remove this check when mars package under py27mu is fixed 29 | raise 30 | read_coo, write_coo = None, None 31 | 32 | __all__ = ["read_coo", "write_coo"] 33 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/tensor/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/mars_extension/legacy/web/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | try: 18 | from . import apihandlers 19 | except ImportError: 20 | pass 21 | -------------------------------------------------------------------------------- /odps/mars_extension/oscar/bearer_token_init.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | 18 | from ...utils import to_str 19 | 20 | 21 | def _load_bearer_token_environ(): 22 | from .cupid_service import CupidServiceClient 23 | 24 | if "CUPID_SERVICE_SOCKET" not in os.environ: 25 | return 26 | bearer_token = CupidServiceClient().get_bearer_token() 27 | os.environ["ODPS_BEARER_TOKEN"] = to_str(bearer_token) 28 | os.environ["ODPS_ENDPOINT"] = os.environ["ODPS_RUNTIME_ENDPOINT"] 29 | 30 | 31 | _load_bearer_token_environ() 32 | -------------------------------------------------------------------------------- /odps/mars_extension/oscar/dataframe/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .datasource import read_odps_table, DataFrameReadTable 18 | from .datastore import write_odps_table, DataFrameWriteTable 19 | -------------------------------------------------------------------------------- /odps/mars_extension/oscar/deploy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/mars_extension/oscar/deploy/config.yml: -------------------------------------------------------------------------------- 1 | "@inherits": "@mars/deploy/oscar/config.yml" 2 | cluster: 3 | backend: cupid_k8s 4 | storage: 5 | backends: [plasma] 6 | plasma: 7 | store_memory: 50% 8 | web: 9 | extra_discovery_modules: 10 | - "odps.mars_extension.oscar.web" 11 | third_party_modules: 12 | - "pyarrow.plasma" 13 | - "odps.mars_extension.oscar.bearer_token_init" -------------------------------------------------------------------------------- /odps/mars_extension/oscar/deploy/logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root,main,tornado,mars_deploy,mars_oscar,mars_dataframe,mars_learn,mars_tensor,mars_services,odps_mars 3 | 4 | [handlers] 5 | keys=stream_handler,null_handler 6 | 7 | [formatters] 8 | keys=formatter 9 | 10 | [logger_root] 11 | level=WARN 12 | handlers=stream_handler 13 | 14 | [logger_main] 15 | level=DEBUG 16 | handlers=stream_handler 17 | qualname=__main__ 18 | propagate=0 19 | 20 | [logger_tornado] 21 | level=WARN 22 | handlers=stream_handler 23 | qualname=tornado 24 | propagate=0 25 | 26 | [logger_mars_deploy] 27 | level=DEBUG 28 | handlers=stream_handler 29 | qualname=mars.deploy 30 | propagate=0 31 | 32 | [logger_mars_oscar] 33 | level=DEBUG 34 | handlers=stream_handler 35 | qualname=mars.oscar 36 | propagate=0 37 | 38 | [logger_mars_dataframe] 39 | level=DEBUG 40 | handlers=stream_handler 41 | qualname=mars.dataframe 42 | propagate=0 43 | 44 | [logger_mars_learn] 45 | level=DEBUG 46 | handlers=stream_handler 47 | qualname=mars.learn 48 | propagate=0 49 | 50 | [logger_mars_tensor] 51 | level=DEBUG 52 | handlers=stream_handler 53 | qualname=mars.tensor 54 | propagate=0 55 | 56 | [logger_mars_services] 57 | level=DEBUG 58 | handlers=stream_handler 59 | qualname=mars.services 60 | propagate=0 61 | 62 | [logger_odps_mars] 63 | level=DEBUG 64 | handlers=stream_handler 65 | qualname=odps.mars_extension 66 | propagate=0 67 | 68 | [handler_null_handler] 69 | class=NullHandler 70 | args=() 71 | 72 | [handler_stream_handler] 73 | class=StreamHandler 74 | formatter=formatter 75 | args=(sys.stderr,) 76 | 77 | [formatter_formatter] 78 | format=%(asctime)s %(name)-12s %(process)d %(levelname)-8s %(message)s 79 | -------------------------------------------------------------------------------- /odps/mars_extension/oscar/deploy/worker.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | 19 | from mars.deploy.kubernetes.worker import K8SWorkerCommandRunner 20 | 21 | from .core import CupidCommandRunnerMixin 22 | 23 | logger = logging.getLogger(__name__) 24 | 25 | 26 | class CupidWorkerCommandRunner(CupidCommandRunnerMixin, K8SWorkerCommandRunner): 27 | def __call__(self, *args, **kwargs): 28 | try: 29 | self.fix_protobuf_import() 30 | self.start_cupid_service() 31 | super().__call__(*args, **kwargs) 32 | except: 33 | import traceback 34 | 35 | traceback.print_exc() 36 | raise 37 | finally: 38 | self.stop_cupid_service() 39 | 40 | def parse_args(self, parser, argv, environ=None): 41 | self.fix_hyper_address() 42 | return super().parse_args(parser, argv, environ) 43 | 44 | async def start_services(self): 45 | await self.write_node_endpoint() 46 | await super().start_services() 47 | 48 | 49 | main = CupidWorkerCommandRunner() 50 | 51 | if __name__ == "__main__": 52 | main() 53 | -------------------------------------------------------------------------------- /odps/mars_extension/oscar/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/mars_extension/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/ml/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .expr import PmmlModel, TablesModel, install_mixin 17 | from .enums import FieldRole, FieldContinuity 18 | from .expr.mixin import merge_data 19 | from .runners import XFlowNodeRunner 20 | from .models import TablesModelObject, install_plugin 21 | 22 | try: 23 | from ..internal.ml import * 24 | except ImportError: 25 | pass 26 | -------------------------------------------------------------------------------- /odps/ml/algolib/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .loader import load_classifiers, load_process_algorithm 17 | from .objects import * 18 | 19 | __all__ = [ 20 | "load_classifiers", 21 | "load_process_algorithm", 22 | "AlgorithmDef", 23 | "XflowAlgorithmDef", 24 | "ParamDef", 25 | "PortDef", 26 | ] 27 | -------------------------------------------------------------------------------- /odps/ml/algolib/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/classifiers/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from ..algolib.loader import load_defined_algorithms 19 | 20 | 21 | load_defined_algorithms(sys.modules[__name__], "classifier") 22 | -------------------------------------------------------------------------------- /odps/ml/classifiers/_customize.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | def linear_svm_set_cost(algo, param_name, param_value): 18 | algo._parameters['positiveCost'].value = param_value 19 | algo._parameters['negativeCost'].value = param_value 20 | 21 | 22 | """ 23 | Random forests exporters 24 | """ 25 | 26 | 27 | def random_forest_algo_types(expr): 28 | algo_type = expr._params["algorithmType"] 29 | tree_num = int(expr._params["treeNum"]) 30 | tree_num = tree_num - 1 if tree_num != 0 else 0 31 | if algo_type == "mix": 32 | return None 33 | elif algo_type == "id3": 34 | return '%d,%d' % (tree_num, tree_num) 35 | elif algo_type == "c45": 36 | return "0,0" 37 | elif "cart": 38 | return "0,%d" % tree_num 39 | return None 40 | 41 | 42 | def random_forest_min_num_per(expr): 43 | ret = '-1' 44 | if expr._params['minNumPerInt'] is None: 45 | return ret 46 | 47 | min_num_per_int = expr._params['minNumPerInt'] 48 | if min_num_per_int.strip() != '': 49 | if int(min_num_per_int) in (-1, 0): 50 | return ret 51 | else: 52 | return int(min_num_per_int) / 100.0 53 | return str(ret) 54 | -------------------------------------------------------------------------------- /odps/ml/classifiers/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/clustering/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from ..algolib.loader import load_defined_algorithms 19 | 20 | 21 | load_defined_algorithms(sys.modules[__name__], "clustering") 22 | -------------------------------------------------------------------------------- /odps/ml/clustering/_customize.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from ..expr.exporters import get_ml_input 17 | 18 | 19 | """ 20 | K-Means related exporters 21 | """ 22 | 23 | 24 | def get_kmeans_input_append_col_idx(expr, input_name): 25 | data_obj = get_ml_input(expr, input_name) 26 | if data_obj is None: 27 | return None 28 | return ','.join([str(idx) for idx, f in enumerate(data_obj._ml_fields) if not f.is_partition]) 29 | -------------------------------------------------------------------------------- /odps/ml/clustering/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/enums.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from ..compat import Enum 17 | 18 | 19 | class FieldRole(Enum): 20 | FEATURE = 'FEATURE' 21 | LABEL = 'LABEL' 22 | WEIGHT = 'WEIGHT' 23 | GROUP_ID = 'GROUP_ID' 24 | PREDICTED_CLASS = 'PREDICTED_CLASS' 25 | PREDICTED_SCORE = 'PREDICTED_SCORE' 26 | PREDICTED_VALUE = 'PREDICTED_VALUE' 27 | PREDICTED_DETAIL = 'PREDICTED_DETAIL' 28 | 29 | 30 | class FieldContinuity(Enum): 31 | CONTINUOUS = 'CONTINUOUS' 32 | DISCRETE = 'DISCRETE' 33 | 34 | 35 | class PortType(Enum): 36 | DATA = 'DATA' 37 | MODEL = 'MODEL' 38 | 39 | 40 | class PortDirection(Enum): 41 | INPUT = 'INPUT' 42 | OUTPUT = 'OUTPUT' 43 | -------------------------------------------------------------------------------- /odps/ml/expr/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .core import AlgoExprMixin, AlgoCollectionExpr, MetricsResultExpr 18 | from .models import ODPSModelExpr, ModelDataCollectionExpr, PmmlModel, TablesModel 19 | 20 | from .mixin import install_mixin 21 | -------------------------------------------------------------------------------- /odps/ml/expr/models/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .base import ODPSModelExpr, ModelDataCollectionExpr, PmmlModel, TablesModel 18 | -------------------------------------------------------------------------------- /odps/ml/expr/models/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/ml/expr/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/feature/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from ..algolib.loader import load_defined_algorithms 19 | 20 | # required by autodoc of sphinx 21 | __all__ = [] 22 | 23 | load_defined_algorithms(sys.modules[__name__], "feature") 24 | -------------------------------------------------------------------------------- /odps/ml/feature/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | import types 18 | 19 | from .classification import * 20 | from .regression import * 21 | from .clustering import * 22 | from .scorer import SCORERS 23 | 24 | from ..algolib.loader import load_defined_algorithms 25 | 26 | # required by autodoc of sphinx 27 | __all__ = [k for k, v in globals().items() 28 | if not k.startswith('_') and isinstance(v, (types.FunctionType, type))] 29 | 30 | load_defined_algorithms(sys.modules[__name__], 'metrics') 31 | -------------------------------------------------------------------------------- /odps/ml/network/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from ..algolib.loader import load_defined_algorithms 19 | from ._customize import NetworkFieldRole 20 | 21 | # required by autodoc of sphinx 22 | __all__ = [] 23 | 24 | 25 | load_defined_algorithms(sys.modules[__name__], "network") 26 | -------------------------------------------------------------------------------- /odps/ml/network/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .core import Pipeline, FeatureUnion 17 | from .steps import AppendID, MergeColumns, Sample 18 | 19 | __all__ = ["Pipeline", "FeatureUnion"] 20 | -------------------------------------------------------------------------------- /odps/ml/pipeline/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/preprocess/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from ._customize import ( 19 | ReplaceNull, 20 | ReplaceEmpty, 21 | ReplaceNullEmpty, 22 | ReplaceCustom, 23 | ReplacePercentile, 24 | ReplaceZScore, 25 | ReplaceConfidence, 26 | ) 27 | 28 | from ..algolib.loader import load_defined_algorithms 29 | 30 | # required by autodoc of sphinx 31 | __all__ = [ 32 | "ReplaceNull", 33 | "ReplaceEmpty", 34 | "ReplaceNullEmpty", 35 | "ReplaceCustom", 36 | "ReplacePercentile", 37 | "ReplaceZScore", 38 | "ReplaceConfidence", 39 | ] 40 | 41 | load_defined_algorithms(sys.modules[__name__], "preprocess") 42 | -------------------------------------------------------------------------------- /odps/ml/preprocess/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/recommend/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from ..algolib.loader import load_defined_algorithms 19 | from ._customize import RecommendFieldRole 20 | 21 | # required by autodoc of sphinx 22 | 23 | __all__ = [] 24 | 25 | 26 | load_defined_algorithms(sys.modules[__name__], "recommend") 27 | -------------------------------------------------------------------------------- /odps/ml/recommend/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/regression/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from ..algolib.loader import load_defined_algorithms 19 | 20 | # required by autodoc of sphinx 21 | __all__ = [] 22 | 23 | 24 | load_defined_algorithms(sys.modules[__name__], "regression") 25 | -------------------------------------------------------------------------------- /odps/ml/regression/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/statistics/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from ..algolib.loader import load_defined_algorithms 19 | 20 | # required by autodoc of sphinx 21 | __all__ = [] 22 | 23 | 24 | load_defined_algorithms(sys.modules[__name__], "statistics") 25 | -------------------------------------------------------------------------------- /odps/ml/statistics/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/tensor/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from ..algolib.loader import load_defined_algorithms 19 | 20 | # required by autodoc of sphinx 21 | __all__ = [] 22 | 23 | 24 | load_defined_algorithms(sys.modules[__name__], 'tensor') 25 | -------------------------------------------------------------------------------- /odps/ml/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/tests/test_cv.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from __future__ import print_function 17 | 18 | import pytest 19 | 20 | from ...df import DataFrame 21 | from ..classifiers import LogisticRegression 22 | from ..cross_validation import cross_val_score 23 | from ..tests.base import MLTestUtil, tn, ci_skip_case 24 | 25 | IONOSPHERE_TABLE = tn('pyodps_test_ml_ionosphere') 26 | 27 | 28 | @pytest.fixture 29 | def utils(odps, tunnel): 30 | util = MLTestUtil(odps, tunnel) 31 | util.create_ionosphere(IONOSPHERE_TABLE) 32 | util.df = DataFrame(odps.get_table(IONOSPHERE_TABLE)).roles(label='class') 33 | return util 34 | 35 | 36 | @ci_skip_case 37 | def test_logistic_regression(utils): 38 | lr = LogisticRegression(epsilon=0.001).set_max_iter(50) 39 | print(cross_val_score(lr, utils.df)) 40 | -------------------------------------------------------------------------------- /odps/ml/tests/test_runner.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from ..runners import BaseNodeRunner 17 | 18 | 19 | def test_param_format(): 20 | assert BaseNodeRunner._format_value(None) == None 21 | assert BaseNodeRunner._format_value(True) == 'true' 22 | assert BaseNodeRunner._format_value(False) == 'false' 23 | assert BaseNodeRunner._format_value('') == '' 24 | assert BaseNodeRunner._format_value([]) == None 25 | assert BaseNodeRunner._format_value([1, 2, 3, 4]) == '1,2,3,4' 26 | assert BaseNodeRunner._format_value(set([1, 2, 3, 4])) == '1,2,3,4' 27 | assert BaseNodeRunner._format_value(12) == '12' 28 | -------------------------------------------------------------------------------- /odps/ml/text/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from ..algolib.loader import load_defined_algorithms 19 | from ._customize import TextFieldRole 20 | 21 | # required by autodoc of sphinx 22 | __all__ = [] 23 | 24 | 25 | load_defined_algorithms(sys.modules[__name__], "text") 26 | -------------------------------------------------------------------------------- /odps/ml/text/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/ml/timeseries/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from ..algolib.loader import load_defined_algorithms 19 | from ._customize import TimeSeriesFieldRole 20 | 21 | # required by autodoc of sphinx 22 | __all__ = [] 23 | 24 | 25 | load_defined_algorithms(sys.modules[__name__], "timeseries") 26 | -------------------------------------------------------------------------------- /odps/ml/timeseries/_customize.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from functools import partial 17 | 18 | from ..expr.exporters import get_input_field_names 19 | from ..expr.mixin import ml_collection_mixin 20 | from ...compat import Enum 21 | 22 | 23 | class TimeSeriesFieldRole(Enum): 24 | TS_GROUP = 'TS_GROUP' 25 | TS_SEQ = 'TS_SEQ' 26 | TS_VALUE = 'TS_VALUE' 27 | 28 | 29 | @ml_collection_mixin 30 | class TimeSeriesMLMixIn(object): 31 | __slots__ = () 32 | 33 | field_role_enum = TimeSeriesFieldRole 34 | non_feature_roles = set((TimeSeriesFieldRole.TS_GROUP, TimeSeriesFieldRole.TS_SEQ)) 35 | 36 | 37 | """ 38 | Common time series exporters 39 | """ 40 | get_ts_group_column = partial(get_input_field_names, field_role=TimeSeriesFieldRole.TS_GROUP) 41 | get_ts_seq_column = partial(get_input_field_names, field_role=TimeSeriesFieldRole.TS_SEQ) 42 | get_ts_value_column = partial(get_input_field_names, field_role=TimeSeriesFieldRole.TS_VALUE) 43 | -------------------------------------------------------------------------------- /odps/models/ml/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .offlinemodel import OfflineModel 18 | from .offlinemodels import OfflineModels 19 | -------------------------------------------------------------------------------- /odps/models/record.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .. import types as _types 18 | from ..config import options 19 | 20 | try: 21 | if not options.force_py: 22 | from ..src.types_c import BaseRecord 23 | 24 | Record = _types.RecordMeta( 25 | "Record", 26 | (_types.RecordReprMixin, BaseRecord), 27 | {"__doc__": _types.Record.__doc__}, 28 | ) 29 | else: 30 | Record = _types.Record 31 | except (ImportError, AttributeError): 32 | if options.force_c: 33 | raise 34 | Record = _types.Record 35 | -------------------------------------------------------------------------------- /odps/models/security/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .config import SecurityConfiguration 18 | from .roles import Role, Roles 19 | from .users import User, Users 20 | -------------------------------------------------------------------------------- /odps/models/session/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .core import SessionMethods 16 | from .v1 import FallbackMode, FallbackPolicy, InSessionInstance, SessionInstance 17 | -------------------------------------------------------------------------------- /odps/models/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .copy import CopyTask 16 | from .core import Task 17 | from .maxframe import MaxFrameTask 18 | from .merge import MergeTask 19 | from .misc import CupidTask 20 | from .sql import SQLCostTask, SQLRTTask, SQLTask 21 | 22 | try: 23 | from ...internal.models.tasks import * # noqa: F401 24 | except ImportError: 25 | pass 26 | -------------------------------------------------------------------------------- /odps/models/tasks/misc.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | 17 | from ... import serializers 18 | from .core import Task, format_cdata 19 | 20 | 21 | class CupidTask(Task): 22 | _root = "CUPID" 23 | 24 | plan = serializers.XMLNodeField("Plan", serialize_callback=format_cdata) 25 | 26 | def __init__(self, name=None, plan=None, hints=None, **kwargs): 27 | kwargs["name"] = name 28 | kwargs["plan"] = plan 29 | super(CupidTask, self).__init__(**kwargs) 30 | hints = hints or {} 31 | self.set_property("type", "cupid") 32 | if hints: 33 | self.set_property("settings", json.dumps(hints)) 34 | -------------------------------------------------------------------------------- /odps/models/tasks/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /odps/models/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/models/tests/test_offline_models.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | def test_offline_models(odps): 17 | assert odps.get_project().offline_models is odps.get_project().offline_models 18 | size = len(list(odps.list_offline_models())) 19 | assert size >= 0 20 | 21 | 22 | def test_instance_exists(odps): 23 | non_exists_offline_model = "a_non_exists_offline_model" 24 | assert odps.exist_offline_model(non_exists_offline_model) is False 25 | -------------------------------------------------------------------------------- /odps/models/tests/test_quotas.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | 17 | 18 | def test_quotas(odps_daily): 19 | odps = odps_daily 20 | 21 | with pytest.raises(TypeError): 22 | odps.get_quota() 23 | 24 | first_quota_nick = next(odps.list_quotas()) 25 | assert first_quota_nick.nickname is not None 26 | 27 | assert not odps.exist_quota("non_exist_quota") 28 | assert odps.exist_quota(first_quota_nick) 29 | assert odps.exist_quota(first_quota_nick.nickname) 30 | 31 | quota_obj = odps.get_quota(first_quota_nick.nickname) 32 | quota_obj.reload() 33 | assert quota_obj.nickname == first_quota_nick.nickname 34 | -------------------------------------------------------------------------------- /odps/models/tests/test_tenant.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ..tenant import Tenant 16 | 17 | 18 | def test_tenant_props(odps_with_schema): 19 | tenant = Tenant(client=odps_with_schema.rest) 20 | assert not tenant._getattr("_loaded") 21 | assert tenant.name is not None 22 | assert tenant._getattr("_loaded") 23 | assert tenant.get_parameter("prop-key") is None 24 | -------------------------------------------------------------------------------- /odps/models/xflow.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .. import serializers, utils 18 | from .core import LazyLoad 19 | 20 | 21 | class XFlow(LazyLoad): 22 | __slots__ = ("xml_source",) 23 | 24 | name = serializers.XMLNodeField("Name") 25 | owner = serializers.XMLNodeField("Owner") 26 | creation_time = serializers.XMLNodeField( 27 | "CreationTime", parse_callback=utils.parse_rfc822 28 | ) 29 | last_modified_time = serializers.XMLNodeField( 30 | "LastModifiedTime", parse_callback=utils.parse_rfc822 31 | ) 32 | 33 | def reload(self): 34 | url = self.resource() 35 | resp = self._client.get(url) 36 | 37 | self.xml_source = resp.content 38 | self.owner = resp.headers.get("x-odps-owner") 39 | self.creation_time = utils.parse_rfc822( 40 | resp.headers.get("x-odps-creation-time") 41 | ) 42 | self.last_modified_time = utils.parse_rfc822(resp.headers.get("Last-Modified")) 43 | 44 | def update(self): 45 | return self._parent.update(self) 46 | 47 | def drop(self): 48 | return self._parent.delete(self) 49 | -------------------------------------------------------------------------------- /odps/src/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/src/crc32c_c.pxd: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from libc.stdint cimport * 17 | 18 | 19 | cdef class Crc32c: 20 | 21 | cdef uint32_t _crc 22 | 23 | cpdef update(self, bytearray buf) 24 | 25 | cpdef uint32_t getvalue(self) 26 | 27 | cpdef reset(self) 28 | -------------------------------------------------------------------------------- /odps/src/crc32c_c.pyx: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from libc.stdint cimport * 17 | 18 | 19 | cdef extern from "crc32.c": 20 | uint32_t crc32c(uint32_t crc, const void *buf, size_t length) nogil 21 | 22 | cdef class Crc32c: 23 | def __cinit__(self): 24 | self._crc = 0 25 | 26 | cpdef uint32_t getvalue(self): 27 | return self._crc 28 | 29 | cpdef update(self, bytearray buf): 30 | cdef char* cstring = buf 31 | self._crc = crc32c(self._crc, cstring, len(buf)) 32 | 33 | cpdef reset(self): 34 | self._crc = 0 35 | -------------------------------------------------------------------------------- /odps/src/stringstream.pxd: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from libcpp.string cimport string 16 | 17 | 18 | cdef extern from "" namespace "std" nogil: 19 | cdef cppclass stringstream: 20 | stringstream() except + 21 | stringstream(const string &str) except + 22 | stringstream& push "operator<<" (bint val) 23 | stringstream& push "operator<<" (short val) 24 | stringstream& push "operator<<" (unsigned short val) 25 | stringstream& push "operator<<" (int val) 26 | stringstream& push "operator<<" (unsigned int val) 27 | stringstream& push "operator<<" (long val) 28 | stringstream& push "operator<<" (unsigned long val) 29 | stringstream& push "operator<<" (float val) 30 | stringstream& push "operator<<" (double val) 31 | stringstream& push "operator<<" (long double val) 32 | stringstream& push "operator<<" (void* val) 33 | stringstream() 34 | unsigned int tellp() 35 | stringstream& put(char c) 36 | stringstream& write(const char *c, size_t n) 37 | string to_string "str" () const 38 | -------------------------------------------------------------------------------- /odps/src/timegm.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | // Algorithm: http://howardhinnant.github.io/date_algorithms.html 4 | 5 | int32_t days_from_epoch(int y, int m, int d) 6 | { 7 | y -= m <= 2; 8 | int era = y / 400; 9 | int yoe = y - era * 400; // [0, 399] 10 | int doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; // [0, 365] 11 | int doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096] 12 | return era * 146097 + doe - 719468; 13 | } 14 | 15 | // It does not modify broken-down time 16 | time_t timegm(struct tm const* t) 17 | { 18 | int year = t->tm_year + 1900; 19 | int month = t->tm_mon; // 0-11 20 | if (month > 11) 21 | { 22 | year += month / 12; 23 | month %= 12; 24 | } 25 | else if (month < 0) 26 | { 27 | int years_diff = (11 - month) / 12; 28 | year -= years_diff; 29 | month += 12 * years_diff; 30 | } 31 | int32_t days_since_epoch = days_from_epoch(year, month + 1, t->tm_mday); 32 | 33 | return 60 * (60 * (24L * days_since_epoch + t->tm_hour) + t->tm_min) + t->tm_sec; 34 | } 35 | 36 | struct tm* gmtime_safe(const time_t* timer, struct tm* buf) 37 | { 38 | #if defined(_WIN32) 39 | struct tm* ptr; 40 | // gmtime is guaranteed as threadsafe in Windows 41 | ptr = gmtime(timer); 42 | *buf = *ptr; 43 | return ptr; 44 | #else 45 | return gmtime_r(timer, buf); 46 | #endif 47 | } 48 | 49 | struct tm* localtime_safe(const time_t* timer, struct tm* buf) 50 | { 51 | #if defined(_WIN32) 52 | struct tm* ptr; 53 | // localtime is guaranteed as threadsafe in Windows 54 | ptr = localtime(timer); 55 | *buf = *ptr; 56 | return ptr; 57 | #else 58 | return localtime_r(timer, buf); 59 | #endif 60 | } 61 | -------------------------------------------------------------------------------- /odps/src/types_c.pxd: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from libc.stdint cimport * 17 | from libcpp.vector cimport vector 18 | 19 | ctypedef object (*_VALIDATE_FUNC)(object val, int64_t max_field_size) 20 | 21 | cdef class SchemaSnapshot: 22 | cdef list _columns 23 | cdef int _col_count 24 | cdef int _partition_col_count 25 | cdef list _col_types 26 | cdef vector[int] _col_type_ids 27 | cdef vector[int] _col_is_partition 28 | cdef vector[int] _col_nullable 29 | cdef list _col_validators 30 | 31 | cdef object validate_value(self, int i, object val, int64_t max_field_size) 32 | 33 | cdef class BaseRecord: 34 | cdef list _c_columns, _c_values 35 | cdef dict _c_name_indexes 36 | cdef SchemaSnapshot _c_schema_snapshot 37 | cdef int64_t _max_field_size 38 | 39 | cpdef object get_by_name(self, object name) 40 | cpdef set_by_name(self, object name, object value) 41 | 42 | cdef size_t _get_non_partition_col_count(self) 43 | 44 | cpdef object _get(self, int i) 45 | cpdef _set(self, int i, object value) 46 | cpdef _sets(self, object values) 47 | -------------------------------------------------------------------------------- /odps/src/utils_c.pxd: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from cpython.datetime cimport date, datetime 16 | from libc.stdint cimport int32_t, int64_t 17 | from libc.time cimport tm 18 | 19 | 20 | cdef class CMillisecondsConverter: 21 | cdef: 22 | object _local_tz, _tz 23 | bint _use_default_tz 24 | bint _default_tz_local 25 | bint _allow_antique 26 | bint _is_dst 27 | bint _tz_has_localize 28 | 29 | cdef int _build_tm_struct(self, datetime dt, tm *p_tm) except? -1 30 | cpdef int64_t to_milliseconds(self, datetime dt) except? -1 31 | cpdef datetime from_milliseconds(self, int64_t milliseconds) 32 | 33 | cdef int32_t to_days(date py_date) except? -1 34 | cdef to_date(int32_t days) 35 | 36 | cpdef inline str to_str(s, encoding=*) 37 | cpdef inline bytes to_binary(s, encoding=*) 38 | cpdef inline unicode to_text(s, encoding=*) 39 | cpdef str to_lower_str(s, encoding=*) 40 | -------------------------------------------------------------------------------- /odps/static/ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pyodps", 3 | "description": "Front-end module of PyODPS", 4 | "version": "0.7.15", 5 | "author": "Alibaba, Inc.", 6 | "repository": { 7 | "type": "git", 8 | "url": "https://github.com/aliyun/aliyun-odps-python-sdk.git" 9 | }, 10 | "license": "Apache-2.0", 11 | "scripts": { 12 | "grunt": "grunt" 13 | }, 14 | "devDependencies": { 15 | "grunt": "~1.3.0", 16 | "grunt-contrib-clean": "~1.0.0", 17 | "grunt-contrib-copy": "~1.0.0", 18 | "grunt-contrib-cssmin": "^4.0.0", 19 | "grunt-contrib-requirejs": "~1.0.0", 20 | "grunt-contrib-uglify": "~1.0.0" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /odps/static/ui/src/chosen-sprite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/odps/static/ui/src/chosen-sprite.png -------------------------------------------------------------------------------- /odps/static/ui/src/chosen-sprite@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/odps/static/ui/src/chosen-sprite@2x.png -------------------------------------------------------------------------------- /odps/static/ui/src/fonts/custom-font.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | @font-face { 16 | font-family: 'CustomFont'; 17 | src: url('../fonts/custom-font.eot') format('embedded-opentype'), url('../fonts/custom-font.woff') format('woff'), url('../fonts/custom-font.ttf') format('truetype'), url('../fonts/custom-font.svg') format('svg'); 18 | font-weight: normal; 19 | font-style: normal; 20 | } 21 | .cf { 22 | display: inline-block; 23 | font: normal normal normal 14px/1 CustomFont; 24 | font-size: inherit; 25 | text-rendering: auto; 26 | -webkit-font-smoothing: antialiased; 27 | -moz-osx-font-smoothing: grayscale; 28 | } 29 | 30 | .cf-scatter-chart:before { 31 | content: "\e800"; 32 | } 33 | -------------------------------------------------------------------------------- /odps/static/ui/src/fonts/custom-font.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/odps/static/ui/src/fonts/custom-font.eot -------------------------------------------------------------------------------- /odps/static/ui/src/fonts/custom-font.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/odps/static/ui/src/fonts/custom-font.ttf -------------------------------------------------------------------------------- /odps/static/ui/src/fonts/custom-font.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/odps/static/ui/src/fonts/custom-font.woff -------------------------------------------------------------------------------- /odps/static/ui/target/chosen-sprite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/odps/static/ui/target/chosen-sprite.png -------------------------------------------------------------------------------- /odps/static/ui/target/chosen-sprite@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/odps/static/ui/target/chosen-sprite@2x.png -------------------------------------------------------------------------------- /odps/static/ui/target/fonts/custom-font.css: -------------------------------------------------------------------------------- 1 | @font-face{font-family:CustomFont;src:url('../fonts/custom-font.eot') format('embedded-opentype'),url('../fonts/custom-font.woff') format('woff'),url('../fonts/custom-font.ttf') format('truetype'),url('../fonts/custom-font.svg') format('svg');font-weight:400;font-style:normal}.cf{display:inline-block;font:normal normal normal 14px/1 CustomFont;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.cf-scatter-chart:before{content:"\e800"} -------------------------------------------------------------------------------- /odps/static/ui/target/fonts/custom-font.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/odps/static/ui/target/fonts/custom-font.eot -------------------------------------------------------------------------------- /odps/static/ui/target/fonts/custom-font.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/odps/static/ui/target/fonts/custom-font.ttf -------------------------------------------------------------------------------- /odps/static/ui/target/fonts/custom-font.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aliyun/aliyun-odps-python-sdk/1b274b34f335a7731b3369113a897f4378e6d53b/odps/static/ui/target/fonts/custom-font.woff -------------------------------------------------------------------------------- /odps/static/ui/target/styles.css: -------------------------------------------------------------------------------- 1 | .pyodps-progress-launcher{margin-right:5px}#pyodps-progress-viewer .generated-time{color:#ccc;padding-right:10px}#pyodps-progress-viewer .stage{width:100%;display:table;table-layout:fixed;margin-top:3px;margin-bottom:3px}#pyodps-progress-viewer .stage-name{display:table-cell;width:30%;overflow:hidden;-ms-text-overflow:ellipsis;text-overflow:ellipsis}#pyodps-progress-viewer .stage-progress{display:table-cell;vertical-align:middle}#pyodps-progress-viewer .progress{margin:0}#pyodps-progress-viewer .progress-bar{border:none;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none;width:0}#pyodps-progress-viewer .progress-legend-panel{font-size:smaller;margin-top:8px}#pyodps-progress-viewer .progress-legend-panel span{float:left!important}#pyodps-progress-viewer .progress-legend{height:10px;width:30px!important;margin-left:5px;margin-right:5px;margin-top:3px}#pyodps-progress-viewer .progress-bar{background-color:#2196f3}#pyodps-progress-viewer .progress-bar-warning{background-color:#ff9800}#pyodps-progress-viewer .task-name{border-bottom:1px solid #ccc}.retry-btn{cursor:pointer;font-size:larger;padding-right:5px;padding-top:4px;color:#4682b4;display:block}.retry-btn:hover{color:#87ceeb}.retry-btn:active{color:#4682b4}.retry-btn:visited{color:#4682b4}.show-table-body{overflow-x:auto}.df-function-nav{margin:0!important}.df-function-nav li a{padding:6px 10px}.df-table-pagination{margin:0!important}.df-agg-selector{padding-left:3px!important;padding-right:0!important} -------------------------------------------------------------------------------- /odps/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /odps/tests/test_crc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .. import crc 18 | from ..tests.core import py_and_c 19 | 20 | 21 | @py_and_c("odps.crc") 22 | def test_crc32c(): 23 | crc_obj = crc.Crc32c() 24 | assert 0 == crc_obj.getvalue() 25 | buf = bytearray(b"abc") 26 | crc_obj.update(buf) 27 | assert 910901175 == crc_obj.getvalue() 28 | buf = bytearray(b"1111111111111111111") 29 | crc_obj.update(buf) 30 | assert 2917307201 == crc_obj.getvalue() 31 | -------------------------------------------------------------------------------- /odps/tunnel/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .instancetunnel import InstanceDownloadSession, InstanceTunnel 17 | from .io import ( 18 | ArrowWriter, 19 | BufferedArrowWriter, 20 | BufferedRecordWriter, 21 | CompressOption, 22 | RecordWriter, 23 | TunnelArrowReader, 24 | TunnelRecordReader, 25 | Upsert, 26 | ) 27 | from .io.reader import TunnelArrowReader, TunnelRecordReader 28 | from .tabletunnel import ( 29 | TableDownloadSession, 30 | TableStreamUploadSession, 31 | TableTunnel, 32 | TableUploadSession, 33 | TableUpsertSession, 34 | ) 35 | from .volumetunnel import ( 36 | VolumeDownloadSession, 37 | VolumeFSTunnel, 38 | VolumeTunnel, 39 | VolumeUploadSession, 40 | ) 41 | 42 | TableUploadStatus = TableUploadSession.Status 43 | TableDownloadStatus = TableDownloadSession.Status 44 | VolumeUploadStatus = VolumeUploadSession.Status 45 | VolumeDownloadStatus = VolumeDownloadSession.Status 46 | InstanceDownloadStatus = InstanceDownloadSession.Status 47 | -------------------------------------------------------------------------------- /odps/tunnel/checksum_c.pxd: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from libc.stdint cimport * 17 | from libc.string cimport * 18 | 19 | 20 | cdef class Checksum: 21 | 22 | cdef uint32_t _checksum 23 | cdef int use_c 24 | 25 | cdef void c_update_bool(self, bint val) noexcept nogil 26 | cdef void c_update_int(self, int32_t val) noexcept nogil 27 | cdef void c_update_long(self, int64_t val) noexcept nogil 28 | cdef void c_update_float(self, float val) noexcept nogil 29 | cdef void c_update_double(self, double val) noexcept nogil 30 | cdef void c_update(self, const char *ptr, size_t length) noexcept nogil 31 | cdef uint32_t c_getvalue(self) nogil 32 | cdef uint32_t c_setvalue(self, uint32_t val) nogil 33 | cdef void c_reset(self) nogil 34 | 35 | cpdef update_bool(self, bint val) 36 | cpdef update_int(self, int32_t val) 37 | cpdef update_long(self, int64_t val) 38 | cpdef update_float(self, float val) 39 | cpdef update_double(self, double val) 40 | cpdef update(self, bytes b) 41 | cpdef uint32_t getvalue(self) 42 | cpdef uint32_t setvalue(self, uint32_t val) 43 | cpdef reset(self) 44 | -------------------------------------------------------------------------------- /odps/tunnel/io/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .reader import ArrowRecordReader, TunnelArrowReader, TunnelRecordReader 17 | from .stream import * 18 | from .writer import ( 19 | ArrowWriter, 20 | BufferedArrowWriter, 21 | BufferedRecordWriter, 22 | RecordWriter, 23 | Upsert, 24 | ) 25 | -------------------------------------------------------------------------------- /odps/tunnel/io/reader_c.pxd: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2025 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from libc.stdint cimport * 17 | from libc.string cimport * 18 | 19 | from ...src.types_c cimport SchemaSnapshot 20 | from ..checksum_c cimport Checksum 21 | from ..pb.decoder_c cimport CDecoder 22 | 23 | 24 | cdef class BaseTunnelRecordReader: 25 | cdef public object _schema 26 | cdef object _columns 27 | cdef object _stream_creator 28 | cdef public CDecoder _reader 29 | cdef public Checksum _crc 30 | cdef Checksum _crccrc 31 | cdef int _curr_cursor 32 | cdef int _attempt_row_count 33 | cdef int _last_n_bytes 34 | cdef int _n_columns 35 | cdef int _read_limit 36 | cdef list _field_readers 37 | cdef SchemaSnapshot _schema_snapshot 38 | cdef list _partition_vals 39 | cdef bint _append_partitions 40 | 41 | cdef public bytes _server_metrics_string 42 | cdef bint _enable_client_metrics 43 | cdef long _c_local_wall_time_ms 44 | cdef long _c_acc_network_time_ms 45 | 46 | cdef int _n_injected_error_cursor 47 | cdef object _injected_error_exc 48 | 49 | cdef public int _set_record_list_value( 50 | self, list record, int i, object value 51 | ) except? -1 52 | cdef _read(self) 53 | cpdef read(self) 54 | -------------------------------------------------------------------------------- /odps/tunnel/pb/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /odps/tunnel/pb/decoder_c.pxd: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from libc.stdint cimport * 17 | from libc.string cimport * 18 | 19 | 20 | cdef class CDecoder: 21 | cdef size_t _pos 22 | cdef object _stream 23 | cdef bytes _buffer 24 | cdef char* _begin 25 | cdef char* _end 26 | cdef bint _is_source_eof 27 | 28 | cdef bint _record_network_time 29 | cdef public long long _network_wall_time_ns 30 | 31 | cdef size_t position(self) nogil 32 | cdef int32_t read_field_number(self, int32_t * p_wire_type) except? -1 nogil 33 | cdef int32_t read_sint32(self) except? -1 nogil 34 | cdef uint32_t read_uint32(self) except? 0xffffffff nogil 35 | cdef int64_t read_sint64(self) except? -1 nogil 36 | cdef uint64_t read_uint64(self) except? 0xffffffff nogil 37 | cdef bint read_bool(self) except? False nogil 38 | cdef double read_double(self) except? -1.0 nogil 39 | cdef float read_float(self) except? -1.0 nogil 40 | cdef bytes read_string(self) 41 | cdef int _load_next_buffer(self) except -1 with gil 42 | 43 | 44 | cdef class Decoder: 45 | cdef CDecoder _decoder 46 | -------------------------------------------------------------------------------- /odps/tunnel/pb/encoder_c.pxd: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from libc.stdint cimport * 17 | from libc.string cimport * 18 | 19 | from ...src.stringstream cimport stringstream 20 | 21 | 22 | cdef class CEncoder: 23 | cdef stringstream *_buffer 24 | 25 | cdef size_t position(self) nogil 26 | cpdef bytes tostring(self) 27 | cdef int append_tag(self, int field_num, int wire_type) except -1 nogil 28 | cdef int append_sint32(self, int32_t value) except -1 nogil 29 | cdef int append_uint32(self, uint32_t value) except -1 nogil 30 | cdef int append_sint64(self, int64_t value) except -1 nogil 31 | cdef int append_uint64(self, uint64_t value) except -1 nogil 32 | cdef int append_bool(self, bint value) except -1 nogil 33 | cdef int append_double(self, double value) except -1 nogil 34 | cdef int append_float(self, float value) except -1 nogil 35 | cdef int append_string(self, const char *ptr, size_t value_len) except -1 nogil 36 | 37 | 38 | cdef class Encoder: 39 | cdef CEncoder _encoder 40 | -------------------------------------------------------------------------------- /odps/tunnel/pb/errors.py: -------------------------------------------------------------------------------- 1 | # Protocol Buffers - Google's data interchange format 2 | # Copyright 2008 Google Inc. 3 | # http://code.google.com/p/protobuf/ 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | """This implementation is modified from google's original Protobuf implementation. 19 | The original author is: robinson@google.com (Will Robinson). 20 | Modified by onesuperclark@gmail.com(onesuper). 21 | """ 22 | 23 | 24 | class Error(Exception): 25 | pass 26 | 27 | 28 | class DecodeError(Error): 29 | pass 30 | 31 | 32 | class EncodeError(Error): 33 | pass 34 | -------------------------------------------------------------------------------- /odps/tunnel/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/tunnel/wireconstants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | class ProtoWireConstants(object): 19 | TUNNEL_META_COUNT = 33554430 20 | TUNNEL_META_CHECKSUM = 33554431 21 | TUNNEL_END_RECORD = 33553408 22 | TUNNEL_END_METRICS = 33554176 23 | -------------------------------------------------------------------------------- /odps/udf/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """UDF debug support""" 16 | 17 | from __future__ import absolute_import 18 | 19 | from .runtime import * 20 | -------------------------------------------------------------------------------- /odps/udf/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /odps/udf/tests/pyou: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export PYTHONPATH=.:misc 4 | 5 | ../../../scripts/pyou $@ 6 | -------------------------------------------------------------------------------- /odps/udf/tests/test_executioncontext.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .. import ExecutionContext, get_execution_context 16 | 17 | 18 | def test_get_counter(): 19 | ctx = get_execution_context() 20 | counters = ctx.get_counters() 21 | c = counters.get_counter("test_group", "test") 22 | assert "test" == c.get_name() 23 | assert 0 == c.get_value() 24 | 25 | c.increment(1) 26 | assert 1 == c.get_value() 27 | 28 | 29 | def test_single_counters(): 30 | ctx1 = get_execution_context() 31 | counters = ctx1.get_counters() 32 | counters.get_counter("test_group", "test") 33 | assert 1 == counters.size() 34 | 35 | ctx2 = ExecutionContext() 36 | counters2 = ctx2.get_counters() 37 | counters2.get_counter("test_group2", "test2") 38 | assert 2 == counters2.size() 39 | -------------------------------------------------------------------------------- /odps/udf/tests/test_resource.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from odps import distcache 16 | 17 | 18 | def test_get_cache_archive(): 19 | assert distcache.get_cache_archive("x") is None 20 | assert distcache.get_cache_archive("x", "y") is None 21 | 22 | 23 | def test_get_cache_tabledesc(): 24 | assert distcache.get_cache_tabledesc("x") is None 25 | 26 | 27 | def test_get_cache_tableinfo(): 28 | assert distcache.get_cache_tableinfo("x") is None 29 | -------------------------------------------------------------------------------- /odps/udf/tests/test_simple_run.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | 17 | from ... import distcache 18 | from ..tools import runners 19 | from .udf_examples import * 20 | 21 | 22 | def test_udf(): 23 | assert [2, 3] == runners.simple_run(Plus, [(1, 1), (2, 1)]) 24 | assert [None] == runners.simple_run(Plus, [(None, 1)]) 25 | 26 | 27 | def test_udaf(): 28 | assert [2] == runners.simple_run(Avg, [(1,), (2,), (3,)]) 29 | 30 | 31 | def test_udtf(): 32 | assert ["a", "b", "ok"] == runners.simple_run(Explode, [("a|b",)]) 33 | 34 | 35 | @pytest.mark.skip("Not implemented yet") 36 | def test_get_cache_table(): 37 | assert distcache.get_cache_table("dual") == ("0",) 38 | -------------------------------------------------------------------------------- /odps/udf/tests/test_types_py2.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | import time 17 | 18 | import pytest 19 | 20 | from ... import types as odps_types 21 | from ..tools.runners import _convert_value 22 | 23 | pytestmark = pytest.mark.skipif( 24 | sys.version_info[0] != 2, reason="Only needed for Python 2" 25 | ) 26 | 27 | 28 | def test_py2_convert(): 29 | cur_mills = int(1000 * time.time()) 30 | assert 123 == _convert_value("123", odps_types.bigint) 31 | assert cur_mills == _convert_value(cur_mills, odps_types.datetime) 32 | assert [cur_mills] == _convert_value( 33 | [cur_mills], odps_types.Array(odps_types.datetime) 34 | ) 35 | assert {"key": cur_mills} == _convert_value( 36 | {"key": cur_mills}, odps_types.Map(odps_types.string, odps_types.datetime) 37 | ) 38 | 39 | struct_type = odps_types.Struct({"key": odps_types.datetime}) 40 | assert (cur_mills,) == _convert_value( 41 | struct_type.namedtuple_type(cur_mills), struct_type 42 | ) 43 | assert (cur_mills,) == _convert_value({"key": cur_mills}, struct_type) 44 | -------------------------------------------------------------------------------- /odps/udf/tools/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /odps/udf/tools/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import os 17 | 18 | from ...compat import ConfigParser 19 | from ...core import ODPS 20 | 21 | CONF_FILENAME = "~/.pyouconfig" 22 | 23 | logger = logging.getLogger(__name__) 24 | _odps = None 25 | 26 | 27 | class UDFToolError(Exception): 28 | pass 29 | 30 | 31 | def require_conf(func): 32 | def f(*args, **kwargs): 33 | if _odps is None: 34 | get_conf() 35 | return func(*args, **kwargs) 36 | 37 | return f 38 | 39 | 40 | def get_conf(): 41 | global _odps 42 | 43 | f = os.path.expanduser(CONF_FILENAME) 44 | if not os.path.exists(f): 45 | return 46 | 47 | config = ConfigParser.RawConfigParser() 48 | config.read(f) 49 | access_id = config.get("access_id") 50 | access_key = config.get("secret_access_key") 51 | end_point = config.get("endpoint") 52 | project = config.get("project") 53 | _odps = ODPS(access_id, access_key, project, end_point) 54 | 55 | 56 | @require_conf 57 | def get_cache_table(name): 58 | odps_entry = _odps or ODPS.from_global() 59 | return odps_entry.read_table(name) 60 | -------------------------------------------------------------------------------- /odps/ui/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .common import html_notify 17 | from .progress import ( 18 | ProgressGroupUI, 19 | create_instance_group, 20 | fetch_instance_group, 21 | reload_instance_status, 22 | ) 23 | -------------------------------------------------------------------------------- /odps/ui/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /odps/ui/tests/test_ui.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2024 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | 17 | from ...compat import six 18 | from ..tests.base import grab_iopub_comm, setup_kernel, ui_case 19 | 20 | TIMEOUT = 10 21 | 22 | 23 | @ui_case 24 | def test_html_notify(): 25 | with setup_kernel() as client: 26 | client.execute("from odps.ui import html_notify") 27 | shell_msg = client.get_shell_msg(timeout=TIMEOUT) 28 | content = shell_msg["content"] 29 | assert content["status"] == "ok" 30 | 31 | msg_id = client.execute('html_notify("TestMessage")') 32 | iopub_data = grab_iopub_comm(client, msg_id) 33 | assert any(u"TestMessage" in json.dumps(l) for l in six.itervalues(iopub_data)) 34 | 35 | shell_msg = client.get_shell_msg(timeout=TIMEOUT) 36 | content = shell_msg["content"] 37 | assert content["status"] == "ok" 38 | -------------------------------------------------------------------------------- /odps_scripts/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2022 Alibaba Group Holding Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /requirements-full.txt: -------------------------------------------------------------------------------- 1 | jupyter>=1.0.0 2 | ipython>=4.0.0 3 | numpy>=1.6.0 4 | pandas>=0.17.0 5 | matplotlib>=1.4 6 | pytz>=2017.2;python<3.9 7 | pyyaml>=3.13 8 | -------------------------------------------------------------------------------- /requirements-wheel.txt: -------------------------------------------------------------------------------- 1 | numpy==1.14.5; python_version<='3.7' and platform_machine!='aarch64' 2 | 3 | # oldest-supported-numpy is deprecated and only applied for older Python versions 4 | oldest-supported-numpy; (python_version>'3.7' or platform_machine=='aarch64') and python_version<'3.9' 5 | 6 | # Force numpy higher than 2.0, so that built wheels are compatible 7 | # with both numpy 1 and 2 8 | numpy>=2.0.0; python_version>='3.9' 9 | 10 | cython>=3.0,<3.1 11 | requests>=2.4.0 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests>=2.4.0 2 | pyarrow>=0.16.0; python_version < "3" 3 | pyarrow>=2.0.0; python_version >= "3" 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | select = 4 | E9, 5 | E101, 6 | E111, 7 | E117, 8 | E127, 9 | E201, 10 | E202, 11 | E223, 12 | E224, 13 | E225, 14 | E231, 15 | E242, 16 | E251, 17 | E273, 18 | E274, 19 | E275, 20 | E301, 21 | E302, 22 | E303, 23 | E304, 24 | E305, 25 | E401, 26 | E703, 27 | E901, 28 | E999, 29 | F7, 30 | F63, 31 | F82, 32 | F401, 33 | F811, 34 | F821, 35 | F822, 36 | F823, 37 | F841, 38 | W191, 39 | W291, 40 | W292, 41 | W293, 42 | W391, 43 | W601, 44 | W602, 45 | W603, 46 | W604, 47 | W605 48 | exclude = 49 | __init__.py 50 | __pycache__ 51 | .git/ 52 | benchmarks/ 53 | build/ 54 | bin/ 55 | cupid/ 56 | dist/ 57 | docs/ 58 | env/ 59 | examples/ 60 | misc/ 61 | odps/compat.py 62 | odps/df/* 63 | odps/lib/* 64 | odps/mars_extension/* 65 | odps/ml/* 66 | odps/udf/* 67 | odps_scripts/* 68 | *.pyi 69 | setup.py 70 | --------------------------------------------------------------------------------