├── .build-bot.json ├── .coveragerc ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── datalab ├── README ├── __init__.py ├── bigquery │ ├── __init__.py │ ├── _api.py │ ├── _csv_options.py │ ├── _dataset.py │ ├── _dialect.py │ ├── _federated_table.py │ ├── _job.py │ ├── _parser.py │ ├── _query.py │ ├── _query_job.py │ ├── _query_results_table.py │ ├── _query_stats.py │ ├── _sampling.py │ ├── _schema.py │ ├── _table.py │ ├── _udf.py │ ├── _utils.py │ ├── _view.py │ └── commands │ │ ├── __init__.py │ │ └── _bigquery.py ├── context │ ├── __init__.py │ ├── _api.py │ ├── _context.py │ ├── _project.py │ ├── _utils.py │ └── commands │ │ ├── __init__.py │ │ └── _projects.py ├── data │ ├── __init__.py │ ├── _csv.py │ ├── _sql_module.py │ ├── _sql_statement.py │ ├── _utils.py │ └── commands │ │ ├── __init__.py │ │ └── _sql.py ├── kernel │ └── __init__.py ├── notebook │ ├── __init__.py │ └── static │ │ ├── bigquery.css │ │ ├── bigquery.js.map │ │ ├── bigquery.ts │ │ ├── charting.css │ │ ├── charting.js.map │ │ ├── charting.ts │ │ ├── element.js.map │ │ ├── element.ts │ │ ├── extern │ │ ├── d3.parcoords.css │ │ ├── d3.parcoords.js │ │ ├── lantern-browser.html │ │ ├── parcoords-LICENSE.txt │ │ ├── sylvester-LICENSE.txt │ │ └── sylvester.js │ │ ├── job.css │ │ ├── job.js.map │ │ ├── job.ts │ │ ├── parcoords.ts │ │ ├── style.js.map │ │ ├── style.ts │ │ ├── visualization.js.map │ │ └── visualization.ts ├── stackdriver │ ├── __init__.py │ ├── commands │ │ ├── __init__.py │ │ └── _monitoring.py │ └── monitoring │ │ ├── __init__.py │ │ ├── _group.py │ │ ├── _metric.py │ │ ├── _query.py │ │ ├── _query_metadata.py │ │ ├── _resource.py │ │ └── _utils.py ├── storage │ ├── __init__.py │ ├── _api.py │ ├── _bucket.py │ ├── _item.py │ └── commands │ │ ├── __init__.py │ │ └── _storage.py └── utils │ ├── __init__.py │ ├── _async.py │ ├── _dataflow_job.py │ ├── _gcp_job.py │ ├── _http.py │ ├── _iterator.py │ ├── _job.py │ ├── _json_encoder.py │ ├── _lambda_job.py │ ├── _lru_cache.py │ ├── _utils.py │ └── commands │ ├── __init__.py │ ├── _chart.py │ ├── _chart_data.py │ ├── _commands.py │ ├── _csv.py │ ├── _extension.py │ ├── _html.py │ ├── _job.py │ ├── _modules.py │ └── _utils.py ├── docs ├── .nojekyll ├── Makefile ├── README ├── conf.py ├── datalab Commands.rst ├── datalab.bigquery.rst ├── datalab.context.rst ├── datalab.data.rst ├── datalab.stackdriver.monitoring.rst ├── datalab.storage.rst ├── gen-magic-rst.ipy ├── google.datalab Commands.rst ├── google.datalab.bigquery.rst ├── google.datalab.data.rst ├── google.datalab.ml.rst ├── google.datalab.rst ├── google.datalab.stackdriver.monitoring.rst ├── google.datalab.storage.rst ├── index.rst ├── make.bat ├── mltoolbox.classification.dnn.rst ├── mltoolbox.classification.linear.rst ├── mltoolbox.image.classification.rst ├── mltoolbox.regression.dnn.rst └── mltoolbox.regression.linear.rst ├── externs └── ts │ └── require │ └── require.d.ts ├── google ├── __init__.py └── datalab │ ├── __init__.py │ ├── _context.py │ ├── _job.py │ ├── bigquery │ ├── __init__.py │ ├── _api.py │ ├── _csv_options.py │ ├── _dataset.py │ ├── _external_data_source.py │ ├── _job.py │ ├── _parser.py │ ├── _query.py │ ├── _query_job.py │ ├── _query_output.py │ ├── _query_results_table.py │ ├── _query_stats.py │ ├── _sampling.py │ ├── _schema.py │ ├── _table.py │ ├── _udf.py │ ├── _utils.py │ ├── _view.py │ └── commands │ │ ├── __init__.py │ │ └── _bigquery.py │ ├── commands │ ├── __init__.py │ └── _datalab.py │ ├── contrib │ ├── __init__.py │ ├── bigquery │ │ ├── __init__.py │ │ ├── commands │ │ │ ├── __init__.py │ │ │ └── _bigquery.py │ │ └── operators │ │ │ ├── __init__.py │ │ │ ├── _bq_execute_operator.py │ │ │ ├── _bq_extract_operator.py │ │ │ └── _bq_load_operator.py │ ├── mlworkbench │ │ ├── __init__.py │ │ ├── _archive.py │ │ ├── _local_predict.py │ │ ├── _prediction_explainer.py │ │ ├── _shell_process.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ └── _ml.py │ └── pipeline │ │ ├── __init__.py │ │ ├── _pipeline.py │ │ ├── airflow │ │ ├── __init__.py │ │ └── _airflow.py │ │ ├── commands │ │ ├── __init__.py │ │ └── _pipeline.py │ │ └── composer │ │ ├── __init__.py │ │ ├── _api.py │ │ └── _composer.py │ ├── data │ ├── __init__.py │ └── _csv_file.py │ ├── kernel │ └── __init__.py │ ├── ml │ ├── __init__.py │ ├── _cloud_models.py │ ├── _cloud_training_config.py │ ├── _confusion_matrix.py │ ├── _dataset.py │ ├── _fasets.py │ ├── _feature_slice_view.py │ ├── _job.py │ ├── _metrics.py │ ├── _summary.py │ ├── _tensorboard.py │ └── _util.py │ ├── notebook │ ├── __init__.py │ └── static │ │ ├── bigquery.css │ │ ├── bigquery.ts │ │ ├── charting.css │ │ ├── charting.ts │ │ ├── element.ts │ │ ├── extern │ │ ├── d3.parcoords.css │ │ ├── d3.parcoords.js │ │ ├── facets-jupyter.html │ │ ├── lantern-browser.html │ │ ├── parcoords-LICENSE.txt │ │ ├── sylvester-LICENSE.txt │ │ └── sylvester.js │ │ ├── job.css │ │ ├── job.ts │ │ ├── parcoords.ts │ │ ├── style.ts │ │ └── visualization.ts │ ├── stackdriver │ ├── __init__.py │ ├── commands │ │ ├── __init__.py │ │ └── _monitoring.py │ └── monitoring │ │ ├── __init__.py │ │ ├── _group.py │ │ ├── _metric.py │ │ ├── _query.py │ │ ├── _query_metadata.py │ │ ├── _resource.py │ │ └── _utils.py │ ├── storage │ ├── __init__.py │ ├── _api.py │ ├── _bucket.py │ ├── _object.py │ └── commands │ │ ├── __init__.py │ │ └── _storage.py │ └── utils │ ├── __init__.py │ ├── _async.py │ ├── _dataflow_job.py │ ├── _gcp_job.py │ ├── _http.py │ ├── _iterator.py │ ├── _json_encoder.py │ ├── _lambda_job.py │ ├── _lru_cache.py │ ├── _utils.py │ ├── commands │ ├── __init__.py │ ├── _chart.py │ ├── _chart_data.py │ ├── _commands.py │ ├── _csv.py │ ├── _html.py │ ├── _job.py │ └── _utils.py │ └── facets │ ├── __init__.py │ ├── base_feature_statistics_generator.py │ ├── base_generic_feature_statistics_generator.py │ ├── feature_statistics_generator.py │ ├── feature_statistics_pb2.py │ └── generic_feature_statistics_generator.py ├── install-no-virtualenv.sh ├── install-virtualenv.sh ├── legacy_tests ├── _util │ ├── __init__.py │ ├── http_tests.py │ ├── lru_cache_tests.py │ └── util_tests.py ├── bigquery │ ├── __init__.py │ ├── api_tests.py │ ├── dataset_tests.py │ ├── federated_table_tests.py │ ├── jobs_tests.py │ ├── parser_tests.py │ ├── query_tests.py │ ├── sampling_tests.py │ ├── schema_tests.py │ ├── table_tests.py │ ├── udf_tests.py │ └── view_tests.py ├── data │ ├── __init__.py │ └── sql_tests.py ├── kernel │ ├── __init__.py │ ├── bigquery_tests.py │ ├── chart_data_tests.py │ ├── chart_tests.py │ ├── commands_tests.py │ ├── html_tests.py │ ├── module_tests.py │ ├── sql_tests.py │ ├── storage_tests.py │ └── utils_tests.py ├── main.py ├── stackdriver │ ├── __init__.py │ ├── commands │ │ ├── __init__.py │ │ └── monitoring_tests.py │ └── monitoring │ │ ├── __init__.py │ │ ├── group_tests.py │ │ ├── metric_tests.py │ │ ├── query_metadata_tests.py │ │ ├── query_tests.py │ │ ├── resource_tests.py │ │ └── utils_tests.py └── storage │ ├── __init__.py │ ├── api_tests.py │ ├── bucket_tests.py │ └── item_tests.py ├── release.sh ├── setup.cfg ├── setup.py ├── solutionbox ├── image_classification │ ├── mltoolbox │ │ ├── __init__.py │ │ └── image │ │ │ ├── __init__.py │ │ │ └── classification │ │ │ ├── __init__.py │ │ │ ├── _api.py │ │ │ ├── _cloud.py │ │ │ ├── _inceptionlib.py │ │ │ ├── _local.py │ │ │ ├── _model.py │ │ │ ├── _predictor.py │ │ │ ├── _preprocess.py │ │ │ ├── _trainer.py │ │ │ ├── _util.py │ │ │ ├── setup.py │ │ │ └── task.py │ └── setup.py ├── ml_workbench │ ├── setup.py │ ├── tensorflow │ │ ├── __init__.py │ │ ├── analyze.py │ │ ├── setup.py │ │ ├── trainer │ │ │ ├── __init__.py │ │ │ ├── feature_analysis.py │ │ │ ├── feature_transforms.py │ │ │ └── task.py │ │ └── transform.py │ ├── test_tensorflow │ │ ├── run_all.sh │ │ ├── test_analyze.py │ │ ├── test_cloud_workflow.py │ │ ├── test_feature_transforms.py │ │ ├── test_training.py │ │ └── test_transform.py │ ├── test_xgboost │ │ ├── run_all.sh │ │ ├── test_analyze.py │ │ └── test_transform.py │ └── xgboost │ │ ├── __init__.py │ │ ├── analyze.py │ │ ├── setup.py │ │ ├── trainer │ │ ├── __init__.py │ │ ├── feature_analysis.py │ │ ├── feature_transforms.py │ │ └── task.py │ │ └── transform.py └── structured_data │ ├── build.sh │ ├── mltoolbox │ ├── __init__.py │ ├── _structured_data │ │ ├── __init__.py │ │ ├── __version__.py │ │ ├── _package.py │ │ ├── master_setup.py │ │ ├── prediction │ │ │ ├── __init__.py │ │ │ └── predict.py │ │ ├── preprocess │ │ │ ├── __init__.py │ │ │ ├── cloud_preprocess.py │ │ │ └── local_preprocess.py │ │ └── trainer │ │ │ ├── __init__.py │ │ │ ├── task.py │ │ │ └── util.py │ ├── classification │ │ ├── __init__.py │ │ ├── dnn │ │ │ ├── __init__.py │ │ │ └── _classification_dnn.py │ │ └── linear │ │ │ ├── __init__.py │ │ │ └── _classification_linear.py │ └── regression │ │ ├── __init__.py │ │ ├── dnn │ │ ├── __init__.py │ │ └── _regression_dnn.py │ │ └── linear │ │ ├── __init__.py │ │ └── _regression_linear.py │ ├── setup.py │ └── test_mltoolbox │ ├── __init__.py │ ├── e2e_functions.py │ ├── test_datalab_e2e.py │ ├── test_package_functions.py │ ├── test_sd_preprocess.py │ └── test_sd_trainer.py ├── tests ├── _util │ ├── __init__.py │ ├── commands_tests.py │ ├── feature_statistics_generator_test.py │ ├── generic_feature_statistics_generator_test.py │ ├── http_tests.py │ ├── lru_cache_tests.py │ └── util_tests.py ├── bigquery │ ├── __init__.py │ ├── api_tests.py │ ├── dataset_tests.py │ ├── external_data_source_tests.py │ ├── jobs_tests.py │ ├── operator_tests.py │ ├── parser_tests.py │ ├── pipeline_tests.py │ ├── query_tests.py │ ├── sampling_tests.py │ ├── schema_tests.py │ ├── table_tests.py │ ├── udf_tests.py │ └── view_tests.py ├── context_tests.py ├── integration │ └── storage_test.py ├── kernel │ ├── __init__.py │ ├── bigquery_tests.py │ ├── chart_data_tests.py │ ├── chart_tests.py │ ├── html_tests.py │ ├── pipeline_tests.py │ ├── storage_tests.py │ └── utils_tests.py ├── main.py ├── ml │ ├── __init__.py │ ├── confusion_matrix_tests.py │ ├── dataset_tests.py │ ├── facets_tests.py │ ├── metrics_tests.py │ ├── summary_tests.py │ └── tensorboard_tests.py ├── ml_workbench │ ├── __init__.py │ └── all_tests.py ├── mltoolbox_structured_data │ ├── __init__.py │ ├── dl_interface_tests.py │ ├── sd_e2e_tests.py │ └── traininglib_tests.py ├── mlworkbench_magic │ ├── __init__.py │ ├── archive_tests.py │ ├── explainer_tests.py │ ├── local_predict_tests.py │ ├── ml_tests.py │ └── shell_process_tests.py ├── pipeline │ ├── __init__.py │ ├── airflow_tests.py │ ├── composer_api_tests.py │ ├── composer_tests.py │ └── pipeline_tests.py ├── stackdriver │ ├── __init__.py │ ├── commands │ │ ├── __init__.py │ │ └── monitoring_tests.py │ └── monitoring │ │ ├── __init__.py │ │ ├── group_tests.py │ │ ├── metric_tests.py │ │ ├── query_metadata_tests.py │ │ ├── query_tests.py │ │ ├── resource_tests.py │ │ └── utils_tests.py └── storage │ ├── __init__.py │ ├── api_tests.py │ ├── bucket_tests.py │ └── object_tests.py └── tox.ini /.build-bot.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googledatalab/pydatalab/8bf007da3e43096aa3a3dca158fc56b286ba6f5c/.build-bot.json -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | # .coveragerc to control coverage.py 2 | [run] 3 | 4 | [report] 5 | include = */site-packages/google/datalab/* 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.pyi 3 | *.map 4 | *.egg-info 5 | *.iml 6 | .idea 7 | .DS_Store 8 | MANIFEST 9 | build 10 | .coverage 11 | dist 12 | datalab.magics.rst 13 | datalab/notebook/static/*.js 14 | google/datalab/notebook/static/*.js 15 | 16 | # Test files 17 | .tox/ 18 | .cache/ 19 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | dist: trusty 3 | sudo: false 4 | 5 | matrix: 6 | include: 7 | - python: 2.7 8 | env: TOX_ENV=py27 9 | - python: 3.5 10 | env: TOX_ENV=py35 11 | - python: 2.7 12 | env: TOX_ENV=flake8 13 | - python: 2.7 14 | env: TOX_ENV=coveralls 15 | 16 | before_install: 17 | - npm install -g typescript@3.0.3 18 | - tsc --module amd --noImplicitAny --outdir datalab/notebook/static datalab/notebook/static/*.ts 19 | # We use tox for actually running tests. 20 | - pip install --upgrade pip tox 21 | 22 | script: 23 | # tox reads its configuration from tox.ini. 24 | - tox -e $TOX_ENV 25 | 26 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Want to contribute? Great! First, read this page (including the small print at the end). 2 | 3 | ### Before you contribute 4 | Before we can use your code, you must sign the 5 | [Google Individual Contributor License Agreement] 6 | (https://cla.developers.google.com/about/google-individual) 7 | (CLA), which you can do online. The CLA is necessary mainly because you own the 8 | copyright to your changes, even after your contribution becomes part of our 9 | codebase, so we need your permission to use and distribute your code. We also 10 | need to be sure of various other things—for instance that you'll tell us if you 11 | know that your code infringes on other people's patents. You don't have to sign 12 | the CLA until after you've submitted your code for review and a member has 13 | approved it, but you must do it before we can put your code into our codebase. 14 | Before you start working on a larger contribution, you should get in touch with 15 | us first through the issue tracker with your idea so that we can help out and 16 | possibly guide you. Coordinating up front makes it much easier to avoid 17 | frustration later on. 18 | 19 | ### Code reviews 20 | All submissions, including submissions by project members, require review. We 21 | use Github pull requests for this purpose. 22 | 23 | ### Running tests 24 | We use [`tox`](https://tox.readthedocs.io/) for running our tests. To run tests 25 | before sending out a pull request, just 26 | [install tox](https://tox.readthedocs.io/en/latest/install.html) and run 27 | 28 | ```shell 29 | $ tox 30 | ``` 31 | 32 | to run tests under all supported environments. (This will skip any environments 33 | for which no interpreter is available.) `tox -l` will provide a list of all 34 | supported environments. 35 | 36 | `tox` will run all tests referenced by `tests/main.py` and 37 | `legacy_tests/main.py`. 38 | 39 | ### The small print 40 | Contributions made by corporations are covered by a different agreement than 41 | the one above, the 42 | [Software Grant and Corporate Contributor License Agreement] 43 | (https://cla.developers.google.com/about/google-corporate). 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Google Cloud DataLab 2 | 3 | Datalab is deprecated. [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench) provides a notebook-based environment that offers capabilities beyond Datalab. We recommend that you use Vertex AI Workbench for new projects and [migrate your Datalab notebooks to Vertex AI Workbench](https://cloud.google.com/datalab/docs/resources/troubleshooting#migrate). For more information, see [Deprecation information](https://cloud.google.com/datalab/docs/resources/deprecation). To get help migrating Datalab projects to Vertex AI Workbench see [Get help](https://cloud.google.com/datalab/docs/resources/support#get-help). 4 | -------------------------------------------------------------------------------- /datalab/README: -------------------------------------------------------------------------------- 1 | Everything under datalab namespace is actively maintained but no new features are being added. Please use corresponding libraries under google.datalab namespace (source code under google/datalab directory). 2 | 3 | To migrate existing code that relies on datalab namespace, since most API interfaces are the same between google.datalab and datalab, usually you just need to change the import namespace. The magic interface is different for bigquery though (%%sql --> %%bq). 4 | 5 | For more details please see https://github.com/googledatalab/pydatalab/wiki/%60datalab%60-to-%60google.datalab%60-Migration-Guide. 6 | -------------------------------------------------------------------------------- /datalab/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /datalab/bigquery/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - BigQuery Functionality.""" 14 | from __future__ import absolute_import 15 | 16 | from ._csv_options import CSVOptions 17 | from ._dataset import Dataset, Datasets 18 | from ._dialect import Dialect 19 | from ._federated_table import FederatedTable 20 | from ._job import Job 21 | from ._query import Query 22 | from ._query_job import QueryJob 23 | from ._query_results_table import QueryResultsTable 24 | from ._query_stats import QueryStats 25 | from ._sampling import Sampling 26 | from ._schema import Schema 27 | from ._table import Table, TableMetadata 28 | from ._udf import UDF 29 | from ._utils import TableName, DatasetName 30 | from ._view import View 31 | 32 | __all__ = ['CSVOptions', 'Dataset', 'Datasets', 'Dialect', 'FederatedTable', 'Query', 'QueryJob', 33 | 'QueryResultsTable', 'QueryStats', 'Sampling', 'Schema', 'Table', 'TableMetadata', 34 | 'UDF', 'TableName', 'DatasetName', 'View'] 35 | 36 | 37 | def wait_any(jobs, timeout=None): 38 | """ Return when at least one of the specified jobs has completed or timeout expires. 39 | 40 | Args: 41 | jobs: a list of Jobs to wait on. 42 | timeout: a timeout in seconds to wait for. None (the default) means no timeout. 43 | Returns: 44 | Once at least one job completes, a list of all completed jobs. 45 | If the call times out then an empty list will be returned. 46 | 47 | """ 48 | return Job.wait_any(jobs, timeout) 49 | 50 | 51 | def wait_all(jobs, timeout=None): 52 | """ Return when all of the specified jobs have completed or timeout expires. 53 | 54 | Args: 55 | jobs: a single Job or list of Jobs to wait on. 56 | timeout: a timeout in seconds to wait for. None (the default) means no timeout. 57 | Returns: 58 | A list of completed Jobs. If the call timed out this will be shorter than the 59 | list of jobs supplied as a parameter. 60 | """ 61 | return Job.wait_all(jobs, timeout) 62 | -------------------------------------------------------------------------------- /datalab/bigquery/_dialect.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - BigQuery SQL Dialect""" 14 | from __future__ import absolute_import 15 | 16 | 17 | class Dialect(object): 18 | """ 19 | Represents the default BigQuery SQL dialect 20 | """ 21 | _global_dialect = None 22 | 23 | def __init__(self, bq_dialect): 24 | self._global_dialect = bq_dialect 25 | 26 | @property 27 | def bq_dialect(self): 28 | """Retrieves the value of the bq_dialect property. 29 | 30 | Returns: 31 | The default BigQuery SQL dialect 32 | """ 33 | return self._global_dialect 34 | 35 | def set_bq_dialect(self, bq_dialect): 36 | """ Set the default BigQuery SQL dialect""" 37 | if bq_dialect in ['legacy', 'standard']: 38 | self._global_dialect = bq_dialect 39 | 40 | @staticmethod 41 | def default(): 42 | """Retrieves the default BigQuery SQL dialect, creating it if necessary. 43 | 44 | Returns: 45 | An initialized and shared instance of a Dialect object. 46 | """ 47 | if Dialect._global_dialect is None: 48 | Dialect._global_dialect = Dialect('legacy') 49 | return Dialect._global_dialect 50 | -------------------------------------------------------------------------------- /datalab/bigquery/_query_stats.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | 14 | """Implements representation of BigQuery query job dry run results.""" 15 | from __future__ import absolute_import 16 | from __future__ import unicode_literals 17 | from builtins import object 18 | 19 | 20 | class QueryStats(object): 21 | """A wrapper for statistics returned by a dry run query. Useful so we can get an HTML 22 | representation in a notebook. 23 | """ 24 | 25 | def __init__(self, total_bytes, is_cached): 26 | self.total_bytes = float(total_bytes) 27 | self.is_cached = is_cached 28 | 29 | def _repr_html_(self): 30 | self.total_bytes = QueryStats._size_formatter(self.total_bytes) 31 | return """ 32 |

Dry run information: %s to process, results %s

33 | """ % (self.total_bytes, "cached" if self.is_cached else "not cached") 34 | 35 | @staticmethod 36 | def _size_formatter(byte_num, suf='B'): 37 | for mag in ['', 'K', 'M', 'G', 'T']: 38 | if byte_num < 1000.0: 39 | if suf == 'B': # Don't do fractional bytes 40 | return "%5d%s%s" % (int(byte_num), mag, suf) 41 | return "%3.1f%s%s" % (byte_num, mag, suf) 42 | byte_num /= 1000.0 43 | return "%.1f%s%s".format(byte_num, 'P', suf) 44 | -------------------------------------------------------------------------------- /datalab/bigquery/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | 14 | from __future__ import absolute_import 15 | 16 | from . import _bigquery 17 | 18 | __all__ = ['_bigquery'] 19 | -------------------------------------------------------------------------------- /datalab/context/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - authorization Context for Cloud services.""" 14 | 15 | from ._context import Context 16 | from ._project import Project, Projects 17 | 18 | __all__ = ['Context', 'Project', 'Projects'] 19 | -------------------------------------------------------------------------------- /datalab/context/_api.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Implements HTTP API wrapper.""" 14 | from __future__ import absolute_import 15 | from __future__ import unicode_literals 16 | from builtins import object 17 | 18 | import datalab.utils 19 | 20 | 21 | class Api(object): 22 | """A helper class to issue API HTTP requests to resource manager API.""" 23 | 24 | _ENDPOINT = 'https://cloudresourcemanager.googleapis.com/v1' 25 | _PROJECT_PATH = '/projects/%s' 26 | _PROJECTS_PATH = '/projects' 27 | 28 | def __init__(self, credentials): 29 | self._credentials = credentials 30 | 31 | def projects_list(self, max_results=0, page_token=None): 32 | url = Api._ENDPOINT + Api._PROJECTS_PATH 33 | args = {} 34 | if max_results != 0: 35 | args['pageSize'] = max_results 36 | if page_token is not None: 37 | args['pageToken'] = page_token 38 | 39 | return datalab.utils.Http.request(url, args=args, credentials=self._credentials) 40 | 41 | def project_get(self, projectId): 42 | url = Api._ENDPOINT + (Api._PROJECT_PATH % projectId) 43 | return datalab.utils.Http.request(url, credentials=self._credentials) 44 | -------------------------------------------------------------------------------- /datalab/context/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | 14 | from __future__ import absolute_import 15 | 16 | from . import _projects 17 | 18 | __all__ = ['_projects'] 19 | -------------------------------------------------------------------------------- /datalab/context/commands/_projects.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Implements listing projects and setting default project.""" 14 | 15 | from __future__ import absolute_import 16 | from __future__ import unicode_literals 17 | try: 18 | import IPython 19 | import IPython.core.magic 20 | import IPython.core.display 21 | except ImportError: 22 | raise Exception('This module can only be loaded in ipython.') 23 | 24 | import fnmatch 25 | 26 | import datalab.utils.commands 27 | import datalab.context 28 | 29 | 30 | @IPython.core.magic.register_line_cell_magic 31 | def projects(line, cell=None): 32 | parser = datalab.utils.commands.CommandParser.create('projects') 33 | 34 | list_parser = parser.subcommand('list', 'List available projects.') 35 | list_parser.add_argument('-f', '--filter', 36 | help='Optional wildcard id filter string used to limit the results') 37 | list_parser.set_defaults(func=_list_line, cell_prohibited=True) 38 | 39 | set_parser = parser.subcommand('set', 'Set the default project.') 40 | set_parser.add_argument('id', help='The ID of the project to use') 41 | set_parser.set_defaults(func=_set_line, cell_prohibited=True) 42 | 43 | return datalab.utils.commands.handle_magic_line(line, cell, parser) 44 | 45 | 46 | def _list_line(args, _): 47 | # TODO(gram): should we use a paginated table? 48 | filter_ = args['filter'] if args['filter'] else '*' 49 | data = [{'id': project.id, 'name': project.name} 50 | for project in datalab.context.Projects() if fnmatch.fnmatch(project.id, filter_)] 51 | return IPython.core.display.HTML(datalab.utils.commands.HtmlBuilder.render_table(data, ['id', 52 | 'name'])) 53 | 54 | 55 | def _set_line(args, _): 56 | id_ = args['id'] if args['id'] else '' 57 | context = datalab.context.Context.default() 58 | context.set_project_id(id_) 59 | datalab.context.Projects.save_default_id(id_) 60 | -------------------------------------------------------------------------------- /datalab/data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - Generic SQL Helpers.""" 14 | from __future__ import absolute_import 15 | from __future__ import unicode_literals 16 | 17 | from ._csv import Csv 18 | from ._sql_module import SqlModule 19 | from ._sql_statement import SqlStatement 20 | from ._utils import tokenize 21 | 22 | __all__ = ['Csv', 'SqlModule', 'SqlStatement', 'tokenize'] 23 | -------------------------------------------------------------------------------- /datalab/data/commands/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | # Copyright 2015 Google Inc. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations under 12 | # the License. 13 | 14 | 15 | from . import _sql 16 | 17 | __all__ = ['_sql'] 18 | -------------------------------------------------------------------------------- /datalab/notebook/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Datalab - notebook extension functionality.""" 14 | 15 | try: 16 | import IPython as _ 17 | except ImportError: 18 | raise Exception('This package requires an IPython notebook installation') 19 | 20 | __all__ = ['_'] 21 | 22 | 23 | def _jupyter_nbextension_paths(): 24 | return [dict(section="notebook", src="static", dest="gcpdatalab")] 25 | -------------------------------------------------------------------------------- /datalab/notebook/static/bigquery.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | 15 | table.bqsv { 16 | font-family: inherit; 17 | font-size: smaller; 18 | } 19 | table.bqsv th, table.bqsv td { 20 | border: solid 1px #cfcfcf; 21 | } 22 | th.bqsv_expanded, th.bqsv_collapsed { 23 | background-color: #f7f7f7; 24 | } 25 | th.bqsv_colheader { 26 | font-weight: bold; 27 | background-color: #e7e7e7; 28 | } 29 | tbody.bqsv_hidden { 30 | display: none; 31 | } 32 | th.bqsv_expanded:before { 33 | content: '\25be ' 34 | } 35 | th.bqsv_collapsed:before { 36 | content: '\25b8 ' 37 | } 38 | -------------------------------------------------------------------------------- /datalab/notebook/static/charting.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | 15 | table.google-visualization-table-table, table.dataframe { 16 | font-family: inherit; 17 | font-size: smaller; 18 | } 19 | tr.gchart-table-row { 20 | } 21 | tr.gchart-table-headerrow, table.dataframe thead th { 22 | font-weight: bold; 23 | background-color: #e7e7e7; 24 | } 25 | tr.gchart-table-oddrow, table.dataframe tr:nth-child(odd) { 26 | background-color: #f7f7f7; 27 | } 28 | tr.gchart-table-selectedTableRow { 29 | background-color: #e3f2fd; 30 | } 31 | tr.gchart-table-hoverrow, table.dataframe tr:hover { 32 | background-color: #bbdefb; 33 | } 34 | td.gchart-table-cell, table.dataframe td { 35 | border: solid 1px #cfcfcf; 36 | } 37 | td.gchart-table-rownumcell, table.dataframe tr th { 38 | border: solid 1px #cfcfcf; 39 | color: #999; 40 | } 41 | th.gchart-table-headercell, table.dataframe th { 42 | border: solid 1px #cfcfcf; 43 | } 44 | div.bqgc { 45 | display: flex; 46 | justify-content: center; 47 | } 48 | div.bqgc img { 49 | max-width: none; // Fix the conflict with maps and Bootstrap that messes up zoom controls. 50 | } 51 | .gchart-slider { 52 | width: 80%; 53 | float: left; 54 | } 55 | .gchart-slider-value { 56 | text-align: center; 57 | float: left; 58 | width: 20%; 59 | } 60 | .gchart-control { 61 | padding-top: 10px; 62 | padding-bottom: 10px; 63 | } 64 | .gchart-controls { 65 | font-size: 14px; 66 | color: #333333; 67 | background: #f4f4f4; 68 | padding: 10px; 69 | width: 180px; 70 | float: left; 71 | } 72 | .bqgc { 73 | padding: 0; 74 | max-width: 100%; 75 | } 76 | .bqgc-controlled { 77 | display: flex; 78 | flex-direction: row; 79 | justify-content:space-between; 80 | } 81 | .bqgc-container { 82 | display: block; 83 | } 84 | .bqgc-ml-metrics { 85 | display: flex; 86 | flex-direction: row; 87 | justify-content:left; 88 | } 89 | -------------------------------------------------------------------------------- /datalab/notebook/static/element.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"element.js","sourceRoot":"","sources":["element.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;;AAEH,0EAA0E;AAE1E,IAAO,OAAO,CA0Cb;AA1CD,WAAO,OAAO,EAAC,CAAC;IAEhB,4CAA4C;IAE1C,YAAY,CAAC;IAEb,IAAI,gBAAgB,GAAQ,IAAI,CAAC;IAEjC,iBAAiB,MAAW;QAC1B,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAClD,CAAC;IAED;QACE,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;YACrB,+EAA+E;YAC/E,IAAI,SAAS,GAAG,gBAAgB,CAAC;YACjC,gBAAgB,GAAG,IAAI,CAAC;YAExB,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,cAAqB,IAAS,EAAE,GAAQ,EAAE,YAAiB,EAAE,MAAW;QACtE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YACnB,YAAY,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,CAAC;YACJ,IAAI,MAAM,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC;YAE9C,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,CAAC;gBACrC,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;oBACtB,gBAAgB,GAAG,EAAE,CAAC;oBACtB,QAAQ,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;gBACzE,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAChC,CAAC;YACD,IAAI,CAAC,CAAC;gBACJ,OAAO,CAAC,MAAM,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAnBe,YAAI,OAmBnB,CAAA;AACH,CAAC,EA1CM,OAAO,KAAP,OAAO,QA0Cb;AAED,iBAAS,OAAO,CAAC"} -------------------------------------------------------------------------------- /datalab/notebook/static/element.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | 15 | /// 16 | 17 | module Element { 18 | 19 | // RequireJS plugin to resolve DOM elements. 20 | 21 | 'use strict'; 22 | 23 | var pendingCallbacks: any = null; 24 | 25 | function resolve(cbInfo: any): void { 26 | cbInfo.cb(document.getElementById(cbInfo.name)); 27 | } 28 | 29 | function domReadyCallback(): void { 30 | if (pendingCallbacks) { 31 | // Clear out pendingCallbacks, so any future requests are immediately resolved. 32 | var callbacks = pendingCallbacks; 33 | pendingCallbacks = null; 34 | 35 | callbacks.forEach(resolve); 36 | } 37 | } 38 | 39 | export function load(name: any, req: any, loadCallback: any, config: any): void { 40 | if (config.isBuild) { 41 | loadCallback(null); 42 | } 43 | else { 44 | var cbInfo = { name: name, cb: loadCallback }; 45 | 46 | if (document.readyState == 'loading') { 47 | if (!pendingCallbacks) { 48 | pendingCallbacks = []; 49 | document.addEventListener('DOMContentLoaded', domReadyCallback, false); 50 | } 51 | 52 | pendingCallbacks.push(cbInfo); 53 | } 54 | else { 55 | resolve(cbInfo); 56 | } 57 | } 58 | } 59 | } 60 | 61 | export = Element; 62 | 63 | -------------------------------------------------------------------------------- /datalab/notebook/static/extern/d3.parcoords.css: -------------------------------------------------------------------------------- 1 | .parcoords > svg, .parcoords > canvas { 2 | /*font: 14px sans-serif;*/ 3 | position: absolute; 4 | } 5 | .parcoords > canvas { 6 | pointer-events: none; 7 | } 8 | .parcoords rect.background { 9 | fill: transparent; 10 | } 11 | .parcoords rect.background:hover { 12 | fill: rgba(120,120,120,0.2); 13 | } 14 | .parcoords .resize rect { 15 | fill: rgba(0,0,0,0.1); 16 | } 17 | .parcoords rect.extent { 18 | fill: rgba(255,255,255,0.25); 19 | stroke: rgba(0,0,0,0.6); 20 | } 21 | .parcoords .axis line, .parcoords .axis path { 22 | fill: none; 23 | stroke: #222; 24 | shape-rendering: crispEdges; 25 | } 26 | .parcoords canvas { 27 | opacity: 1; 28 | -moz-transition: opacity 0.3s; 29 | -webkit-transition: opacity 0.3s; 30 | -o-transition: opacity 0.3s; 31 | } 32 | .parcoords canvas.faded { 33 | opacity: 0.25; 34 | } 35 | .parcoords_grid { text-align: center; } 36 | .parcoords_grid .row, .header { clear: left; font-size: 16px; line-height: 18px; height: 18px; } 37 | .parcoords_grid .row:nth-child(odd) { background: rgba(0,0,0,0.05); } 38 | .parcoords_grid .row:hover { background: green; } 39 | .parcoords_grid .header { font-weight: bold; } 40 | .parcoords_grid .cell { float: left; overflow: hidden; white-space: nowrap; width: 120px; height: 18px; } 41 | .parcoords_grid .col-0 { width: 110px; } 42 | -------------------------------------------------------------------------------- /datalab/notebook/static/extern/parcoords-LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Kai Chang 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * The name Kai Chang may not be used to endorse or promote products 15 | derived from this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT, 21 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 24 | OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 25 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /datalab/notebook/static/extern/sylvester-LICENSE.txt: -------------------------------------------------------------------------------- 1 | (The MIT License) 2 | 3 | Copyright (c) 2007-2015 James Coglan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /datalab/notebook/static/job.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | 15 | p.jobfail { 16 | color: red; 17 | } 18 | p.jobsucceed { 19 | color: green; 20 | } 21 | p.jobfooter { 22 | font-size: smaller; 23 | } 24 | -------------------------------------------------------------------------------- /datalab/notebook/static/job.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"job.js","sourceRoot":"","sources":["job.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;;AAEH,0EAA0E;AAK1E,IAAO,GAAG,CAuCT;AAvCD,WAAO,GAAG,EAAC,CAAC;IAEV,iBAAiB,GAAQ,EAAE,QAAa,EAAE,QAAa,EAAE,QAAa,EAClE,eAAuB,EAAE,eAAuB;QAClD,IAAI,IAAI,GAAG,mBAAmB,GAAG,QAAQ,GAAG,GAAG,GAAG,QAAQ,CAAC;QAC3D,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,KAAU,EAAE,OAAY;YAC9D,KAAK,GAAG,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC;YAC/B,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;gBACV,GAAG,CAAC,SAAS,GAAG,4CAA4C,GAAG,KAAK;sBAC9D,MAAM,CAAC;gBACb,MAAM,CAAC;YACT,CAAC;YACD,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;gBACpB,GAAG,CAAC,SAAS,GAAG,gCAAgC,CAAC;YACnD,CAAC;YAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;gBACxB,GAAG,CAAC,SAAS,GAAG,2DAA2D;oBAC3D,eAAe,CAAC;YAClC,CAAC;YAAC,IAAI,CAAC,CAAC;gBACN,GAAG,CAAC,SAAS,GAAG,6CAA6C;sBACzD,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,GAAG,MAAM,GAAG,eAAe,CAAC;gBAC/D,UAAU,CAAC;oBACL,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,CAAC;gBAC/E,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kEAAkE;IAClE,gBAAuB,GAAQ,EAAE,MAAW,EAAE,QAAgB,EAAE,QAAgB,EAC5E,QAAa,EAAE,eAAuB,EAAE,eAAuB;QACjE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;YAC3C,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,CAAC;YAC7E,MAAM,CAAC;QACT,CAAC;QACD,sDAAsD;QACtD,MAAM,CAAC,EAAE,CAAC,qBAAqB,EAAE,UAAS,CAAM;YAC9C,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;IACL,CAAC;IAVe,UAAM,SAUrB,CAAA;AACH,CAAC,EAvCM,GAAG,KAAH,GAAG,QAuCT;AAGD,iBAAS,GAAG,CAAC"} -------------------------------------------------------------------------------- /datalab/notebook/static/style.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"style.js","sourceRoot":"","sources":["style.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;;AAEH,wCAAwC;AAExC,0EAA0E;AAE1E,IAAO,KAAK,CA2DX;AA3DD,WAAO,KAAK,EAAC,CAAC;IAEZ,YAAY,CAAC;IAEb,gFAAgF;IAChF,IAAI,iBAAiB,GAAQ,EAAE,CAAC;IAEhC,mEAAmE;IACnE,IAAI,kBAAkB,GAAkB,IAAI,CAAC;IAE7C,uBAAuB,GAAW;QAChC,iBAAiB,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;QAE9B,IAAI,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAChD,UAAU,CAAC,IAAI,GAAG,UAAU,CAAC;QAC7B,UAAU,CAAC,GAAG,GAAG,YAAY,CAAC;QAC9B,UAAU,CAAC,IAAI,GAAG,GAAG,CAAC;QAEtB,QAAQ,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACnE,CAAC;IAED;QACE,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC;YACvB,8EAA8E;YAC9E,IAAI,WAAW,GAAkB,kBAAkB,CAAC;YACpD,kBAAkB,GAAG,IAAI,CAAC;YAE1B,WAAW,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,cAAqB,GAAW,EAAE,GAAQ,EAAE,YAAiB,EAAE,MAAW;QACxE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YACnB,YAAY,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,CAAC;YACJ,sFAAsF;YACtF,mCAAmC;YACnC,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;YAE5B,uEAAuE;YACvE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC3B,MAAM,CAAC;YACT,CAAC;YACD,iBAAiB,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;YAE9B,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,CAAC;gBACrC,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC;oBACxB,kBAAkB,GAAG,EAAE,CAAC;oBACxB,QAAQ,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;gBACzE,CAAC;gBAED,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/B,CAAC;YACD,IAAI,CAAC,CAAC;gBACJ,aAAa,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IA3Be,UAAI,OA2BnB,CAAA;AACH,CAAC,EA3DM,KAAK,KAAL,KAAK,QA2DX;AAED,iBAAS,KAAK,CAAC"} -------------------------------------------------------------------------------- /datalab/notebook/static/visualization.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"visualization.js","sourceRoot":"","sources":["visualization.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;;AAIH,0EAA0E;AAK1E,IAAO,aAAa,CA6DnB;AA7DD,WAAO,aAAa,EAAC,CAAC;IAEpB,YAAY,CAAC;IAEb,kFAAkF;IAClF,IAAI,KAAK,GAAQ;QACf,QAAQ,EAAE,EAAE;QACZ,SAAS,EAAE,EAAE;KACd,CAAC;IAEF,6BAA6B,QAAa;QACxC,6DAA6D;QAC7D,2FAA2F;QAC3F,IAAI,YAAY,GAAW,2BAA2B,CAAC;QACvD,MAAM,CAAC,YAAY,CAAC,GAAG,QAAQ,CAAC;QAEhC,IAAI,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,CAAC,IAAI,GAAG,iBAAiB,CAAC;QAChC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,MAAM,CAAC,GAAG,GAAG,wCAAwC,GAAG,YAAY,CAAC;QACrE,QAAQ,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAC/D,CAAC;IAED,qCAAqC,EAAO;QAC1C,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAC3B,CAAC;IAED,mCAAmC,KAAU,EAAE,SAAc;QAC3D,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YACjB,IAAI,oBAAoB,GAAG;gBACzB,QAAQ,EAAE,KAAK;gBACf,QAAQ,EAAE,cAAa,SAAS,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,CAAC;aACzE,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,oBAAoB,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,mBAAmB,CAAC;QAClB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACV,yBAAyB,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YAC3D,KAAK,GAAG,IAAI,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,cAAqB,IAAS,EAAE,GAAQ,EAAE,QAAa,EAAE,MAAW;QAClE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YACnB,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,CAAC;YACJ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;gBACV,sFAAsF;gBACtF,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC1B,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;YACD,IAAI,CAAC,CAAC;gBACJ,8EAA8E;gBAC9E,yBAAyB,CAAC,CAAE,IAAI,CAAE,EAAE,CAAE,QAAQ,CAAE,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;IACH,CAAC;IAfe,kBAAI,OAenB,CAAA;AACH,CAAC,EA7DM,aAAa,KAAb,aAAa,QA6DnB;AAED,iBAAS,aAAa,CAAC"} -------------------------------------------------------------------------------- /datalab/stackdriver/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - Stackdriver Functionality.""" 14 | -------------------------------------------------------------------------------- /datalab/stackdriver/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | 14 | from __future__ import absolute_import 15 | 16 | from . import _monitoring 17 | 18 | __all__ = ['_monitoring'] 19 | -------------------------------------------------------------------------------- /datalab/stackdriver/monitoring/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - Monitoring Functionality.""" 14 | 15 | from __future__ import absolute_import 16 | 17 | from google.cloud.monitoring import enums 18 | from ._group import Groups 19 | from ._metric import MetricDescriptors 20 | from ._query import Query 21 | from ._query_metadata import QueryMetadata 22 | from ._resource import ResourceDescriptors 23 | 24 | Aligner = enums.Aggregation.Aligner 25 | Reducer = enums.Aggregation.Reducer 26 | 27 | __all__ = ['Aligner', 'Reducer', 'Groups', 'MetricDescriptors', 'Query', 'QueryMetadata', 28 | 'ResourceDescriptors'] 29 | -------------------------------------------------------------------------------- /datalab/storage/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - Cloud Storage Functionality.""" 14 | from __future__ import absolute_import 15 | 16 | from ._bucket import Bucket, Buckets 17 | from ._item import Item, Items 18 | 19 | __all__ = ['Bucket', 'Buckets', 'Item', 'Items'] 20 | -------------------------------------------------------------------------------- /datalab/storage/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | 14 | from __future__ import absolute_import 15 | 16 | from . import _storage 17 | 18 | __all__ = ['_storage'] 19 | -------------------------------------------------------------------------------- /datalab/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - Internal Helpers.""" 14 | 15 | from ._async import async_, async_function, async_method 16 | from ._gcp_job import GCPJob 17 | from ._http import Http, RequestException 18 | from ._iterator import Iterator 19 | from ._job import Job, JobError 20 | from ._json_encoder import JSONEncoder 21 | from ._lru_cache import LRUCache 22 | from ._lambda_job import LambdaJob 23 | from ._dataflow_job import DataflowJob 24 | from ._utils import print_exception_with_last_stack, get_item, compare_datetimes, \ 25 | pick_unused_port, is_http_running_on, gcs_copy_file 26 | 27 | __all__ = ['async_', 'async_function', 'async_method', 'GCPJob', 'Http', 'RequestException', 28 | 'Iterator', 'Job', 'JobError', 'JSONEncoder', 'LRUCache', 'LambdaJob', 'DataflowJob', 29 | 'print_exception_with_last_stack', 'get_item', 'compare_datetimes', 'pick_unused_port', 30 | 'is_http_running_on', 'gcs_copy_file'] 31 | -------------------------------------------------------------------------------- /datalab/utils/_dataflow_job.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Implements DataFlow Job functionality.""" 14 | 15 | 16 | from . import _job 17 | 18 | 19 | class DataflowJob(_job.Job): 20 | """Represents a DataFlow Job. 21 | """ 22 | 23 | def __init__(self, runner_results): 24 | """Initializes an instance of a DataFlow Job. 25 | 26 | Args: 27 | runner_results: a DataflowPipelineResult returned from Pipeline.run(). 28 | """ 29 | super(DataflowJob, self).__init__(runner_results._job.name) 30 | self._runner_results = runner_results 31 | 32 | def _refresh_state(self): 33 | """ Refresh the job info. """ 34 | 35 | # DataFlow's DataflowPipelineResult does not refresh state, so we have to do it ourselves 36 | # as a workaround. 37 | self._runner_results._job = ( 38 | self._runner_results._runner.dataflow_client.get_job(self._runner_results.job_id())) 39 | self._is_complete = self._runner_results.state in ['STOPPED', 'DONE', 'FAILED', 'CANCELLED'] 40 | self._fator_error = getattr(self._runner_results._runner, 'last_error_msg', None) 41 | -------------------------------------------------------------------------------- /datalab/utils/_gcp_job.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Implements GCP Job functionality.""" 14 | from __future__ import absolute_import 15 | from __future__ import unicode_literals 16 | 17 | import datalab.context 18 | from . import _job 19 | 20 | 21 | class GCPJob(_job.Job): 22 | """Represents a BigQuery Job. 23 | """ 24 | 25 | def __init__(self, job_id, context): 26 | """Initializes an instance of a Job. 27 | 28 | Args: 29 | job_id: the BigQuery job ID corresponding to this job. 30 | context: a Context object providing project_id and credentials. 31 | """ 32 | super(GCPJob, self).__init__(job_id) 33 | if context is None: 34 | context = datalab.context.Context.default() 35 | self._context = context 36 | self._api = self._create_api(context) 37 | 38 | def _create_api(self, context): 39 | raise Exception('_create_api must be defined in a derived class') 40 | 41 | def __repr__(self): 42 | """Returns a representation for the job for showing in the notebook. 43 | """ 44 | return 'Job %s/%s %s' % (self._context.project_id, self._job_id, self.state) 45 | -------------------------------------------------------------------------------- /datalab/utils/_iterator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Iterator class for iterable cloud lists.""" 14 | from __future__ import absolute_import 15 | from __future__ import unicode_literals 16 | from builtins import object 17 | 18 | 19 | class Iterator(object): 20 | """An iterator implementation that handles paging over a cloud list.""" 21 | 22 | def __init__(self, retriever): 23 | """Initializes an instance of an Iterator. 24 | 25 | Args: 26 | retriever: a function that can retrieve the next page of items. 27 | """ 28 | self._page_token = None 29 | self._first_page = True 30 | self._retriever = retriever 31 | self._count = 0 32 | 33 | def __iter__(self): 34 | """Provides iterator functionality.""" 35 | while self._first_page or (self._page_token is not None): 36 | items, next_page_token = self._retriever(self._page_token, self._count) 37 | 38 | self._page_token = next_page_token 39 | self._first_page = False 40 | if self._count == 0: 41 | self._count = len(items) 42 | 43 | for item in items: 44 | yield item 45 | 46 | def reset(self): 47 | """Resets the current iteration.""" 48 | self._page_token = None 49 | self._first_page = True 50 | self._count = 0 51 | -------------------------------------------------------------------------------- /datalab/utils/_json_encoder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """ JSON encoder that can handle Python datetime objects. """ 14 | 15 | from __future__ import absolute_import 16 | from __future__ import unicode_literals 17 | import datetime 18 | import json 19 | 20 | 21 | class JSONEncoder(json.JSONEncoder): 22 | """ A JSON encoder that can handle Python datetime objects. """ 23 | 24 | def default(self, obj): 25 | if isinstance(obj, datetime.date) or isinstance(obj, datetime.datetime): 26 | return obj.isoformat() 27 | elif isinstance(obj, datetime.timedelta): 28 | return (datetime.datetime.min + obj).time().isoformat() 29 | else: 30 | return super(JSONEncoder, self).default(obj) 31 | -------------------------------------------------------------------------------- /datalab/utils/_lambda_job.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Implements OS shell Job functionality.""" 14 | from __future__ import absolute_import 15 | from __future__ import unicode_literals 16 | 17 | from . import _async 18 | from . import _job 19 | 20 | 21 | class LambdaJob(_job.Job): 22 | """Represents an lambda function as a Job. 23 | """ 24 | 25 | def __init__(self, fn, job_id, *args, **kwargs): 26 | """Initializes an instance of a Job. 27 | 28 | Args: 29 | fn: the lambda function to execute asyncronously 30 | job_id: an optional ID for the job. If None, a UUID will be generated. 31 | """ 32 | super(LambdaJob, self).__init__(job_id) 33 | self._future = _async.async_.executor.submit(fn, *args, **kwargs) 34 | 35 | def __repr__(self): 36 | """Returns a representation for the job for showing in the notebook. 37 | """ 38 | return 'Job %s %s' % (self._job_id, self.state) 39 | 40 | # TODO: ShellJob, once we need it, should inherit on LambdaJob: 41 | # import subprocess 42 | # LambdaJob(subprocess.check_output, id, command_line, shell=True) 43 | -------------------------------------------------------------------------------- /datalab/utils/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | # flake8: noqa 14 | 15 | from __future__ import absolute_import 16 | from __future__ import unicode_literals 17 | 18 | # Support functions for magics and display help. 19 | from ._commands import CommandParser 20 | from ._html import Html, HtmlBuilder 21 | from ._utils import * 22 | 23 | # Magics 24 | from . import _chart 25 | from . import _chart_data 26 | from . import _csv 27 | from . import _extension 28 | from . import _job 29 | from . import _modules 30 | -------------------------------------------------------------------------------- /datalab/utils/commands/_extension.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - Extension cell magic.""" 14 | from __future__ import absolute_import 15 | from __future__ import unicode_literals 16 | 17 | try: 18 | import IPython 19 | import IPython.core.display 20 | import IPython.core.magic 21 | except ImportError: 22 | raise Exception('This module can only be loaded in ipython.') 23 | 24 | from . import _commands 25 | from . import _utils 26 | 27 | 28 | @IPython.core.magic.register_line_cell_magic 29 | def extension(line, cell=None): 30 | """ Load an extension. Use %extension --help for more details. """ 31 | parser = _commands.CommandParser(prog='%extension', description=""" 32 | Load an extension into Datalab. Currently only mathjax is supported. 33 | """) 34 | subparser = parser.subcommand('mathjax', 'Enabled MathJaX support in Datalab.') 35 | subparser.set_defaults(ext='mathjax') 36 | parser.set_defaults(func=_extension) 37 | return _utils.handle_magic_line(line, cell, parser) 38 | 39 | 40 | def _extension(args, cell): 41 | ext = args['ext'] 42 | if ext == 'mathjax': 43 | # TODO: remove this with the next version update 44 | # MathJax is now loaded by default for all notebooks 45 | return 46 | raise Exception('Unsupported extension %s' % ext) 47 | -------------------------------------------------------------------------------- /datalab/utils/commands/_modules.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Implementation of various module magics""" 14 | from __future__ import absolute_import 15 | from __future__ import unicode_literals 16 | 17 | try: 18 | import IPython 19 | import IPython.core.magic 20 | except ImportError: 21 | raise Exception('This module can only be loaded in ipython.') 22 | 23 | import sys 24 | import types 25 | 26 | from . import _commands 27 | from . import _utils 28 | 29 | 30 | @IPython.core.magic.register_line_cell_magic 31 | def pymodule(line, cell=None): 32 | """Creates and subsequently auto-imports a python module. 33 | """ 34 | parser = _commands.CommandParser.create('pymodule') 35 | parser.add_argument('-n', '--name', 36 | help='the name of the python module to create and import') 37 | parser.set_defaults(func=_pymodule_cell) 38 | return _utils.handle_magic_line(line, cell, parser) 39 | 40 | 41 | def _pymodule_cell(args, cell): 42 | if cell is None: 43 | raise Exception('The code for the module must be included') 44 | 45 | name = args['name'] 46 | module = _create_python_module(name, cell) 47 | 48 | # Automatically import the newly created module by assigning it to a variable 49 | # named the same name as the module name. 50 | ipy = IPython.get_ipython() 51 | ipy.push({name: module}) 52 | 53 | 54 | def _create_python_module(name, code): 55 | # By convention the module is associated with a file name matching the module name 56 | module = types.ModuleType(str(name)) 57 | module.__file__ = name 58 | module.__name__ = name 59 | 60 | exec(code, module.__dict__) 61 | 62 | # Hold on to the module if the code executed successfully 63 | sys.modules[name] = module 64 | return module 65 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googledatalab/pydatalab/8bf007da3e43096aa3a3dca158fc56b286ba6f5c/docs/.nojekyll -------------------------------------------------------------------------------- /docs/README: -------------------------------------------------------------------------------- 1 | To use, install the prerequisites and the pydatalab module: 2 | 3 | pip install sphinx sphinx_rtd_theme sphinxcontrib-napoleon 4 | pip install .. # from docs directory 5 | 6 | then in the docs directory, do 'make html' (or epub, or text, etc). 7 | 8 | Output will be in $BUILDDIR, defaulting to ../../datalab-docs. 9 | -------------------------------------------------------------------------------- /docs/datalab.bigquery.rst: -------------------------------------------------------------------------------- 1 | datalab.bigquery Module 2 | ======================= 3 | 4 | .. automodule:: datalab.bigquery 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autoclass:: datalab.bigquery.CSVOptions 10 | :members: 11 | 12 | .. autoclass:: datalab.bigquery.Dataset 13 | :members: 14 | 15 | .. autoclass:: datalab.bigquery.DatasetName 16 | :members: 17 | 18 | .. autoclass:: datalab.bigquery.Datasets 19 | :members: 20 | 21 | .. autoclass:: datalab.bigquery.FederatedTable 22 | :members: 23 | 24 | .. autoclass:: datalab.bigquery.Job 25 | :members: 26 | 27 | .. autoclass:: datalab.bigquery.Query 28 | :members: 29 | 30 | .. autoclass:: datalab.bigquery.QueryJob 31 | :members: 32 | 33 | .. autoclass:: datalab.bigquery.QueryResultsTable 34 | :members: 35 | 36 | .. autoclass:: datalab.bigquery.QueryStats 37 | :members: 38 | 39 | .. autoclass:: datalab.bigquery.Sampling 40 | :members: 41 | 42 | .. autoclass:: datalab.bigquery.Schema 43 | :members: 44 | 45 | .. autoclass:: datalab.bigquery.Table 46 | :members: 47 | 48 | .. autoclass:: datalab.bigquery.TableMetadata 49 | :members: 50 | 51 | .. autoclass:: datalab.bigquery.TableName 52 | :members: 53 | 54 | .. autoclass:: datalab.bigquery.UDF 55 | :members: 56 | 57 | .. autoclass:: datalab.bigquery.View 58 | :members: 59 | -------------------------------------------------------------------------------- /docs/datalab.context.rst: -------------------------------------------------------------------------------- 1 | datalab.context Module 2 | ====================== 3 | 4 | .. autoclass:: datalab.context.Context 5 | :members: 6 | 7 | .. autoclass:: datalab.context.Project 8 | :members: 9 | 10 | .. autoclass:: datalab.context.Projects 11 | :members: 12 | 13 | -------------------------------------------------------------------------------- /docs/datalab.data.rst: -------------------------------------------------------------------------------- 1 | datalab.data Module 2 | =================== 3 | 4 | .. automodule:: datalab.data 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autoclass:: datalab.data.Csv 10 | :members: 11 | 12 | .. autoclass:: datalab.data.SqlModule 13 | :members: 14 | 15 | .. autoclass:: datalab.data.SqlStatement 16 | :members: 17 | 18 | -------------------------------------------------------------------------------- /docs/datalab.stackdriver.monitoring.rst: -------------------------------------------------------------------------------- 1 | datalab.stackdriver.monitoring Module 2 | ===================================== 3 | 4 | .. autoclass:: datalab.stackdriver.monitoring.Groups 5 | :members: 6 | 7 | .. autoclass:: datalab.stackdriver.monitoring.MetricDescriptors 8 | :members: 9 | 10 | .. autoclass:: datalab.stackdriver.monitoring.ResourceDescriptors 11 | :members: 12 | 13 | .. autoclass:: datalab.stackdriver.monitoring.Query 14 | :members: 15 | 16 | .. autoclass:: datalab.stackdriver.monitoring.QueryMetadata 17 | :members: 18 | -------------------------------------------------------------------------------- /docs/datalab.storage.rst: -------------------------------------------------------------------------------- 1 | datalab.storage Module 2 | ====================== 3 | 4 | .. automodule:: datalab.storage 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autoclass:: datalab.storage.Bucket 10 | :members: 11 | 12 | .. autoclass:: datalab.storage.Buckets 13 | :members: 14 | 15 | .. autoclass:: datalab.storage.Item 16 | :members: 17 | 18 | .. autoclass:: datalab.storage.Items 19 | :members: 20 | 21 | -------------------------------------------------------------------------------- /docs/gen-magic-rst.ipy: -------------------------------------------------------------------------------- 1 | import subprocess, pkgutil, importlib, sys 2 | from cStringIO import StringIO 3 | 4 | # import submodules 5 | datalab_submodules = ['datalab.' + s + '.commands' for _,s,_ in pkgutil.iter_modules(['../datalab'])] 6 | google_submodules = ['google.datalab.' + s + '.commands' for _,s,_ in pkgutil.iter_modules(['../google/datalab'])] 7 | 8 | def generate_magic_docs(submodules, header, dir, ignored_magics=None): 9 | if not ignored_magics: 10 | ignored_magics = [] 11 | for m in submodules: 12 | try: 13 | importlib.import_module(m) 14 | except: 15 | sys.stderr.write('WARNING, could not find module ' + m + '. Ignoring..\n') 16 | 17 | magic_regex = "find " + dir + " -name '*.py' -exec perl -e '$f=join(\"\",<>); print \"$1\n\" if $f=~/register_line_cell_magic\ndef ([^\(]+)/m' {} \;" 18 | magics = subprocess.check_output(magic_regex, shell=True) 19 | 20 | reSTfile = open(header + '.rst', 'w') 21 | indent = '\n ' 22 | 23 | reSTfile.write(header + '\n') 24 | reSTfile.write('=======================\n\n') 25 | 26 | for m in sorted(magics.split()): 27 | if m in ignored_magics: 28 | sys.stderr.write('Ignoring magic ' + m + '\n') 29 | else: 30 | print('working on magic: '+ m) 31 | reSTfile.write('.. attribute:: %' + m + '\n') 32 | reSTfile.write('.. parsed-literal::\n') 33 | # hijack stdout since the ipython kernel call writes to stdout/err directly 34 | # and does not return its output 35 | tmpStdout, sys.stdout = sys.stdout, StringIO() 36 | get_ipython().magic(m + ' -h') 37 | resultout = sys.stdout.getvalue().splitlines() 38 | sys.stdout = tmpStdout 39 | reSTfile.writelines(indent + indent.join(resultout) + '\n\n') 40 | 41 | 42 | generate_magic_docs(datalab_submodules, 'datalab Commands', '../datalab', ignored_magics=['chart', 'csv']); 43 | generate_magic_docs(google_submodules, 'google.datalab Commands', '../google'); 44 | 45 | -------------------------------------------------------------------------------- /docs/google.datalab.bigquery.rst: -------------------------------------------------------------------------------- 1 | google.datalab.bigquery Module 2 | ============================== 3 | 4 | .. automodule:: google.datalab.bigquery 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autoclass:: google.datalab.bigquery.CSVOptions 10 | :members: 11 | 12 | .. autoclass:: google.datalab.bigquery.Dataset 13 | :members: 14 | 15 | .. autoclass:: google.datalab.bigquery.DatasetName 16 | :members: 17 | 18 | .. autoclass:: google.datalab.bigquery.Datasets 19 | :members: 20 | 21 | .. autoclass:: google.datalab.bigquery.ExternalDataSource 22 | :members: 23 | 24 | .. autoclass:: google.datalab.bigquery.Query 25 | :members: 26 | 27 | .. autoclass:: google.datalab.bigquery.QueryOutput 28 | :members: 29 | 30 | .. autoclass:: google.datalab.bigquery.QueryResultsTable 31 | :members: 32 | 33 | .. autoclass:: google.datalab.bigquery.QueryStats 34 | :members: 35 | 36 | .. autoclass:: google.datalab.bigquery.Sampling 37 | :members: 38 | 39 | .. autoclass:: google.datalab.bigquery.Schema 40 | :members: 41 | 42 | .. autoclass:: google.datalab.bigquery.SchemaField 43 | :members: 44 | 45 | .. autoclass:: google.datalab.bigquery.Table 46 | :members: 47 | 48 | .. autoclass:: google.datalab.bigquery.TableMetadata 49 | :members: 50 | 51 | .. autoclass:: google.datalab.bigquery.TableName 52 | :members: 53 | 54 | .. autoclass:: google.datalab.bigquery.UDF 55 | :members: 56 | 57 | .. autoclass:: google.datalab.bigquery.View 58 | :members: 59 | -------------------------------------------------------------------------------- /docs/google.datalab.data.rst: -------------------------------------------------------------------------------- 1 | google.datalab.data Module 2 | ========================== 3 | 4 | .. automodule:: google.datalab.data 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autoclass:: google.datalab.data.CsvFile 10 | :members: 11 | 12 | -------------------------------------------------------------------------------- /docs/google.datalab.ml.rst: -------------------------------------------------------------------------------- 1 | google.datalab.ml Module 2 | ======================== 3 | 4 | .. automodule:: google.datalab.ml 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autoclass:: google.datalab.ml.Job 10 | :members: 11 | 12 | .. autoclass:: google.datalab.ml.Jobs 13 | :members: 14 | 15 | .. autoclass:: google.datalab.ml.Summary 16 | :members: 17 | 18 | .. autoclass:: google.datalab.ml.TensorBoard 19 | :members: 20 | 21 | .. autoclass:: google.datalab.ml.CsvDataSet 22 | :members: 23 | 24 | .. autoclass:: google.datalab.ml.BigQueryDataSet 25 | :members: 26 | 27 | .. autoclass:: google.datalab.ml.Models 28 | :members: 29 | 30 | .. autoclass:: google.datalab.ml.ModelVersions 31 | :members: 32 | 33 | .. autoclass:: google.datalab.ml.ConfusionMatrix 34 | :members: 35 | 36 | .. autoclass:: google.datalab.ml.FeatureSliceView 37 | :members: 38 | 39 | .. autoclass:: google.datalab.ml.CloudTrainingConfig 40 | :members: 41 | 42 | -------------------------------------------------------------------------------- /docs/google.datalab.rst: -------------------------------------------------------------------------------- 1 | google.datalab Module 2 | ===================== 3 | 4 | .. automodule:: google.datalab 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autoclass:: google.datalab.Context 10 | :members: 11 | 12 | .. autoclass:: google.datalab.Job 13 | :members: 14 | 15 | -------------------------------------------------------------------------------- /docs/google.datalab.stackdriver.monitoring.rst: -------------------------------------------------------------------------------- 1 | google.datalab.stackdriver.monitoring Module 2 | ============================================ 3 | 4 | .. autoclass:: google.datalab.stackdriver.monitoring.Groups 5 | :members: 6 | 7 | .. autoclass:: google.datalab.stackdriver.monitoring.MetricDescriptors 8 | :members: 9 | 10 | .. autoclass:: google.datalab.stackdriver.monitoring.ResourceDescriptors 11 | :members: 12 | 13 | .. autoclass:: google.datalab.stackdriver.monitoring.Query 14 | :members: 15 | 16 | .. autoclass:: google.datalab.stackdriver.monitoring.QueryMetadata 17 | :members: 18 | -------------------------------------------------------------------------------- /docs/google.datalab.storage.rst: -------------------------------------------------------------------------------- 1 | google.datalab.storage Module 2 | ============================= 3 | 4 | .. automodule:: google.datalab.storage 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autoclass:: google.datalab.storage.Bucket 10 | :members: 11 | 12 | .. autoclass:: google.datalab.storage.Buckets 13 | :members: 14 | 15 | .. autoclass:: google.datalab.storage.Object 16 | :members: 17 | 18 | .. autoclass:: google.datalab.storage.Objects 19 | :members: 20 | 21 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to Cloud Datalab's documentation 2 | ======================================== 3 | 4 | google.datalab namespace 5 | ######################## 6 | 7 | Contents: 8 | 9 | .. toctree:: 10 | google.datalab 11 | google.datalab.bigquery 12 | google.datalab.data 13 | google.datalab.ml 14 | google.datalab.stackdriver.monitoring 15 | google.datalab.storage 16 | google.datalab Commands 17 | 18 | ML Toolbox: 19 | 20 | .. toctree:: 21 | mltoolbox.classification.dnn 22 | mltoolbox.classification.linear 23 | mltoolbox.regression.dnn 24 | mltoolbox.regression.linear 25 | mltoolbox.image.classification 26 | 27 | datalab namespace 28 | ################# 29 | 30 | Please note, this namespace is planned to be phased out. You are strongly encouraged to move to the new google.datalab namespace above. 31 | 32 | Contents: 33 | 34 | .. toctree:: 35 | datalab.bigquery 36 | datalab.context 37 | datalab.data 38 | datalab.stackdriver.monitoring 39 | datalab.storage 40 | datalab Commands 41 | 42 | 43 | Indices and tables 44 | ================== 45 | 46 | * :ref:`genindex` 47 | * :ref:`modindex` 48 | * :ref:`search` 49 | 50 | -------------------------------------------------------------------------------- /docs/mltoolbox.classification.dnn.rst: -------------------------------------------------------------------------------- 1 | mltoolbox.classification.dnn 2 | ============================ 3 | 4 | .. automodule:: mltoolbox.classification.dnn 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autofunction:: mltoolbox.classification.dnn.analyze 10 | 11 | .. autofunction:: mltoolbox.classification.dnn.analyze_async 12 | 13 | .. autofunction:: mltoolbox.classification.dnn.batch_predict 14 | 15 | .. autofunction:: mltoolbox.classification.dnn.batch_predict_async 16 | 17 | .. autofunction:: mltoolbox.classification.dnn.predict 18 | 19 | .. autofunction:: mltoolbox.classification.dnn.train 20 | 21 | .. autofunction:: mltoolbox.classification.dnn.train_async 22 | 23 | -------------------------------------------------------------------------------- /docs/mltoolbox.classification.linear.rst: -------------------------------------------------------------------------------- 1 | mltoolbox.classification.linear 2 | =============================== 3 | 4 | .. automodule:: mltoolbox.classification.linear 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autofunction:: mltoolbox.classification.linear.analyze 10 | 11 | .. autofunction:: mltoolbox.classification.linear.analyze_async 12 | 13 | .. autofunction:: mltoolbox.classification.linear.batch_predict 14 | 15 | .. autofunction:: mltoolbox.classification.linear.batch_predict_async 16 | 17 | .. autofunction:: mltoolbox.classification.linear.predict 18 | 19 | .. autofunction:: mltoolbox.classification.linear.train 20 | 21 | .. autofunction:: mltoolbox.classification.linear.train_async 22 | 23 | -------------------------------------------------------------------------------- /docs/mltoolbox.image.classification.rst: -------------------------------------------------------------------------------- 1 | mltoolbox.image.classification 2 | ============================== 3 | 4 | .. automodule:: mltoolbox.image.classification 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autofunction:: mltoolbox.image.classification.preprocess 10 | 11 | .. autofunction:: mltoolbox.image.classification.preprocess_async 12 | 13 | .. autofunction:: mltoolbox.image.classification.train 14 | 15 | .. autofunction:: mltoolbox.image.classification.train_async 16 | 17 | .. autofunction:: mltoolbox.image.classification.predict 18 | 19 | .. autofunction:: mltoolbox.image.classification.batch_predict 20 | 21 | .. autofunction:: mltoolbox.image.classification.batch_predict_async 22 | 23 | -------------------------------------------------------------------------------- /docs/mltoolbox.regression.dnn.rst: -------------------------------------------------------------------------------- 1 | mltoolbox.regression.dnn 2 | ======================== 3 | 4 | .. automodule:: mltoolbox.regression.dnn 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autofunction:: mltoolbox.regression.dnn.analyze 10 | 11 | .. autofunction:: mltoolbox.regression.dnn.analyze_async 12 | 13 | .. autofunction:: mltoolbox.regression.dnn.batch_predict 14 | 15 | .. autofunction:: mltoolbox.regression.dnn.batch_predict_async 16 | 17 | .. autofunction:: mltoolbox.regression.dnn.predict 18 | 19 | .. autofunction:: mltoolbox.regression.dnn.train 20 | 21 | .. autofunction:: mltoolbox.regression.dnn.train_async 22 | 23 | -------------------------------------------------------------------------------- /docs/mltoolbox.regression.linear.rst: -------------------------------------------------------------------------------- 1 | mltoolbox.regression.linear 2 | =========================== 3 | 4 | .. automodule:: mltoolbox.regression.linear 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. autofunction:: mltoolbox.regression.linear.analyze 10 | 11 | .. autofunction:: mltoolbox.regression.linear.analyze_async 12 | 13 | .. autofunction:: mltoolbox.regression.linear.batch_predict 14 | 15 | .. autofunction:: mltoolbox.regression.linear.batch_predict_async 16 | 17 | .. autofunction:: mltoolbox.regression.linear.predict 18 | 19 | .. autofunction:: mltoolbox.regression.linear.train 20 | 21 | .. autofunction:: mltoolbox.regression.linear.train_async 22 | 23 | -------------------------------------------------------------------------------- /google/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /google/datalab/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from google.datalab._context import Context 14 | from google.datalab._job import Job, JobError 15 | import warnings 16 | 17 | __all__ = ['Context', 'Job', 'JobError'] 18 | 19 | warnings.warn("Datalab is deprecated. For more information, see https://cloud.google.com/datalab/docs/resources/deprecation.", DeprecationWarning) -------------------------------------------------------------------------------- /google/datalab/bigquery/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - BigQuery Functionality.""" 14 | from __future__ import absolute_import 15 | 16 | from ._csv_options import CSVOptions 17 | from ._dataset import Dataset, Datasets 18 | from ._external_data_source import ExternalDataSource 19 | from ._query import Query 20 | from ._query_output import QueryOutput 21 | from ._query_results_table import QueryResultsTable 22 | from ._query_stats import QueryStats 23 | from ._sampling import Sampling 24 | from ._schema import Schema, SchemaField 25 | from ._table import Table, TableMetadata 26 | from ._udf import UDF 27 | from ._utils import TableName, DatasetName 28 | from ._view import View 29 | 30 | __all__ = ['CSVOptions', 'Dataset', 'Datasets', 'ExternalDataSource', 'Query', 'QueryOutput', 31 | 'QueryResultsTable', 'QueryStats', 'Sampling', 'Schema', 'SchemaField', 'Table', 32 | 'TableMetadata', 'UDF', 'TableName', 'DatasetName', 'View'] 33 | -------------------------------------------------------------------------------- /google/datalab/bigquery/_query_stats.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | 14 | """Implements representation of BigQuery query job dry run results.""" 15 | from __future__ import absolute_import 16 | from __future__ import unicode_literals 17 | from builtins import object 18 | 19 | 20 | class QueryStats(object): 21 | """A wrapper for statistics returned by a dry run query. Useful so we can get an HTML 22 | representation in a notebook. 23 | """ 24 | 25 | def __init__(self, total_bytes, is_cached): 26 | self.total_bytes = float(total_bytes) 27 | self.is_cached = is_cached 28 | 29 | def _repr_html_(self): 30 | self.total_bytes = QueryStats._size_formatter(self.total_bytes) 31 | return """ 32 |

Dry run information: %s to process, results %s

33 | """ % (self.total_bytes, "cached" if self.is_cached else "not cached") 34 | 35 | @staticmethod 36 | def _size_formatter(byte_num, suf='B'): 37 | for mag in ['', 'K', 'M', 'G', 'T']: 38 | if byte_num < 1000.0: 39 | if suf == 'B': # Don't do fractional bytes 40 | return "%5d%s%s" % (int(byte_num), mag, suf) 41 | return "%3.1f%s%s" % (byte_num, mag, suf) 42 | byte_num /= 1000.0 43 | return "%.1f%s%s".format(byte_num, 'P', suf) 44 | -------------------------------------------------------------------------------- /google/datalab/bigquery/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | 14 | from __future__ import absolute_import 15 | 16 | from . import _bigquery 17 | 18 | __all__ = ['_bigquery'] 19 | -------------------------------------------------------------------------------- /google/datalab/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """This module defines the `datalab` magics""" 16 | 17 | from __future__ import absolute_import 18 | 19 | from . import _datalab 20 | 21 | __all__ = ['_datalab'] 22 | -------------------------------------------------------------------------------- /google/datalab/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /google/datalab/contrib/bigquery/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /google/datalab/contrib/bigquery/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | from ._bigquery import get_airflow_spec_from_config # noqa 13 | -------------------------------------------------------------------------------- /google/datalab/contrib/bigquery/operators/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /google/datalab/contrib/bigquery/operators/_bq_extract_operator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import google 14 | from airflow.models import BaseOperator 15 | from airflow.utils.decorators import apply_defaults 16 | 17 | 18 | class ExtractOperator(BaseOperator): 19 | 20 | template_fields = ('table', 'path') 21 | 22 | @apply_defaults 23 | def __init__(self, path, table, format='csv', csv_options=None, *args, **kwargs): 24 | super(ExtractOperator, self).__init__(*args, **kwargs) 25 | self.table = table 26 | self.path = path 27 | self.format = format 28 | self.csv_options = csv_options or {} 29 | 30 | def execute(self, context): 31 | source_table = google.datalab.bigquery.Table(self.table, context=None) 32 | 33 | csv_kwargs = {} 34 | if 'delimiter' in self.csv_options: 35 | csv_kwargs['csv_delimiter'] = self.csv_options['delimiter'] 36 | if 'header' in self.csv_options: 37 | csv_kwargs['csv_header'] = self.csv_options['header'] 38 | if 'compress' in self.csv_options: 39 | csv_kwargs['compress'] = self.csv_options['compress'] 40 | 41 | job = source_table.extract( 42 | self.path, format='CSV' if self.format == 'csv' else 'NEWLINE_DELIMITED_JSON', **csv_kwargs) 43 | 44 | if job.failed: 45 | raise Exception('Extract failed: %s' % str(job.fatal_error)) 46 | elif job.errors: 47 | raise Exception('Extract completed with errors: %s' % str(job.errors)) 48 | return { 49 | 'result': job.result() 50 | } 51 | -------------------------------------------------------------------------------- /google/datalab/contrib/mlworkbench/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | 14 | from __future__ import absolute_import 15 | 16 | from ._local_predict import get_prediction_results, get_probs_for_labels, local_batch_predict 17 | from ._prediction_explainer import PredictionExplainer 18 | 19 | __all__ = ['get_prediction_results', 'get_probs_for_labels', 'local_batch_predict', 20 | 'PredictionExplainer'] 21 | -------------------------------------------------------------------------------- /google/datalab/contrib/mlworkbench/_archive.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Google Cloud Platform library - ml cell magic.""" 16 | from __future__ import absolute_import 17 | from __future__ import unicode_literals 18 | 19 | import os 20 | import shutil 21 | import tempfile 22 | import tensorflow as tf 23 | 24 | import google.datalab.contrib.mlworkbench._shell_process as _shell_process 25 | 26 | 27 | def extract_archive(archive_path, dest): 28 | """Extract a local or GCS archive file to a folder. 29 | 30 | Args: 31 | archive_path: local or gcs path to a *.tar.gz or *.tar file 32 | dest: local folder the archive will be extracted to 33 | """ 34 | # Make the dest folder if it does not exist 35 | if not os.path.isdir(dest): 36 | os.makedirs(dest) 37 | 38 | try: 39 | tmpfolder = None 40 | 41 | if (not tf.gfile.Exists(archive_path)) or tf.gfile.IsDirectory(archive_path): 42 | raise ValueError('archive path %s is not a file' % archive_path) 43 | 44 | if archive_path.startswith('gs://'): 45 | # Copy the file to a local temp folder 46 | tmpfolder = tempfile.mkdtemp() 47 | cmd_args = ['gsutil', 'cp', archive_path, tmpfolder] 48 | _shell_process.run_and_monitor(cmd_args, os.getpid()) 49 | archive_path = os.path.join(tmpfolder, os.path.name(archive_path)) 50 | 51 | if archive_path.lower().endswith('.tar.gz'): 52 | flags = '-xzf' 53 | elif archive_path.lower().endswith('.tar'): 54 | flags = '-xf' 55 | else: 56 | raise ValueError('Only tar.gz or tar.Z files are supported.') 57 | 58 | cmd_args = ['tar', flags, archive_path, '-C', dest] 59 | _shell_process.run_and_monitor(cmd_args, os.getpid()) 60 | finally: 61 | if tmpfolder: 62 | shutil.rmtree(tmpfolder) 63 | -------------------------------------------------------------------------------- /google/datalab/contrib/mlworkbench/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | 14 | from __future__ import absolute_import 15 | 16 | from . import _ml 17 | 18 | __all__ = ['_ml'] 19 | -------------------------------------------------------------------------------- /google/datalab/contrib/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /google/datalab/contrib/pipeline/airflow/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | from ._airflow import Airflow # noqa 13 | -------------------------------------------------------------------------------- /google/datalab/contrib/pipeline/airflow/_airflow.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | import google.datalab.storage as storage 14 | 15 | 16 | class Airflow(object): 17 | """ Represents a Airflow object that encapsulates a set of functionality relating to the 18 | Cloud Airflow service. 19 | 20 | This object can be used to generate the python airflow spec. 21 | """ 22 | 23 | def __init__(self, gcs_dag_bucket, gcs_dag_file_path=None): 24 | """ Initializes an instance of a Airflow object. 25 | 26 | Args: 27 | gcs_dag_bucket: Bucket where Airflow expects dag files to be uploaded. 28 | gcs_dag_file_path: File path of the Airflow dag files. 29 | """ 30 | self._gcs_dag_bucket = gcs_dag_bucket 31 | self._gcs_dag_file_path = gcs_dag_file_path or '' 32 | 33 | def deploy(self, name, dag_string): 34 | if self._gcs_dag_file_path is not '' and self._gcs_dag_file_path.endswith('/') is False: 35 | self._gcs_dag_file_path = self._gcs_dag_file_path + '/' 36 | file_name = '{0}{1}.py'.format(self._gcs_dag_file_path, name) 37 | 38 | bucket = storage.Bucket(self._gcs_dag_bucket) 39 | file_object = bucket.object(file_name) 40 | file_object.write_stream(dag_string, 'text/plain') 41 | -------------------------------------------------------------------------------- /google/datalab/contrib/pipeline/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /google/datalab/contrib/pipeline/composer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | from ._composer import Composer # noqa 13 | -------------------------------------------------------------------------------- /google/datalab/contrib/pipeline/composer/_api.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Implements Composer HTTP API wrapper.""" 14 | import google.datalab.utils 15 | 16 | 17 | class Api(object): 18 | """A helper class to issue Composer HTTP requests.""" 19 | 20 | _ENDPOINT = 'https://composer.googleapis.com/v1alpha1' 21 | _ENVIRONMENTS_PATH_FORMAT = '/projects/%s/locations/%s/environments/%s' 22 | 23 | @staticmethod 24 | def get_environment_details(zone, environment): 25 | """ Issues a request to Composer to get the environment details. 26 | 27 | Args: 28 | zone: GCP zone of the composer environment 29 | environment: name of the Composer environment 30 | Returns: 31 | A parsed result object. 32 | Raises: 33 | Exception if there is an error performing the operation. 34 | """ 35 | default_context = google.datalab.Context.default() 36 | url = (Api._ENDPOINT + (Api._ENVIRONMENTS_PATH_FORMAT % (default_context.project_id, zone, 37 | environment))) 38 | 39 | return google.datalab.utils.Http.request(url, credentials=default_context.credentials) 40 | -------------------------------------------------------------------------------- /google/datalab/data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - Generic SQL Helpers.""" 14 | from __future__ import absolute_import 15 | from __future__ import unicode_literals 16 | 17 | from ._csv_file import CsvFile 18 | 19 | __all__ = ['CsvFile'] 20 | -------------------------------------------------------------------------------- /google/datalab/ml/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | # flake8: noqa 14 | 15 | """CloudML Helper Library.""" 16 | 17 | from __future__ import absolute_import 18 | 19 | from ._job import Jobs, Job 20 | from ._summary import Summary 21 | from ._tensorboard import TensorBoard 22 | from ._dataset import CsvDataSet, BigQueryDataSet, TransformedDataSet 23 | from ._cloud_models import Models, ModelVersions 24 | from ._confusion_matrix import ConfusionMatrix 25 | from ._feature_slice_view import FeatureSliceView 26 | from ._cloud_training_config import CloudTrainingConfig 27 | from ._fasets import FacetsOverview, FacetsDiveview 28 | from ._metrics import Metrics 29 | from ._util import * 30 | -------------------------------------------------------------------------------- /google/datalab/notebook/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Datalab - notebook extension functionality.""" 14 | 15 | try: 16 | import IPython as _ 17 | except ImportError: 18 | raise Exception('This package requires an IPython notebook installation') 19 | 20 | __all__ = ['_'] 21 | 22 | 23 | def _jupyter_nbextension_paths(): 24 | return [dict(section="notebook", src="static", dest="gcpdatalab")] 25 | -------------------------------------------------------------------------------- /google/datalab/notebook/static/bigquery.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | 15 | table.bqsv { 16 | font-family: inherit; 17 | font-size: smaller; 18 | } 19 | table.bqsv th, table.bqsv td { 20 | border: solid 1px #cfcfcf; 21 | } 22 | th.bqsv_expanded, th.bqsv_collapsed { 23 | background-color: #f7f7f7; 24 | } 25 | th.bqsv_colheader { 26 | font-weight: bold; 27 | background-color: #e7e7e7; 28 | } 29 | tbody.bqsv_hidden { 30 | display: none; 31 | } 32 | th.bqsv_expanded:before { 33 | content: '\25be ' 34 | } 35 | th.bqsv_collapsed:before { 36 | content: '\25b8 ' 37 | } 38 | -------------------------------------------------------------------------------- /google/datalab/notebook/static/charting.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | 15 | table.google-visualization-table-table, table.dataframe { 16 | font-family: inherit; 17 | font-size: smaller; 18 | } 19 | tr.gchart-table-row { 20 | } 21 | tr.gchart-table-headerrow, table.dataframe thead th { 22 | font-weight: bold; 23 | background-color: #e7e7e7; 24 | } 25 | tr.gchart-table-oddrow, table.dataframe tr:nth-child(odd) { 26 | background-color: #f7f7f7; 27 | } 28 | tr.gchart-table-selectedTableRow { 29 | background-color: #e3f2fd; 30 | } 31 | tr.gchart-table-hoverrow, table.dataframe tr:hover { 32 | background-color: #bbdefb; 33 | } 34 | td.gchart-table-cell, table.dataframe td { 35 | border: solid 1px #cfcfcf; 36 | } 37 | td.gchart-table-rownumcell, table.dataframe tr th { 38 | border: solid 1px #cfcfcf; 39 | color: #999; 40 | } 41 | th.gchart-table-headercell, table.dataframe th { 42 | border: solid 1px #cfcfcf; 43 | } 44 | div.bqgc { 45 | display: flex; 46 | justify-content: center; 47 | } 48 | div.bqgc img { 49 | max-width: none; // Fix the conflict with maps and Bootstrap that messes up zoom controls. 50 | } 51 | .gchart-slider { 52 | width: 80%; 53 | float: left; 54 | } 55 | .gchart-slider-value { 56 | text-align: center; 57 | float: left; 58 | width: 20%; 59 | } 60 | .gchart-control { 61 | padding-top: 10px; 62 | padding-bottom: 10px; 63 | } 64 | .gchart-controls { 65 | font-size: 14px; 66 | color: #333333; 67 | background: #f4f4f4; 68 | padding: 10px; 69 | width: 180px; 70 | float: left; 71 | } 72 | .bqgc { 73 | padding: 0; 74 | max-width: 100%; 75 | } 76 | .bqgc-controlled { 77 | display: flex; 78 | flex-direction: row; 79 | justify-content:space-between; 80 | } 81 | .bqgc-container { 82 | display: block; 83 | } 84 | .bqgc-ml-metrics { 85 | display: flex; 86 | flex-direction: row; 87 | justify-content:left; 88 | } 89 | -------------------------------------------------------------------------------- /google/datalab/notebook/static/element.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | 15 | /// 16 | 17 | module Element { 18 | 19 | // RequireJS plugin to resolve DOM elements. 20 | 21 | 'use strict'; 22 | 23 | var pendingCallbacks: any = null; 24 | 25 | function resolve(cbInfo: any): void { 26 | cbInfo.cb(document.getElementById(cbInfo.name)); 27 | } 28 | 29 | function domReadyCallback(): void { 30 | if (pendingCallbacks) { 31 | // Clear out pendingCallbacks, so any future requests are immediately resolved. 32 | var callbacks = pendingCallbacks; 33 | pendingCallbacks = null; 34 | 35 | callbacks.forEach(resolve); 36 | } 37 | } 38 | 39 | export function load(name: any, req: any, loadCallback: any, config: any): void { 40 | if (config.isBuild) { 41 | loadCallback(null); 42 | } 43 | else { 44 | var cbInfo = { name: name, cb: loadCallback }; 45 | 46 | if (document.readyState == 'loading') { 47 | if (!pendingCallbacks) { 48 | pendingCallbacks = []; 49 | document.addEventListener('DOMContentLoaded', domReadyCallback, false); 50 | } 51 | 52 | pendingCallbacks.push(cbInfo); 53 | } 54 | else { 55 | resolve(cbInfo); 56 | } 57 | } 58 | } 59 | } 60 | 61 | export = Element; 62 | 63 | -------------------------------------------------------------------------------- /google/datalab/notebook/static/extern/d3.parcoords.css: -------------------------------------------------------------------------------- 1 | .parcoords > svg, .parcoords > canvas { 2 | /*font: 14px sans-serif;*/ 3 | position: absolute; 4 | } 5 | .parcoords > canvas { 6 | pointer-events: none; 7 | } 8 | .parcoords rect.background { 9 | fill: transparent; 10 | } 11 | .parcoords rect.background:hover { 12 | fill: rgba(120,120,120,0.2); 13 | } 14 | .parcoords .resize rect { 15 | fill: rgba(0,0,0,0.1); 16 | } 17 | .parcoords rect.extent { 18 | fill: rgba(255,255,255,0.25); 19 | stroke: rgba(0,0,0,0.6); 20 | } 21 | .parcoords .axis line, .parcoords .axis path { 22 | fill: none; 23 | stroke: #222; 24 | shape-rendering: crispEdges; 25 | } 26 | .parcoords canvas { 27 | opacity: 1; 28 | -moz-transition: opacity 0.3s; 29 | -webkit-transition: opacity 0.3s; 30 | -o-transition: opacity 0.3s; 31 | } 32 | .parcoords canvas.faded { 33 | opacity: 0.25; 34 | } 35 | .parcoords_grid { text-align: center; } 36 | .parcoords_grid .row, .header { clear: left; font-size: 16px; line-height: 18px; height: 18px; } 37 | .parcoords_grid .row:nth-child(odd) { background: rgba(0,0,0,0.05); } 38 | .parcoords_grid .row:hover { background: green; } 39 | .parcoords_grid .header { font-weight: bold; } 40 | .parcoords_grid .cell { float: left; overflow: hidden; white-space: nowrap; width: 120px; height: 18px; } 41 | .parcoords_grid .col-0 { width: 110px; } 42 | -------------------------------------------------------------------------------- /google/datalab/notebook/static/extern/parcoords-LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Kai Chang 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * The name Kai Chang may not be used to endorse or promote products 15 | derived from this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT, 21 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 24 | OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 25 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /google/datalab/notebook/static/extern/sylvester-LICENSE.txt: -------------------------------------------------------------------------------- 1 | (The MIT License) 2 | 3 | Copyright (c) 2007-2015 James Coglan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /google/datalab/notebook/static/job.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | 15 | p.jobfail { 16 | color: red; 17 | } 18 | p.jobsucceed { 19 | color: green; 20 | } 21 | p.jobfooter { 22 | font-size: smaller; 23 | } 24 | -------------------------------------------------------------------------------- /google/datalab/stackdriver/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - Stackdriver Functionality.""" 14 | -------------------------------------------------------------------------------- /google/datalab/stackdriver/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | 14 | from __future__ import absolute_import 15 | 16 | from . import _monitoring 17 | 18 | __all__ = ['_monitoring'] 19 | -------------------------------------------------------------------------------- /google/datalab/stackdriver/monitoring/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - Monitoring Functionality.""" 14 | 15 | from __future__ import absolute_import 16 | 17 | from google.cloud.monitoring import enums 18 | from ._group import Groups 19 | from ._metric import MetricDescriptors 20 | from ._query import Query 21 | from ._query_metadata import QueryMetadata 22 | from ._resource import ResourceDescriptors 23 | 24 | Aligner = enums.Aggregation.Aligner 25 | Reducer = enums.Aggregation.Reducer 26 | 27 | __all__ = ['Aligner', 'Reducer', 'Groups', 'MetricDescriptors', 'Query', 'QueryMetadata', 28 | 'ResourceDescriptors'] 29 | -------------------------------------------------------------------------------- /google/datalab/storage/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - Cloud Storage Functionality.""" 14 | from __future__ import absolute_import 15 | 16 | from ._bucket import Bucket, Buckets 17 | from ._object import Object, Objects 18 | 19 | __all__ = ['Bucket', 'Buckets', 'Object', 'Objects'] 20 | -------------------------------------------------------------------------------- /google/datalab/storage/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | 14 | from __future__ import absolute_import 15 | 16 | from . import _storage 17 | 18 | __all__ = ['_storage'] 19 | -------------------------------------------------------------------------------- /google/datalab/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Google Cloud Platform library - Internal Helpers.""" 14 | 15 | from ._async import async_, async_function, async_method 16 | from ._http import Http, RequestException 17 | from ._iterator import Iterator 18 | from ._json_encoder import JSONEncoder 19 | from ._lru_cache import LRUCache 20 | from ._lambda_job import LambdaJob 21 | from ._dataflow_job import DataflowJob 22 | from ._utils import print_exception_with_last_stack, get_item, compare_datetimes, \ 23 | pick_unused_port, is_http_running_on, gcs_copy_file, python_portable_string 24 | 25 | 26 | __all__ = ['async_', 'async_function', 'async_method', 'Http', 'RequestException', 'Iterator', 27 | 'JSONEncoder', 'LRUCache', 'LambdaJob', 'DataflowJob', 28 | 'print_exception_with_last_stack', 'get_item', 'compare_datetimes', 'pick_unused_port', 29 | 'is_http_running_on', 'gcs_copy_file', 'python_portable_string'] 30 | -------------------------------------------------------------------------------- /google/datalab/utils/_dataflow_job.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Implements DataFlow Job functionality.""" 14 | 15 | 16 | from google.datalab import _job 17 | 18 | 19 | class DataflowJob(_job.Job): 20 | """Represents a DataFlow Job. 21 | """ 22 | 23 | def __init__(self, runner_results): 24 | """Initializes an instance of a DataFlow Job. 25 | 26 | Args: 27 | runner_results: a DataflowPipelineResult returned from Pipeline.run(). 28 | """ 29 | super(DataflowJob, self).__init__(runner_results._job.name) 30 | self._runner_results = runner_results 31 | 32 | def _refresh_state(self): 33 | """ Refresh the job info. """ 34 | 35 | # DataFlow's DataflowPipelineResult does not refresh state, so we have to do it ourselves 36 | # as a workaround. 37 | # TODO(Change this to use runner_results.state once it refreshes itself) 38 | dataflow_internal_job = ( 39 | self._runner_results._runner.dataflow_client.get_job(self._runner_results.job_id())) 40 | self._is_complete = str(dataflow_internal_job.currentState) in ['JOB_STATE_STOPPED', 41 | 'JOB_STATE_DONE', 42 | 'JOB_STATE_FAILED', 43 | 'JOB_STATE_CANCELLED'] 44 | self._fatal_error = getattr(self._runner_results._runner, 'last_error_msg', None) 45 | # Sometimes Dataflow does not populate runner.last_error_msg even if the job fails. 46 | if self._fatal_error is None and self._runner_results.state == 'FAILED': 47 | self._fatal_error = 'FAILED' 48 | -------------------------------------------------------------------------------- /google/datalab/utils/_gcp_job.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Implements GCP Job functionality.""" 14 | from __future__ import absolute_import 15 | from __future__ import unicode_literals 16 | 17 | import google.datalab 18 | from google.datalab import _job 19 | 20 | 21 | class GCPJob(_job.Job): 22 | """Represents a BigQuery Job. 23 | """ 24 | 25 | def __init__(self, job_id, context): 26 | """Initializes an instance of a Job. 27 | 28 | Args: 29 | job_id: the BigQuery job ID corresponding to this job. 30 | context: a Context object providing project_id and credentials. 31 | """ 32 | super(GCPJob, self).__init__(job_id) 33 | if context is None: 34 | context = google.datalab.Context.default() 35 | self._context = context 36 | self._api = self._create_api(context) 37 | 38 | def _create_api(self, context): 39 | raise Exception('_create_api must be defined in a derived class') 40 | 41 | def __repr__(self): 42 | """Returns a representation for the job for showing in the notebook. 43 | """ 44 | return 'Job %s/%s %s' % (self._context.project_id, self._job_id, self.state) 45 | -------------------------------------------------------------------------------- /google/datalab/utils/_iterator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Iterator class for iterable cloud lists.""" 14 | from __future__ import absolute_import 15 | from __future__ import unicode_literals 16 | from builtins import object 17 | 18 | 19 | class Iterator(object): 20 | """An iterator implementation that handles paging over a cloud list.""" 21 | 22 | def __init__(self, retriever): 23 | """Initializes an instance of an Iterator. 24 | 25 | Args: 26 | retriever: a function that can retrieve the next page of items. 27 | """ 28 | self._page_token = None 29 | self._first_page = True 30 | self._retriever = retriever 31 | self._count = 0 32 | 33 | def __iter__(self): 34 | """Provides iterator functionality.""" 35 | while self._first_page or (self._page_token is not None): 36 | items, next_page_token = self._retriever(self._page_token, self._count) 37 | 38 | self._page_token = next_page_token 39 | self._first_page = False 40 | self._count += len(items) 41 | 42 | for item in items: 43 | yield item 44 | 45 | def reset(self): 46 | """Resets the current iteration.""" 47 | self._page_token = None 48 | self._first_page = True 49 | self._count = 0 50 | -------------------------------------------------------------------------------- /google/datalab/utils/_json_encoder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """ JSON encoder that can handle Python datetime objects. """ 14 | 15 | from __future__ import absolute_import 16 | from __future__ import unicode_literals 17 | import datetime 18 | import json 19 | 20 | 21 | class JSONEncoder(json.JSONEncoder): 22 | """ A JSON encoder that can handle Python datetime objects. """ 23 | 24 | def default(self, obj): 25 | if isinstance(obj, datetime.date) or isinstance(obj, datetime.datetime): 26 | return obj.isoformat() 27 | elif isinstance(obj, datetime.timedelta): 28 | return (datetime.datetime.min + obj).time().isoformat() 29 | else: 30 | return super(JSONEncoder, self).default(obj) 31 | -------------------------------------------------------------------------------- /google/datalab/utils/_lambda_job.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """Implements OS shell Job functionality.""" 14 | from __future__ import absolute_import 15 | from __future__ import unicode_literals 16 | 17 | from . import _async 18 | from google.datalab import _job 19 | 20 | 21 | class LambdaJob(_job.Job): 22 | """Represents an lambda function as a Job. 23 | """ 24 | 25 | def __init__(self, fn, job_id, *args, **kwargs): 26 | """Initializes an instance of a Job. 27 | 28 | Args: 29 | fn: the lambda function to execute asyncronously 30 | job_id: an optional ID for the job. If None, a UUID will be generated. 31 | """ 32 | super(LambdaJob, self).__init__(job_id) 33 | self._future = _async.async_.executor.submit(fn, *args, **kwargs) 34 | 35 | def __repr__(self): 36 | """Returns a representation for the job for showing in the notebook. 37 | """ 38 | return 'Job %s %s' % (self._job_id, self.state) 39 | 40 | # TODO: ShellJob, once we need it, should inherit on LambdaJob: 41 | # import subprocess 42 | # LambdaJob(subprocess.check_output, id, command_line, shell=True) 43 | -------------------------------------------------------------------------------- /google/datalab/utils/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | # flake8: noqa 14 | 15 | from __future__ import absolute_import 16 | from __future__ import unicode_literals 17 | 18 | # Support functions for magics and display help. 19 | from ._commands import CommandParser 20 | from ._html import Html, HtmlBuilder 21 | from ._utils import * 22 | 23 | # Magics 24 | from . import _chart 25 | from . import _chart_data 26 | from . import _csv 27 | from . import _job 28 | -------------------------------------------------------------------------------- /google/datalab/utils/facets/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | # The files in this directory are copied from 17 | # https://github.com/PAIR-code/facets/tree/master/facets_overview/python 18 | -------------------------------------------------------------------------------- /google/datalab/utils/facets/generic_feature_statistics_generator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Code for generating the feature_statistics proto from generic data. 16 | 17 | The proto is used as input for the Overview visualization. 18 | """ 19 | 20 | import warnings 21 | from .base_generic_feature_statistics_generator import BaseGenericFeatureStatisticsGenerator 22 | from . import feature_statistics_pb2 as fs 23 | 24 | 25 | class GenericFeatureStatisticsGenerator(BaseGenericFeatureStatisticsGenerator): 26 | """Generator of stats proto from generic data.""" 27 | 28 | def __init__(self): 29 | BaseGenericFeatureStatisticsGenerator.__init__( 30 | self, fs.FeatureNameStatistics, fs.DatasetFeatureStatisticsList, 31 | fs.Histogram) 32 | 33 | 34 | def ProtoFromDataFrames(dataframes): 35 | """Creates a feature statistics proto from a set of pandas dataframes. 36 | 37 | Args: 38 | dataframes: A list of dicts describing tables for each dataset for the 39 | proto. Each entry contains a 'table' field of the dataframe of the 40 | data 41 | and a 'name' field to identify the dataset in the proto. 42 | 43 | Returns: 44 | The feature statistics proto for the provided tables. 45 | """ 46 | warnings.warn( 47 | 'Use GenericFeatureStatisticsGenerator class method instead.', 48 | DeprecationWarning) 49 | return GenericFeatureStatisticsGenerator().ProtoFromDataFrames(dataframes) 50 | -------------------------------------------------------------------------------- /install-no-virtualenv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # Copyright 2016 Google Inc. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 6 | # in compliance with the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software distributed under the License 11 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 12 | # or implied. See the License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | # Build a distribution package 16 | tsc --module amd --noImplicitAny --outdir datalab/notebook/static datalab/notebook/static/*.ts 17 | pip install . 18 | jupyter nbextension install --py datalab.notebook 19 | rm datalab/notebook/static/*.js 20 | 21 | 22 | -------------------------------------------------------------------------------- /install-virtualenv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # Copyright 2016 Google Inc. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 6 | # in compliance with the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software distributed under the License 11 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 12 | # or implied. See the License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | # Build a distribution package 16 | tsc --module amd --noImplicitAny --outdir datalab/notebook/static datalab/notebook/static/*.ts 17 | pip install . 18 | jupyter nbextension install --py datalab.notebook --sys-prefix 19 | rm datalab/notebook/static/*.js 20 | -------------------------------------------------------------------------------- /legacy_tests/_util/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /legacy_tests/_util/lru_cache_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import unittest 16 | 17 | from datalab.utils._lru_cache import LRUCache 18 | 19 | 20 | class TestCases(unittest.TestCase): 21 | 22 | def test_cache_no_entry(self): 23 | cache = LRUCache(3) 24 | with self.assertRaises(KeyError): 25 | cache['a'] 26 | 27 | def test_cache_lookup(self): 28 | cache = LRUCache(4) 29 | for x in ['a', 'b', 'c', 'd']: 30 | cache[x] = x 31 | 32 | for x in ['a', 'b', 'c', 'd']: 33 | self.assertEqual(x, cache[x]) 34 | 35 | def test_cache_overflow(self): 36 | cache = LRUCache(3) 37 | for x in ['a', 'b', 'c', 'd']: 38 | cache[x] = x 39 | 40 | for x in ['b', 'c', 'd']: 41 | self.assertEqual(x, cache[x]) 42 | 43 | with self.assertRaises(KeyError): 44 | cache['a'] 45 | 46 | cache['b'] 47 | cache['d'] 48 | # 'c' should be LRU now 49 | cache['e'] = 'e' 50 | 51 | with self.assertRaises(KeyError): 52 | cache['c'] 53 | 54 | for x in ['b', 'd', 'e']: 55 | self.assertEqual(x, cache[x]) 56 | -------------------------------------------------------------------------------- /legacy_tests/_util/util_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import imp 16 | import unittest 17 | 18 | from datalab.utils._utils import get_item 19 | 20 | 21 | class TestCases(unittest.TestCase): 22 | 23 | @staticmethod 24 | def _get_data(): 25 | m = imp.new_module('baz') 26 | exec('x = 99', m.__dict__) 27 | data = { 28 | 'foo': { 29 | 'bar': { 30 | 'xyz': 0 31 | }, 32 | 'm': m 33 | } 34 | } 35 | return data 36 | 37 | def test_no_entry(self): 38 | data = TestCases._get_data() 39 | self.assertIsNone(get_item(data, 'x')) 40 | self.assertIsNone(get_item(data, 'bar.x')) 41 | self.assertIsNone(get_item(data, 'foo.bar.x')) 42 | self.assertIsNone(get_item(globals(), 'datetime.bar.x')) 43 | 44 | def test_entry(self): 45 | data = TestCases._get_data() 46 | self.assertEquals(data['foo']['bar']['xyz'], get_item(data, 'foo.bar.xyz')) 47 | self.assertEquals(data['foo']['bar'], get_item(data, 'foo.bar')) 48 | self.assertEquals(data['foo'], get_item(data, 'foo')) 49 | self.assertEquals(data['foo']['m'], get_item(data, 'foo.m')) 50 | self.assertEquals(99, get_item(data, 'foo.m.x')) 51 | -------------------------------------------------------------------------------- /legacy_tests/bigquery/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /legacy_tests/bigquery/udf_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import mock 16 | import unittest 17 | 18 | import google.auth 19 | import datalab.bigquery 20 | import datalab.context 21 | 22 | 23 | class TestCases(unittest.TestCase): 24 | 25 | def test_sql_building(self): 26 | context = self._create_context() 27 | table = datalab.bigquery.Table('test:requestlogs.today', context=context) 28 | 29 | udf = self._create_udf() 30 | query = datalab.bigquery.Query('SELECT * FROM foo($t)', t=table, udfs=[udf], context=context) 31 | 32 | expected_js = '\nfoo=function(r,emit) { emit({output1: r.field2, output2: r.field1 }); };\n' +\ 33 | 'bigquery.defineFunction(\'foo\', ["field1", "field2"], ' +\ 34 | '[{"name": "output1", "type": "integer"}, ' +\ 35 | '{"name": "output2", "type": "string"}], foo);' 36 | self.assertEqual(query.sql, 'SELECT * FROM ' 37 | '(SELECT output1, output2 FROM foo([test:requestlogs.today]))') 38 | self.assertEqual(udf._code, expected_js) 39 | 40 | @staticmethod 41 | def _create_udf(): 42 | inputs = [('field1', 'string'), ('field2', 'integer')] 43 | outputs = [('output1', 'integer'), ('output2', 'string')] 44 | impl = 'function(r,emit) { emit({output1: r.field2, output2: r.field1 }); }' 45 | udf = datalab.bigquery.UDF(inputs, outputs, 'foo', impl) 46 | return udf 47 | 48 | @staticmethod 49 | def _create_context(): 50 | project_id = 'test' 51 | creds = mock.Mock(spec=google.auth.credentials.Credentials) 52 | return datalab.context.Context(project_id, creds) 53 | -------------------------------------------------------------------------------- /legacy_tests/data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /legacy_tests/kernel/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /legacy_tests/kernel/chart_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import unittest 16 | 17 | # import Python so we can mock the parts we need to here. 18 | import IPython.core.display 19 | import IPython.core.magic 20 | 21 | import datalab.utils.commands 22 | 23 | 24 | def noop_decorator(func): 25 | return func 26 | 27 | 28 | IPython.core.magic.register_line_cell_magic = noop_decorator 29 | IPython.core.magic.register_line_magic = noop_decorator 30 | IPython.core.magic.register_cell_magic = noop_decorator 31 | IPython.core.display.HTML = lambda x: x 32 | IPython.core.display.JSON = lambda x: x 33 | 34 | 35 | class TestCases(unittest.TestCase): 36 | 37 | def test_chart_cell(self): 38 | t = [{'country': 'US', 'quantity': 100}, {'country': 'ZA', 'quantity': 50}] 39 | IPython.get_ipython().user_ns = {} 40 | chart = datalab.utils.commands._chart._chart_cell({'chart': 'geo', 'data': t, 'fields': None}, 41 | '') 42 | self.assertTrue(chart.find('charts.render(') > 0) 43 | self.assertTrue(chart.find('\'geo\'') > 0) 44 | self.assertTrue(chart.find('"fields": "*"') > 0) 45 | self.assertTrue(chart.find('{"c": [{"v": "US"}, {"v": 100}]}') > 0 or 46 | chart.find('{"c": [{"v": 100}, {"v": "US"}]}') > 0) 47 | self.assertTrue(chart.find('{"c": [{"v": "ZA"}, {"v": 50}]}') > 0 or 48 | chart.find('{"c": [{"v": 50}, {"v": "ZA"}]}') > 0) 49 | 50 | def test_chart_magic(self): 51 | # TODO(gram): complete this test 52 | pass 53 | -------------------------------------------------------------------------------- /legacy_tests/kernel/commands_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import mock 16 | import unittest 17 | 18 | # import Python so we can mock the parts we need to here. 19 | import IPython 20 | import IPython.core.magic 21 | 22 | 23 | def noop_decorator(func): 24 | return func 25 | 26 | 27 | IPython.core.magic.register_line_cell_magic = noop_decorator 28 | IPython.core.magic.register_line_magic = noop_decorator 29 | IPython.core.magic.register_cell_magic = noop_decorator 30 | IPython.get_ipython = mock.Mock() 31 | 32 | 33 | class TestCases(unittest.TestCase): 34 | 35 | def test_create_args(self): 36 | # TODO(gram): complete this test 37 | pass 38 | -------------------------------------------------------------------------------- /legacy_tests/kernel/html_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import mock 16 | import unittest 17 | 18 | # import Python so we can mock the parts we need to here. 19 | import IPython 20 | import IPython.core.magic 21 | 22 | 23 | def noop_decorator(func): 24 | return func 25 | 26 | 27 | IPython.core.magic.register_line_cell_magic = noop_decorator 28 | IPython.core.magic.register_line_magic = noop_decorator 29 | IPython.core.magic.register_cell_magic = noop_decorator 30 | IPython.get_ipython = mock.Mock() 31 | 32 | 33 | class TestCases(unittest.TestCase): 34 | 35 | def test_render_table(self): 36 | # TODO(gram): complete this test 37 | pass 38 | 39 | def test_render_text(self): 40 | # TODO(gram): complete this test 41 | pass 42 | -------------------------------------------------------------------------------- /legacy_tests/kernel/module_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import mock 16 | import sys 17 | import unittest 18 | 19 | # import Python so we can mock the parts we need to here. 20 | import IPython 21 | import IPython.core.magic 22 | 23 | import datalab.utils.commands 24 | 25 | 26 | def noop_decorator(func): 27 | return func 28 | 29 | 30 | IPython.core.magic.register_line_cell_magic = noop_decorator 31 | IPython.core.magic.register_line_magic = noop_decorator 32 | IPython.core.magic.register_cell_magic = noop_decorator 33 | IPython.get_ipython = mock.Mock() 34 | 35 | 36 | class TestCases(unittest.TestCase): 37 | 38 | def test_create_python_module(self): 39 | datalab.utils.commands._modules._create_python_module('bar', 'y=1') 40 | self.assertIsNotNone(sys.modules['bar']) 41 | self.assertEqual(1, sys.modules['bar'].y) 42 | 43 | def test_pymodule(self): 44 | datalab.utils.commands._modules.pymodule('--name foo', 'x=1') 45 | self.assertIsNotNone(sys.modules['foo']) 46 | self.assertEqual(1, sys.modules['foo'].x) 47 | 48 | @mock.patch('datalab.utils.commands._modules._pymodule_cell', autospec=True) 49 | def test_pymodule_magic(self, mock_pymodule_cell): 50 | datalab.utils.commands._modules.pymodule('-n foo') 51 | mock_pymodule_cell.assert_called_with({ 52 | 'name': 'foo', 53 | 'func': datalab.utils.commands._modules._pymodule_cell 54 | }, None) 55 | -------------------------------------------------------------------------------- /legacy_tests/stackdriver/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /legacy_tests/stackdriver/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /legacy_tests/stackdriver/monitoring/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /legacy_tests/stackdriver/monitoring/utils_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | import mock 15 | import unittest 16 | 17 | import google.auth 18 | import google.datalab 19 | import google.datalab.stackdriver.monitoring as gcm 20 | 21 | 22 | class TestCases(unittest.TestCase): 23 | 24 | def test_make_client(self): 25 | context = self._create_context() 26 | client = gcm._utils.make_client(context) 27 | 28 | self.assertEqual(client.project, context.project_id) 29 | 30 | @mock.patch('google.datalab.Context.default') 31 | def test_make_client_w_defaults(self, mock_context_default): 32 | default_context = self._create_context() 33 | mock_context_default.return_value = default_context 34 | client = gcm._utils.make_client() 35 | 36 | self.assertEqual(client.project, default_context.project_id) 37 | 38 | @staticmethod 39 | def _create_context(): 40 | creds = mock.Mock(spec=google.auth.credentials.Credentials) 41 | return google.datalab.Context('test_project', creds) 42 | -------------------------------------------------------------------------------- /legacy_tests/storage/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /legacy_tests/storage/bucket_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import mock 16 | import unittest 17 | 18 | import google.auth 19 | import datalab.context 20 | import datalab.storage 21 | import datalab.utils 22 | 23 | 24 | class TestCases(unittest.TestCase): 25 | 26 | @mock.patch('datalab.storage._api.Api.buckets_get') 27 | def test_bucket_existence(self, mock_api_buckets): 28 | mock_api_buckets.return_value = TestCases._create_buckets_get_result() 29 | 30 | buckets = datalab.storage.Buckets(context=TestCases._create_context()) 31 | self.assertTrue(buckets.contains('test_bucket')) 32 | 33 | mock_api_buckets.side_effect = datalab.utils.RequestException(404, 'failed') 34 | self.assertFalse(buckets.contains('test_bucket_2')) 35 | 36 | @mock.patch('datalab.storage._api.Api.buckets_get') 37 | def test_bucket_metadata(self, mock_api_buckets): 38 | mock_api_buckets.return_value = TestCases._create_buckets_get_result() 39 | 40 | b = TestCases._create_bucket() 41 | m = b.metadata 42 | 43 | self.assertEqual(m.name, 'test_bucket') 44 | 45 | @staticmethod 46 | def _create_bucket(name='test_bucket'): 47 | return datalab.storage.Bucket(name, context=TestCases._create_context()) 48 | 49 | @staticmethod 50 | def _create_context(): 51 | project_id = 'test' 52 | creds = mock.Mock(spec=google.auth.credentials.Credentials) 53 | return datalab.context.Context(project_id, creds) 54 | 55 | @staticmethod 56 | def _create_buckets_get_result(): 57 | return {'name': 'test_bucket'} 58 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # Copyright 2017 Google Inc. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Compiles the typescript sources to javascript and submits the files 17 | # to the pypi server specified as first parameter, defaults to testpypi 18 | # In order to run this script locally, make sure you have the following: 19 | # - A Python 3 environment (due to urllib issues) 20 | # - Typescript installed 21 | # - A configured ~/.pypirc containing your pypi/testpypi credentials with 22 | # the server names matching the name you're passing in. Do not include 23 | # the repository URLs in the config file, this has been deprecated. 24 | # - Make sure the package version string in the setup.py file is updated. 25 | # It will get rejected by the server if it already exists 26 | # - If this is a new release, make sure the release notes are updated 27 | # and create a new release tag 28 | 29 | tsc --module amd --noImplicitAny datalab/notebook/static/*.ts 30 | tsc --module amd --noImplicitAny google/datalab/notebook/static/*.ts 31 | 32 | # This is the test url, you should change this to 33 | # https://upload.pypi.org/legacy/ for prod binaries 34 | server="${1:-https://test.pypi.python.org/pypi}" 35 | echo "Submitting package to ${server}" 36 | 37 | # Build and upload a distribution package 38 | rm -rf dist/* 39 | python setup.py sdist 40 | twine upload --repository-url "${server}" dist/* 41 | 42 | # Clean up 43 | rm -f datalab/notebook/static/*.js 44 | rm -f google/datalab/notebook/static/*.js 45 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | 4 | [flake8] 5 | max-line-length = 100 6 | exclude = 7 | docs 8 | ignore = 9 | # Indentation is not a multiple of four 10 | E111, 11 | # Indentation is not a multiple of four (comment) 12 | E114, 13 | # Continuation line under-indented for hanging indent 14 | E121 15 | -------------------------------------------------------------------------------- /solutionbox/image_classification/mltoolbox/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | __import__('pkg_resources').declare_namespace(__name__) 17 | -------------------------------------------------------------------------------- /solutionbox/image_classification/mltoolbox/image/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /solutionbox/image_classification/mltoolbox/image/classification/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | """ 14 | Single label image classification solution. Typical usage is: 15 | 16 | Run preprocess() or preprocess_async() to preprocess data for training. 17 | Run train() or train_async() to train models. 18 | Run predict(), batch_predict(), batch_predict_async() to perform predictions. 19 | 20 | The trained model can also be deployed online with google.datalab.ml.ModelVersions.deploy() call. 21 | """ 22 | 23 | from ._api import preprocess, preprocess_async, train, train_async, predict, batch_predict, \ 24 | batch_predict_async 25 | 26 | __all__ = ['preprocess', 'preprocess_async', 'train', 'train_async', 'predict', 'batch_predict', 27 | 'batch_predict_async'] 28 | -------------------------------------------------------------------------------- /solutionbox/image_classification/mltoolbox/image/classification/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | # To publish to PyPi use: python setup.py bdist_wheel upload -r pypi 14 | 15 | import datetime 16 | from setuptools import setup 17 | 18 | minor = datetime.datetime.now().strftime("%y%m%d%H%M") 19 | version = '0.2' 20 | 21 | setup( 22 | name='mltoolbox_datalab_image_classification', 23 | namespace_packages=['mltoolbox'], 24 | version=version, 25 | packages=[ 26 | 'mltoolbox', 27 | 'mltoolbox.image', 28 | 'mltoolbox.image.classification', 29 | ], 30 | 31 | description='Google Cloud Datalab Inception Package', 32 | author='Google', 33 | author_email='google-cloud-datalab-feedback@googlegroups.com', 34 | keywords=[ 35 | ], 36 | license="Apache Software License", 37 | classifiers=[ 38 | "Programming Language :: Python", 39 | "Programming Language :: Python :: 2", 40 | "Development Status :: 4 - Beta", 41 | "Environment :: Other Environment", 42 | "Intended Audience :: Developers", 43 | "License :: OSI Approved :: Apache Software License", 44 | "Operating System :: OS Independent", 45 | "Topic :: Software Development :: Libraries :: Python Modules" 46 | ], 47 | long_description=""" 48 | """, 49 | install_requires=[ 50 | 'pillow==3.4.1', 51 | ], 52 | package_data={ 53 | } 54 | ) 55 | -------------------------------------------------------------------------------- /solutionbox/image_classification/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | # To publish to PyPi use: python setup.py bdist_wheel upload -r pypi 14 | 15 | import datetime 16 | from setuptools import setup 17 | 18 | minor = datetime.datetime.now().strftime("%y%m%d%H%M") 19 | version = '0.2' 20 | 21 | setup( 22 | name='mltoolbox_datalab_image_classification', 23 | namespace_packages=['mltoolbox'], 24 | version=version, 25 | packages=[ 26 | 'mltoolbox', 27 | 'mltoolbox.image', 28 | 'mltoolbox.image.classification', 29 | ], 30 | 31 | description='Google Cloud Datalab Inception Package', 32 | author='Google', 33 | author_email='google-cloud-datalab-feedback@googlegroups.com', 34 | keywords=[ 35 | ], 36 | license="Apache Software License", 37 | classifiers=[ 38 | "Programming Language :: Python", 39 | "Programming Language :: Python :: 2", 40 | "Development Status :: 4 - Beta", 41 | "Environment :: Other Environment", 42 | "Intended Audience :: Developers", 43 | "License :: OSI Approved :: Apache Software License", 44 | "Operating System :: OS Independent", 45 | "Topic :: Software Development :: Libraries :: Python Modules" 46 | ], 47 | long_description=""" 48 | """, 49 | install_requires=[ 50 | 'pillow==6.2.0', 51 | ], 52 | package_data={ 53 | } 54 | ) 55 | -------------------------------------------------------------------------------- /solutionbox/ml_workbench/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from setuptools import setup 14 | 15 | 16 | setup( 17 | name='mltoolbox_code_free', 18 | namespace_packages=['mltoolbox'], 19 | version='1.0.0', 20 | packages=[ 21 | 'mltoolbox', 22 | 'mltoolbox.code_free_ml', 23 | 'mltoolbox.code_free_ml.trainer', 24 | ], 25 | description='Google Cloud Datalab Structured Data Package', 26 | author='Google', 27 | author_email='google-cloud-datalab-feedback@googlegroups.com', 28 | keywords=[ 29 | ], 30 | license="Apache Software License", 31 | classifiers=[ 32 | "Programming Language :: Python", 33 | "Programming Language :: Python :: 2", 34 | "Development Status :: 4 - Beta", 35 | "Environment :: Other Environment", 36 | "Intended Audience :: Developers", 37 | "License :: OSI Approved :: Apache Software License", 38 | "Operating System :: OS Independent", 39 | "Topic :: Software Development :: Libraries :: Python Modules" 40 | ], 41 | long_description=""" 42 | """, 43 | install_requires=[ # TODO(brandondutra): fill this in. Add pydatalab? 44 | ], 45 | package_data={ 46 | }, 47 | data_files=[], 48 | ) 49 | -------------------------------------------------------------------------------- /solutionbox/ml_workbench/tensorflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googledatalab/pydatalab/8bf007da3e43096aa3a3dca158fc56b286ba6f5c/solutionbox/ml_workbench/tensorflow/__init__.py -------------------------------------------------------------------------------- /solutionbox/ml_workbench/tensorflow/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | # This setup file is used when running cloud training or cloud dataflow jobs. 14 | from setuptools import setup, find_packages 15 | 16 | 17 | setup( 18 | name='trainer', 19 | version='1.0.0', 20 | packages=find_packages(), 21 | description='Google Cloud Datalab helper sub-package', 22 | author='Google', 23 | author_email='google-cloud-datalab-feedback@googlegroups.com', 24 | keywords=[ 25 | ], 26 | license="Apache Software License", 27 | long_description=""" 28 | """, 29 | install_requires=[ 30 | 'tensorflow==1.15.2', 31 | 'protobuf==3.1.0', 32 | 'pillow==6.2.0', # ML Engine does not have PIL installed 33 | ], 34 | package_data={ 35 | }, 36 | data_files=[], 37 | ) 38 | -------------------------------------------------------------------------------- /solutionbox/ml_workbench/tensorflow/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googledatalab/pydatalab/8bf007da3e43096aa3a3dca158fc56b286ba6f5c/solutionbox/ml_workbench/tensorflow/trainer/__init__.py -------------------------------------------------------------------------------- /solutionbox/ml_workbench/test_tensorflow/run_all.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | echo '*** Running tensorflow test_analyze.py ***' 5 | python test_analyze.py --verbose 6 | 7 | echo '*** Running tensorflow test_feature_transforms.py ***' 8 | python test_feature_transforms.py --verbose 9 | 10 | echo '*** Running tensorflow test_transform.py ***' 11 | python test_transform.py --verbose 12 | 13 | echo '*** Running tensorflow test_training.py ***' 14 | python test_training.py --verbose 15 | 16 | echo 'Finished tensorflow run_all.sh!' 17 | -------------------------------------------------------------------------------- /solutionbox/ml_workbench/test_xgboost/run_all.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | echo '*** Running xgboost test_analyze.py ***' 5 | python test_analyze.py --verbose 6 | 7 | echo '*** Running xgboost test_transform.py ***' 8 | python test_transform.py --verbose 9 | 10 | echo 'Finished xgboost run_all.sh!' 11 | -------------------------------------------------------------------------------- /solutionbox/ml_workbench/xgboost/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googledatalab/pydatalab/8bf007da3e43096aa3a3dca158fc56b286ba6f5c/solutionbox/ml_workbench/xgboost/__init__.py -------------------------------------------------------------------------------- /solutionbox/ml_workbench/xgboost/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | # This setup file is used when running cloud training or cloud dataflow jobs. 14 | from setuptools import setup, find_packages 15 | 16 | 17 | setup( 18 | name='trainer', 19 | version='1.0.0', 20 | packages=find_packages(), 21 | description='Google Cloud Datalab helper sub-package', 22 | author='Google', 23 | author_email='google-cloud-datalab-feedback@googlegroups.com', 24 | keywords=[ 25 | ], 26 | license="Apache Software License", 27 | long_description=""" 28 | """, 29 | install_requires=[ 30 | 'tensorflow==1.15.2', 31 | 'protobuf==3.4.0', 32 | 'pillow==6.2.0', # ML Engine does not have PIL installed 33 | 'xgboost==0.6a2', 34 | ], 35 | package_data={ 36 | }, 37 | data_files=[], 38 | ) 39 | -------------------------------------------------------------------------------- /solutionbox/ml_workbench/xgboost/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googledatalab/pydatalab/8bf007da3e43096aa3a3dca158fc56b286ba6f5c/solutionbox/ml_workbench/xgboost/trainer/__init__.py -------------------------------------------------------------------------------- /solutionbox/structured_data/build.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | rm -fr dist 4 | cp setup.py mltoolbox/_structured_data/master_setup.py 5 | python setup.py sdist 6 | 7 | 8 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | __import__('pkg_resources').declare_namespace(__name__) 14 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/_structured_data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from ._package import analyze, analyze_async, train_async, predict, batch_predict, \ 14 | batch_predict_async 15 | 16 | __all__ = ['analyze', 'analyze_async', 'train_async', 'predict', 'batch_predict', 17 | 'batch_predict_async'] 18 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/_structured_data/__version__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | # Source of truth for the version of this package. 14 | __version__ = '1.0.1' 15 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/_structured_data/prediction/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from __future__ import absolute_import 16 | 17 | from . import predict 18 | 19 | __all__ = ['predict'] 20 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/_structured_data/preprocess/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from __future__ import absolute_import 16 | 17 | from . import cloud_preprocess 18 | from . import local_preprocess 19 | 20 | __all__ = ['cloud_preprocess', 'local_preprocess'] 21 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/_structured_data/trainer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from __future__ import absolute_import 16 | 17 | from . import task 18 | 19 | __all__ = ['task'] 20 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/classification/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from mltoolbox._structured_data.__version__ import __version__ 17 | 18 | __all__ = ['__version__'] 19 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/classification/dnn/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """This module contains functions for classification problems modeled as a fully connected 17 | feedforward deep neural network. 18 | 19 | Every function can run locally or use Google Cloud Platform. 20 | """ 21 | 22 | from ._classification_dnn import train, train_async 23 | from mltoolbox._structured_data import analyze, analyze_async, predict, batch_predict, \ 24 | batch_predict_async 25 | from mltoolbox._structured_data.__version__ import __version__ 26 | 27 | __all__ = ['train', 'train_async', 'analyze', 'analyze_async', 'predict', 'batch_predict', 28 | 'batch_predict_async', '__version__'] 29 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/classification/linear/__init__.py: -------------------------------------------------------------------------------- 1 | """This module contains functions for multinomial logistic regression problems. 2 | 3 | Every function can run locally or use Google Cloud Platform. 4 | """ 5 | 6 | from ._classification_linear import train, train_async 7 | from mltoolbox._structured_data import analyze, analyze_async, predict, batch_predict, \ 8 | batch_predict_async 9 | from mltoolbox._structured_data.__version__ import __version__ 10 | 11 | __all__ = ['train', 'train_async', 'analyze', 'analyze_async', 'predict', 'batch_predict', 12 | 'batch_predict_async', '__version__'] 13 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/regression/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from mltoolbox._structured_data.__version__ import __version__ 14 | 15 | __all__ = ['__version__'] 16 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/regression/dnn/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """This module contains functions for regression problems modeled as a fully connected 17 | feedforward deep neural network. 18 | 19 | Every function can run locally or use Google Cloud Platform. 20 | """ 21 | 22 | from ._regression_dnn import train, train_async 23 | from mltoolbox._structured_data import analyze, analyze_async, predict, batch_predict, \ 24 | batch_predict_async 25 | from mltoolbox._structured_data.__version__ import __version__ 26 | 27 | __all__ = ['train', 'train_async', 'analyze', 'analyze_async', 'predict', 'batch_predict', 28 | 'batch_predict_async', '__version__'] 29 | -------------------------------------------------------------------------------- /solutionbox/structured_data/mltoolbox/regression/linear/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """This module contains functions for linear regression problems. 17 | 18 | Every function can run locally or use Google Cloud Platform. 19 | """ 20 | 21 | from ._regression_linear import train, train_async 22 | from mltoolbox._structured_data import analyze, analyze_async, predict, batch_predict, \ 23 | batch_predict_async 24 | from mltoolbox._structured_data.__version__ import __version__ 25 | 26 | __all__ = ['train', 'train_async', 'analyze', 'analyze_async', 'predict', 'batch_predict', 27 | 'batch_predict_async', '__version__'] 28 | -------------------------------------------------------------------------------- /solutionbox/structured_data/test_mltoolbox/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googledatalab/pydatalab/8bf007da3e43096aa3a3dca158fc56b286ba6f5c/solutionbox/structured_data/test_mltoolbox/__init__.py -------------------------------------------------------------------------------- /tests/_util/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /tests/_util/lru_cache_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import unittest 16 | 17 | from google.datalab.utils._lru_cache import LRUCache 18 | 19 | 20 | class TestCases(unittest.TestCase): 21 | 22 | def test_cache_no_entry(self): 23 | cache = LRUCache(3) 24 | with self.assertRaises(KeyError): 25 | cache['a'] 26 | 27 | def test_cache_lookup(self): 28 | cache = LRUCache(4) 29 | for x in ['a', 'b', 'c', 'd']: 30 | cache[x] = x 31 | 32 | for x in ['a', 'b', 'c', 'd']: 33 | self.assertEqual(x, cache[x]) 34 | 35 | def test_cache_overflow(self): 36 | cache = LRUCache(3) 37 | for x in ['a', 'b', 'c', 'd']: 38 | cache[x] = x 39 | 40 | for x in ['b', 'c', 'd']: 41 | self.assertEqual(x, cache[x]) 42 | 43 | with self.assertRaises(KeyError): 44 | cache['a'] 45 | 46 | cache['b'] 47 | cache['d'] 48 | # 'c' should be LRU now 49 | cache['e'] = 'e' 50 | 51 | with self.assertRaises(KeyError): 52 | cache['c'] 53 | 54 | for x in ['b', 'd', 'e']: 55 | self.assertEqual(x, cache[x]) 56 | -------------------------------------------------------------------------------- /tests/bigquery/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /tests/integration/storage_test.py: -------------------------------------------------------------------------------- 1 | """Integration tests for google.datalab.storage.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import logging 8 | import random 9 | import string 10 | import unittest 11 | 12 | import google.datalab 13 | from google.datalab import storage 14 | 15 | 16 | class StorageTest(unittest.TestCase): 17 | 18 | def setUp(self): 19 | self._context = google.datalab.Context.default() 20 | logging.info('Using project: %s', self._context.project_id) 21 | 22 | suffix = ''.join(random.choice(string.lowercase) for _ in range(8)) 23 | self._test_bucket_name = '{}-{}'.format(self._context.project_id, suffix) 24 | logging.info('test bucket: %s', self._test_bucket_name) 25 | 26 | def test_object_deletion_consistency(self): 27 | b = storage.Bucket(self._test_bucket_name, context=self._context) 28 | b.create() 29 | o = b.object('sample') 30 | o.write_stream('contents', 'text/plain') 31 | o.delete() 32 | b.delete() 33 | 34 | 35 | if __name__ == '__main__': 36 | logging.basicConfig(level=logging.INFO) 37 | unittest.main() 38 | -------------------------------------------------------------------------------- /tests/kernel/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | -------------------------------------------------------------------------------- /tests/kernel/chart_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import unittest 16 | 17 | # import Python so we can mock the parts we need to here. 18 | import IPython.core.display 19 | import IPython.core.magic 20 | 21 | 22 | def noop_decorator(func): 23 | return func 24 | 25 | 26 | IPython.core.magic.register_line_cell_magic = noop_decorator 27 | IPython.core.magic.register_line_magic = noop_decorator 28 | IPython.core.magic.register_cell_magic = noop_decorator 29 | IPython.core.display.HTML = lambda x: x 30 | IPython.core.display.JSON = lambda x: x 31 | 32 | 33 | import google.datalab.utils.commands # noqa 34 | 35 | 36 | class TestCases(unittest.TestCase): 37 | 38 | def test_chart_cell(self): 39 | t = [{'country': 'US', 'quantity': 100}, {'country': 'ZA', 'quantity': 50}] 40 | IPython.get_ipython().user_ns = {} 41 | chart = google.datalab.utils.commands._chart._chart_cell({'chart': 'geo', 'data': t, 42 | 'fields': None}, '') 43 | self.assertTrue(chart.find('charts.render(') > 0) 44 | self.assertTrue(chart.find('\'geo\'') > 0) 45 | self.assertTrue(chart.find('"fields": "*"') > 0) 46 | self.assertTrue(chart.find('{"c": [{"v": "US"}, {"v": 100}]}') > 0 or 47 | chart.find('{"c": [{"v": 100}, {"v": "US"}]}') > 0) 48 | self.assertTrue(chart.find('{"c": [{"v": "ZA"}, {"v": 50}]}') > 0 or 49 | chart.find('{"c": [{"v": 50}, {"v": "ZA"}]}') > 0) 50 | 51 | def test_chart_magic(self): 52 | # TODO(gram): complete this test 53 | pass 54 | -------------------------------------------------------------------------------- /tests/kernel/html_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import mock 16 | import unittest 17 | 18 | # import Python so we can mock the parts we need to here. 19 | import IPython 20 | import IPython.core.magic 21 | 22 | import google.datalab 23 | 24 | 25 | def noop_decorator(func): 26 | return func 27 | 28 | 29 | IPython.core.magic.register_line_cell_magic = noop_decorator 30 | IPython.core.magic.register_line_magic = noop_decorator 31 | IPython.core.magic.register_cell_magic = noop_decorator 32 | IPython.get_ipython = mock.Mock() 33 | 34 | 35 | class TestCases(unittest.TestCase): 36 | 37 | def test_render_table(self): 38 | builder = google.datalab.utils.commands.HtmlBuilder() 39 | builder._render_objects({ 40 | 'cols': [ 41 | {'label': 'col1'}, 42 | {'label': 'col2'}, 43 | ], 44 | 'rows': [ 45 | {'c': [ 46 | {'v': 'val1'}, 47 | {'v': 'val2'} 48 | ]}, 49 | {'c': [ 50 | {'v': 'val3'}, 51 | {'v': 'val4'} 52 | ]} 53 | ] 54 | }, ['col1', 'col2'], 'chartdata') 55 | expected_html = ''.join(''' 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 |
col1col2
val1val2
val3val4
70 | '''.split()) 71 | self.assertEqual(builder._to_html(), expected_html) 72 | 73 | def test_render_text(self): 74 | # TODO(gram): complete this test 75 | pass 76 | -------------------------------------------------------------------------------- /tests/ml/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googledatalab/pydatalab/8bf007da3e43096aa3a3dca158fc56b286ba6f5c/tests/ml/__init__.py -------------------------------------------------------------------------------- /tests/ml/facets_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | # in compliance with the License. You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software distributed under the License 9 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | # or implied. See the License for the specific language governing permissions and limitations under 11 | # the License. 12 | 13 | from __future__ import absolute_import 14 | from __future__ import unicode_literals 15 | import unittest 16 | 17 | import pandas as pd 18 | 19 | from google.datalab.ml import FacetsOverview, FacetsDiveview 20 | 21 | 22 | class TestFacets(unittest.TestCase): 23 | """Tests facets visualization components.""" 24 | 25 | def _create_test_data(self): 26 | data1 = [ 27 | {'num1': 1.2, 'weekday': 'Monday', 'occupation': 'software engineer'}, 28 | {'num1': 3.2, 'weekday': 'Tuesday', 'occupation': 'medical doctor'}, 29 | ] 30 | 31 | data2 = [ 32 | {'num1': -2.8, 'weekday': 'Friday', 'occupation': 'musician'}, 33 | ] 34 | 35 | data1 = pd.DataFrame(data1) 36 | data2 = pd.DataFrame(data2) 37 | return data1, data2 38 | 39 | def test_overview_plot(self): 40 | """Tests overview.""" 41 | 42 | data1, data2 = self._create_test_data() 43 | output = FacetsOverview().plot({'data1': data1, 'data2': data2}) 44 | # Output is an html. Ideally we can parse the html and verify nodes, but since the html 45 | # is output by a polymer component which is tested separately, we just verify 46 | # minumum keywords. 47 | self.assertIn("facets-overview", output) 48 | self.assertIn("