├── MANIFEST.in ├── setup.cfg ├── CONTRIBUTING.md ├── .gitignore ├── tests ├── colabtools_loads_test.py ├── simple.ipynb ├── test_api_surface.py ├── test_altair_import_hook.py ├── test_files_handler.py └── serverextension │ └── test_handlers.py ├── .travis.yml ├── tox.ini ├── cloudbuild ├── cloudbuild.yaml └── Dockerfile ├── google ├── colab │ ├── html │ │ ├── _resources.py │ │ ├── __init__.py │ │ ├── js │ │ │ └── _proxy.js │ │ ├── templates │ │ │ └── _element.mustache │ │ ├── _background_server.py │ │ ├── _provide.py │ │ └── _html.py │ ├── _import_hooks │ │ ├── __init__.py │ │ └── _altair.py │ ├── output │ │ ├── __init__.py │ │ ├── _util.py │ │ ├── _publish.py │ │ ├── _area.py │ │ ├── _js.py │ │ ├── _tags.py │ │ └── _js_builder.py │ ├── _autocomplete │ │ ├── __init__.py │ │ ├── _dictionary.py │ │ ├── _splitter.py │ │ └── _inference.py │ ├── errors.py │ ├── widgets │ │ ├── __init__.py │ │ ├── _tabbar.py │ │ ├── _widget.py │ │ └── _grid.py │ ├── _ipython.py │ ├── _files_handler.py │ ├── resources │ │ ├── tabbar.css │ │ └── files.js │ ├── _serverextension │ │ ├── _handlers.py │ │ ├── __init__.py │ │ └── _resource_monitor.py │ ├── __init__.py │ ├── _kernel.py │ ├── _shell.py │ ├── auth.py │ ├── _message.py │ ├── drive.py │ ├── files.py │ ├── _shell_customizations.py │ └── _system_commands.py └── __init__.py ├── README.md ├── setup.py ├── notebooks └── colab-github-demo.ipynb └── LICENSE /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include google * 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | See the README. 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.py[cod] 3 | *.egg-info/ 4 | .cache/ 5 | build/ 6 | dist/ 7 | distribute-* 8 | 9 | # Test files 10 | .tox/ 11 | .pytest_cache/ 12 | 13 | # Coverage related 14 | .coverage 15 | coverage.xml 16 | htmlcov/ 17 | -------------------------------------------------------------------------------- /tests/colabtools_loads_test.py: -------------------------------------------------------------------------------- 1 | """Tests that colabtools loads.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import unittest 8 | 9 | import google.colab 10 | 11 | 12 | class ColabtoolsLoadsTest(unittest.TestCase): 13 | 14 | def testSomethingBasic(self): 15 | self.assertIn('auth', dir(google.colab)) 16 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # The Travis-CI configuration for colabtools is just a stub; tests for 2 | # colabtools are run internally. 3 | 4 | language: python 5 | sudo: false 6 | python: 7 | - "2.7" 8 | - "3.5" 9 | - "3.6" 10 | - "3.7-dev" 11 | 12 | matrix: 13 | allow_failures: 14 | - python: "3.7-dev" 15 | 16 | notifications: 17 | recipients: 18 | - colaboratory-team+travis-ci@google.com 19 | 20 | install: pip install tox-travis 21 | script: tox 22 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # This tox configuration is a stub; tests for colabtools are run internally. 2 | 3 | [tox] 4 | envlist = {py27,py35,py36}-tornado{45,5} 5 | skip_missing_interpreters = True 6 | 7 | [testenv] 8 | deps = 9 | altair~=2.2.2 10 | google-auth~=1.4.0 11 | google-cloud-bigquery~=1.1.0 12 | portpicker~=1.2.0 13 | psutil~=5.4.0 14 | pytest~=3.8.0 15 | py27: mock~=2.0.0 16 | six~=1.11.0 17 | tornado45: tornado>=4.5,<5 18 | tornado5: tornado>=5,<6 19 | commands = py.test [] 20 | 21 | -------------------------------------------------------------------------------- /cloudbuild/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | substitutions: 2 | _PREFIX: '' 3 | steps: 4 | - name: 'gcr.io/cloud-builders/docker' 5 | args: ['build', '-f', 'cloudbuild/Dockerfile', '-t', 'colabtools', '.'] 6 | - name: 'gcr.io/cloud-builders/docker' 7 | args: ['run', '-v', '/workspace:/out', 'colabtools', 'cp', '/colabtools/dist/google_colab-0.0.1a1-py2.py3-none-any.whl', '/out'] 8 | - name: 'gcr.io/cloud-builders/gsutil' 9 | args: ['cp', '/workspace/google_colab-0.0.1a1-py2.py3-none-any.whl', 'gs://colabtools/${COMMIT_SHA}${_PREFIX}/'] 10 | -------------------------------------------------------------------------------- /google/colab/html/_resources.py: -------------------------------------------------------------------------------- 1 | """Fetches resources.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import pkgutil 8 | 9 | _cache = {} 10 | 11 | 12 | def get_data(module, relative_path): 13 | """Gets data using `pkgutil` module should be passed in as __name__.""" 14 | key = module + relative_path 15 | if key in _cache: 16 | return _cache[key] 17 | data = pkgutil.get_data(module, relative_path) 18 | _cache[key] = data 19 | return data 20 | -------------------------------------------------------------------------------- /google/colab/html/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colabs html package.""" 15 | # pylint: disable=g-multiple-import 16 | from google.colab.html import _provide 17 | from google.colab.html._html import Element 18 | 19 | create_resource = _provide.create 20 | -------------------------------------------------------------------------------- /tests/simple.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "Simple", 7 | "version": "0.3.2", 8 | "views": {}, 9 | "default_view": {}, 10 | "provenance": [], 11 | "collapsed_sections": [] 12 | }, 13 | "kernelspec": { 14 | "name": "python3", 15 | "display_name": "Python 3" 16 | } 17 | }, 18 | "cells": [ 19 | { 20 | "metadata": { 21 | "id": "NwK2eKbS4X96", 22 | "colab_type": "code", 23 | "colab": { 24 | "autoexec": { 25 | "startup": false, 26 | "wait_interval": 0 27 | } 28 | } 29 | }, 30 | "cell_type": "code", 31 | "source": [ 32 | "1+1" 33 | ], 34 | "execution_count": 0, 35 | "outputs": [] 36 | } 37 | ] 38 | } -------------------------------------------------------------------------------- /google/colab/_import_hooks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language govestylerning permissions and 13 | # limitations under the License. 14 | """Colab import customizations to the IPython runtime.""" 15 | 16 | from google.colab._import_hooks import _altair 17 | 18 | 19 | def _register_hooks(): 20 | _altair._register_hook() # pylint:disable=protected-access 21 | -------------------------------------------------------------------------------- /google/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Google namespace package.""" 15 | 16 | # pylint:disable=g-import-not-at-top 17 | try: 18 | import pkg_resources 19 | pkg_resources.declare_namespace(__name__) 20 | except ImportError: 21 | import pkgutil 22 | __path__ = pkgutil.extend_path(__path__, __name__) 23 | -------------------------------------------------------------------------------- /google/colab/output/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colabs output package.""" 15 | # pylint: disable=g-multiple-import 16 | from google.colab.output._area import redirect_to_element, to_default_area, to_footer_area, to_header_area 17 | from google.colab.output._js import eval_js, register_callback 18 | from google.colab.output._tags import clear, temporary, use_tags 19 | -------------------------------------------------------------------------------- /google/colab/_autocomplete/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colabs _autocomplete package.""" 15 | from __future__ import absolute_import 16 | 17 | import IPython 18 | 19 | from google.colab._autocomplete import _dictionary 20 | from google.colab._autocomplete import _splitter 21 | 22 | 23 | def enable(): 24 | ip = IPython.get_ipython() 25 | 26 | _dictionary.enable() 27 | _splitter.enable(ip) 28 | -------------------------------------------------------------------------------- /google/colab/errors.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Common error types used across Colab python functions.""" 15 | 16 | from __future__ import absolute_import as _ 17 | from __future__ import division as _ 18 | from __future__ import print_function as _ 19 | 20 | 21 | class Error(Exception): 22 | """Base class for all Colab errors.""" 23 | 24 | 25 | class AuthorizationError(Error): 26 | """Authorization-related failures.""" 27 | 28 | 29 | class WidgetException(Exception): 30 | """colab.widgets failures.""" 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Google Colaboratory 2 | 3 | [Colaboratory](https://colab.research.google.com) is a research project created 4 | to help disseminate machine learning education and research. It’s a Jupyter 5 | notebook environment that requires no setup to use. For more information, see 6 | our [FAQ](https://research.google.com/colaboratory/faq.html). 7 | 8 | This repository contains the code for the Python libraries available in the 9 | Colab. 10 | 11 | ## Contacting Us 12 | 13 | For support or help using Colab, please submit questions tagged with 14 | `google-colaboratory` 15 | on 16 | [StackOverflow](https://stackoverflow.com/questions/tagged/google-colaboratory). 17 | 18 | For any product issues, you can 19 | either [submit an issue](https://github.com/googlecolab/colabtools/issues) or 20 | "Help" -> "Send Feedback" in Colab. 21 | 22 | ## Contributing 23 | 24 | If you have a problem, or see something that could be improved, please file an 25 | issue. However, we don't have the bandwidth to support review of external 26 | contributions, and we don't want user PRs to languish, so we aren't accepting 27 | any external contributions right now. 28 | -------------------------------------------------------------------------------- /google/colab/widgets/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language govestylerning permissions and 13 | # limitations under the License. 14 | """High level widgets for display in the output area. 15 | 16 | Many of these widgets allow to output standard channels (e.g. print statements, 17 | or matplotlib) into specific part of the widget, such as individual tab or 18 | grid's cell. This allows to build complex interactive outputs, using libraries 19 | that are not even aware of the widget's existence. 20 | """ 21 | from google.colab.widgets._grid import create_grid 22 | from google.colab.widgets._grid import Grid 23 | from google.colab.widgets._tabbar import TabBar 24 | -------------------------------------------------------------------------------- /google/colab/_ipython.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """IPython compatibility layer.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import IPython 21 | 22 | 23 | def get_ipython(): 24 | return IPython.get_ipython() 25 | 26 | 27 | def get_kernel(): 28 | return get_ipython().kernel 29 | 30 | 31 | def get_kernelapp(): 32 | return get_ipython().kernel.parent 33 | 34 | 35 | def in_ipython(): 36 | """Return True iff we're in a IPython like environment.""" 37 | ip = IPython.get_ipython() 38 | return hasattr(ip, 'kernel') 39 | -------------------------------------------------------------------------------- /google/colab/_files_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Custom Jupyter FilesHandler.""" 15 | 16 | from notebook.base import handlers 17 | 18 | 19 | class ColabAuthenticatedFileHandler(handlers.AuthenticatedFileHandler): 20 | 21 | def set_extra_headers(self, path): 22 | super(ColabAuthenticatedFileHandler, self).set_extra_headers(path) 23 | # The Content-Length header may be removed by upstream proxies (e.g. using 24 | # Transfer-Encoding=chunked). As such, we explicitly set a header containing 25 | # the size so that clients have the ability to show progress. 26 | size = self.get_content_size() 27 | if size: 28 | self.add_header('X-File-Size', size) 29 | -------------------------------------------------------------------------------- /google/colab/resources/tabbar.css: -------------------------------------------------------------------------------- 1 | .goog-tab{position:relative;padding:4px 8px;color:#00c;text-decoration:underline;cursor:default}.goog-tab-bar-top .goog-tab{margin:1px 4px 0 0;border-bottom:0;float:left}.goog-tab-bar-top:after,.goog-tab-bar-bottom:after{content:" ";display:block;height:0;clear:both;visibility:hidden}.goog-tab-bar-bottom .goog-tab{margin:0 4px 1px 0;border-top:0;float:left}.goog-tab-bar-start .goog-tab{margin:0 0 4px 1px;border-right:0}.goog-tab-bar-end .goog-tab{margin:0 1px 4px 0;border-left:0}.goog-tab-hover{background:#eee}.goog-tab-disabled{color:#666}.goog-tab-selected{color:#000;background:#fff;text-decoration:none;font-weight:bold;border:1px solid #6b90da}.goog-tab-bar-top{padding-top:5px!important;padding-left:5px!important;border-bottom:1px solid #6b90da!important}.goog-tab-bar-top .goog-tab-selected{top:1px;margin-top:0;padding-bottom:5px}.goog-tab-bar-bottom .goog-tab-selected{top:-1px;margin-bottom:0;padding-top:5px}.goog-tab-bar-start .goog-tab-selected{left:1px;margin-left:0;padding-right:9px}.goog-tab-bar-end .goog-tab-selected{left:-1px;margin-right:0;padding-left:9px}.goog-tab-bar{margin:0;border:0;padding:0;list-style:none;cursor:default;outline:none;background:#ebeff9}.goog-tab-bar-clear{clear:both;height:0;overflow:hidden}.goog-tab-bar-start{float:left}.goog-tab-bar-end{float:right}* html .goog-tab-bar-start{margin-right:-3px}* html .goog-tab-bar-end{margin-left:-3px} -------------------------------------------------------------------------------- /google/colab/output/_util.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language govestylerning permissions and 13 | # limitations under the License. 14 | """Private utility functions.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import sys 21 | 22 | _id_counter = 0 23 | 24 | 25 | def flush_all(): 26 | """Flushes stdout/stderr/matplotlib.""" 27 | sys.stdout.flush() 28 | sys.stderr.flush() 29 | # pylint: disable=g-import-not-at-top 30 | try: 31 | from ipykernel.pylab.backend_inline import flush_figures 32 | except ImportError: 33 | # older ipython 34 | from IPython.kernel.zmq.pylab.backend_inline import flush_figures 35 | 36 | flush_figures() 37 | 38 | 39 | def get_locally_unique_id(prefix='id'): 40 | """"Returns id which is unique with the session.""" 41 | global _id_counter 42 | _id_counter += 1 43 | return prefix + str(_id_counter) 44 | -------------------------------------------------------------------------------- /cloudbuild/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM ubuntu:17.10 16 | MAINTAINER Google Colaboratory Team 17 | 18 | # Setup OS and core packages 19 | RUN apt-get update && \ 20 | apt-get install --no-install-recommends -y -q \ 21 | build-essential \ 22 | git \ 23 | python \ 24 | python-dev \ 25 | python-pip \ 26 | python-setuptools \ 27 | python-zmq \ 28 | python3 \ 29 | python3-dev \ 30 | python3-pip \ 31 | python3-setuptools \ 32 | python3-zmq \ 33 | && \ 34 | python3 -m pip install wheel && \ 35 | python2 -m pip install wheel tox 36 | 37 | # Fetch the colabtools source. 38 | ADD . /colabtools 39 | 40 | # Run tests and build a new wheel. 41 | RUN cd /colabtools && \ 42 | tox && \ 43 | python setup.py bdist_wheel 44 | -------------------------------------------------------------------------------- /google/colab/_serverextension/_handlers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Custom Jupyter notebook API handlers.""" 15 | 16 | import json 17 | 18 | from notebook.base import handlers 19 | 20 | import tornado 21 | 22 | from google.colab._serverextension import _resource_monitor 23 | 24 | _XSSI_PREFIX = ")]}'\n" 25 | 26 | 27 | class ResourceUsageHandler(handlers.APIHandler): 28 | """Handles requests for memory usage of Colab kernels.""" 29 | 30 | def initialize(self, kernel_manager): 31 | self._kernel_manager = kernel_manager 32 | 33 | @tornado.web.authenticated 34 | def get(self, *unused_args, **unused_kwargs): 35 | ram = _resource_monitor.get_ram_usage(self._kernel_manager) 36 | gpu = _resource_monitor.get_gpu_usage() 37 | disk = _resource_monitor.get_disk_usage() 38 | self.set_header('Content-Type', 'application/json') 39 | self.finish(_XSSI_PREFIX + json.dumps({ 40 | 'ram': ram, 41 | 'gpu': gpu, 42 | 'disk': disk 43 | })) 44 | -------------------------------------------------------------------------------- /google/colab/html/js/_proxy.js: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License 14 | 15 | /** 16 | * Proxies execution of object across cells. 17 | * @param {string} id 18 | * @param {*} msg 19 | * @param {*} defaultResponse response if id does not exist 20 | * @returns {Promise} 21 | */ 22 | async function proxy(id, msg, defaultResponse) { 23 | let elementProxy = null; 24 | for (let i = 0; i < window.parent.frames.length; ++i) { 25 | const frame = window.parent.frames[i]; 26 | // The frames will include cross-origin frames which will generate errors 27 | // when accessing the contents, so guard against that. 28 | try { 29 | const html = frame.window.google.colab.html; 30 | if (html) { 31 | elementProxy = html.elements[id]; 32 | if (elementProxy) { 33 | break; 34 | } 35 | } 36 | } catch (e) { 37 | // Continue to the next frame. 38 | } 39 | } 40 | if (!elementProxy) { 41 | return defaultResponse; 42 | } 43 | return elementProxy.call(msg); 44 | } 45 | 46 | //# sourceURL=/google/colab/html/js/_proxy.js 47 | -------------------------------------------------------------------------------- /google/colab/_import_hooks/_altair.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Import hook for ensuring that Altair's Colab renderer is registered.""" 15 | 16 | import imp 17 | import logging 18 | import os 19 | import sys 20 | 21 | 22 | class _AltairImportHook(object): 23 | """Enables Altair's Colab renderer upon import.""" 24 | 25 | def find_module(self, fullname, path=None): 26 | if fullname != 'altair': 27 | return None 28 | self.path = path 29 | return self 30 | 31 | def load_module(self, name): 32 | """Loads Altair normally and runs pre-initialization code.""" 33 | previously_loaded = name in sys.modules 34 | 35 | module_info = imp.find_module(name, self.path) 36 | altair_module = imp.load_module(name, *module_info) 37 | 38 | if not previously_loaded: 39 | try: 40 | altair_module.renderers.enable('colab') 41 | except: # pylint: disable=bare-except 42 | logging.exception('Error enabling Altair Colab renderer.') 43 | os.environ['COLAB_ALTAIR_IMPORT_HOOK_EXCEPTION'] = '1' 44 | 45 | return altair_module 46 | 47 | 48 | def _register_hook(): 49 | sys.meta_path = [_AltairImportHook()] + sys.meta_path 50 | -------------------------------------------------------------------------------- /google/colab/_serverextension/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colab-specific Jupyter serverextensions.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | # Allow imports of submodules without Jupyter 21 | try: 22 | # pylint: disable=g-import-not-at-top 23 | from notebook import utils 24 | from notebook.base import handlers 25 | from google.colab._serverextension import _handlers 26 | except ImportError: 27 | pass 28 | 29 | 30 | def _jupyter_server_extension_paths(): 31 | return [{ 32 | 'module': 'google.colab._serverextension', 33 | }] 34 | 35 | 36 | def load_jupyter_server_extension(nb_server_app): 37 | """Called by Jupyter when starting the notebook manager.""" 38 | app = nb_server_app.web_app 39 | 40 | url_maker = lambda path: utils.url_path_join(app.settings['base_url'], path) 41 | monitor_relative_path = '/api/colab/resources' 42 | 43 | app.add_handlers('.*$', [ 44 | (url_maker(monitor_relative_path), _handlers.ResourceUsageHandler, { 45 | 'kernel_manager': app.settings['kernel_manager'] 46 | }), 47 | ]) 48 | nb_server_app.log.info('google.colab serverextension initialized.') 49 | -------------------------------------------------------------------------------- /google/colab/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colab Python APIs.""" 15 | 16 | from __future__ import absolute_import as _ 17 | from __future__ import division as _ 18 | from __future__ import print_function as _ 19 | 20 | from google.colab import _import_hooks 21 | from google.colab import _shell_customizations 22 | from google.colab import _system_commands 23 | from google.colab import auth 24 | from google.colab import drive 25 | from google.colab import files 26 | from google.colab import output 27 | from google.colab import widgets 28 | 29 | __all__ = ['auth', 'drive', 'files', 'output', 'widgets'] 30 | 31 | __version__ = '0.0.1a2' 32 | 33 | 34 | def _jupyter_nbextension_paths(): 35 | # See: 36 | # http://testnb.readthedocs.io/en/latest/examples/Notebook/Distributing%20Jupyter%20Extensions%20as%20Python%20Packages.html#Defining-the-server-extension-and-nbextension 37 | return [{ 38 | 'dest': 'google.colab', 39 | 'section': 'notebook', 40 | 'src': 'resources', 41 | }] 42 | 43 | 44 | def load_ipython_extension(ipython): 45 | """Called by IPython when this module is loaded as an IPython extension.""" 46 | _shell_customizations.initialize() 47 | _system_commands._register_magics(ipython) # pylint:disable=protected-access 48 | _import_hooks._register_hooks() # pylint:disable=protected-access 49 | -------------------------------------------------------------------------------- /tests/test_api_surface.py: -------------------------------------------------------------------------------- 1 | """Test that our definition of the API for modules in google.colab is correct. 2 | 3 | """ 4 | 5 | from __future__ import absolute_import 6 | from __future__ import division 7 | from __future__ import print_function 8 | 9 | import os 10 | import unittest 11 | 12 | import google.colab 13 | 14 | 15 | class ApiTest(unittest.TestCase): 16 | 17 | def testAll(self): 18 | # Check that __all__ is a subset of dir. If it is not, then someone has made 19 | # a typo in defining __all__. 20 | self.assertLess(set(google.colab.__all__), set(dir(google.colab))) 21 | 22 | def testTopLevelSubmodules(self): 23 | # Check that __all__ is a subset of dir for each module in 24 | # google.colab.__all__ 25 | for module_name in google.colab.__all__: 26 | module = google.colab.__dict__[module_name] 27 | # No __all__ only allowed for __init__s. 28 | is_init = '__init__.py' in os.path.basename(module.__file__) 29 | has_all = hasattr(module, '__all__') 30 | if is_init and not has_all: 31 | continue 32 | self.assertTrue(has_all, 'No __all__ in ' + module.__name__) 33 | self.assertLessEqual( 34 | set(module.__all__), set(dir(module)), 35 | '__all__ contains name not in dir(module) for ' + module.__name__) 36 | 37 | def testUnderscorePrefixing(self): 38 | for module_name in google.colab.__all__: 39 | module = google.colab.__dict__[module_name] 40 | # Ignore __init__.py files, which specify their own interface without 41 | # using an __all__. 42 | if '__init__.py' in os.path.basename(module.__file__): 43 | continue 44 | public_names = set([n for n in dir(module) if not n.startswith('_')]) 45 | all_set = set(module.__all__) 46 | self.assertLessEqual( 47 | public_names, all_set, 48 | 'Non-_-prefixed symbol(s) {} not in __all__ in {}'.format( 49 | public_names - all_set, module.__name__)) 50 | -------------------------------------------------------------------------------- /google/colab/output/_publish.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Provides a bunch of shortcuts to display popular types of content. 15 | 16 | This module is nothing by a thin wrapper over IPython.display but it allows 17 | shorter and cleaner code when all we want is to publish common content types. 18 | """ 19 | 20 | from __future__ import absolute_import 21 | from __future__ import division 22 | from __future__ import print_function 23 | 24 | import hashlib 25 | 26 | from IPython import display 27 | 28 | 29 | def html(content): 30 | """Publishes given html content into the output.""" 31 | display.display(display.HTML(content)) 32 | 33 | 34 | def css(content=None, url=None): 35 | """Publishes css content.""" 36 | if url is not None: 37 | html('' % url) 38 | else: 39 | html('') 40 | 41 | 42 | def javascript(content=None, url=None, script_id=None): 43 | """Publishes javascript content into the output.""" 44 | if (content is None) == (url is None): 45 | raise ValueError('exactly one of content and url should be none') 46 | if url is not None: 47 | # Note: display.javascript will try to download script from python 48 | # which is very rarely useful. 49 | html('' % url) 50 | return 51 | if not script_id and 'sourceURL=' not in content: 52 | script_id = 'js_' + hashlib.md5(content.encode('utf8')).hexdigest()[:10] 53 | 54 | if script_id: 55 | content += '\n//# sourceURL=%s' % script_id 56 | display.display(display.Javascript(content)) 57 | -------------------------------------------------------------------------------- /tests/test_altair_import_hook.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for _AltairImportHook.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import importlib 21 | import os 22 | import sys 23 | import unittest 24 | 25 | from six.moves import reload_module 26 | 27 | from google.colab._import_hooks import _altair 28 | 29 | 30 | class AltairImportHookTest(unittest.TestCase): 31 | 32 | @classmethod 33 | def setUpClass(cls): 34 | cls.orig_meta_path = sys.meta_path 35 | cls.orig_env = dict(os.environ) 36 | 37 | def setUp(self): 38 | sys.meta_path = self.orig_meta_path 39 | 40 | os.environ.clear() 41 | os.environ.update(self.orig_env) 42 | 43 | sys.modules.pop('altair', None) 44 | 45 | def testRunsInitCodeOnImportWithFailure(self): 46 | _altair._register_hook() 47 | 48 | altair = importlib.import_module('altair') 49 | 50 | self.assertNotIn('COLAB_ALTAIR_IMPORT_HOOK_EXCEPTION', os.environ) 51 | self.assertIn('altair', sys.modules) 52 | self.assertEqual('colab', altair.renderers.active) 53 | 54 | # Reload of the module should not re-execute code. 55 | # Modify the active renderer and ensure that a reload doesn't reset it to 56 | # colab. 57 | altair.renderers.enable('default') 58 | self.assertEqual('default', altair.renderers.active) 59 | 60 | altair = reload_module(altair) 61 | self.assertNotIn('COLAB_ALTAIR_IMPORT_HOOK_EXCEPTION', os.environ) 62 | self.assertIn('altair', sys.modules) 63 | self.assertEqual('default', altair.renderers.active) 64 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Configuration for the google.colab package.""" 15 | 16 | from setuptools import find_packages 17 | from setuptools import setup 18 | 19 | DEPENDENCIES = ( 20 | 'google-auth~=1.4.0', 21 | 'ipykernel~=4.6.0', 22 | 'ipython~=5.5.0', 23 | 'notebook~=5.2.0', 24 | 'six~=1.11.0', 25 | 'portpicker~=1.2.0', 26 | 'requests~=2.18.0', 27 | 'tornado~=4.5.0', 28 | ) 29 | 30 | setup( 31 | name='google-colab', 32 | version='0.0.1a1', 33 | author='Google Colaboratory team', 34 | author_email='colaboratory-team@google.com', 35 | description='Google Colaboratory tools', 36 | long_description='Colaboratory-specific python libraries.', 37 | url='https://colaboratory.research.google.com/', 38 | packages=find_packages(exclude=('tests*',)), 39 | install_requires=DEPENDENCIES, 40 | namespace_packages=('google',), 41 | license='Apache 2.0', 42 | keywords='google colab ipython jupyter', 43 | classifiers=( 44 | 'Programming Language :: Python :: 2', 45 | 'Programming Language :: Python :: 2.7', 46 | 'Programming Language :: Python :: 3', 47 | 'Programming Language :: Python :: 3.5', 48 | 'Programming Language :: Python :: 3.6', 49 | 'Intended Audience :: Developers', 50 | 'License :: OSI Approved :: Apache Software License', 51 | 'Operating System :: POSIX', 52 | 'Operating System :: Microsoft :: Windows', 53 | 'Operating System :: MacOS :: MacOS X', 54 | 'Operating System :: OS Independent', 55 | 'Topic :: Internet :: WWW/HTTP', 56 | ), 57 | include_package_data=True, 58 | ) 59 | -------------------------------------------------------------------------------- /google/colab/output/_area.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language govestylerning permissions and 13 | # limitations under the License. 14 | """Support for custom output areas in colab.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import contextlib 21 | import six 22 | from google.colab.output import _js_builder 23 | 24 | _jsapi = _js_builder.Js('google.colab') 25 | 26 | 27 | def _set_output_area(selector): 28 | if isinstance(selector, six.string_types): 29 | element = _js_builder.Js('document').querySelector(selector) 30 | else: 31 | element = selector 32 | _jsapi.output.setActiveOutputArea(element) 33 | 34 | 35 | @contextlib.contextmanager 36 | def redirect_to_element(selector): 37 | """Will redirect all output to a given element. 38 | 39 | Args: 40 | selector: either a javascript query selector, or 41 | Js expression. 42 | 43 | Yields: 44 | context where the output is redirected 45 | """ 46 | old_area = _jsapi.output.getActiveOutputArea() 47 | _set_output_area(selector) 48 | try: 49 | yield 50 | finally: 51 | _set_output_area(old_area) 52 | 53 | 54 | @contextlib.contextmanager 55 | def to_header_area(): 56 | """Will redirect output to a header.""" 57 | with redirect_to_element('#output-header') as s: 58 | yield s 59 | 60 | 61 | @contextlib.contextmanager 62 | def to_footer_area(): 63 | """Will redirect output to a footer.""" 64 | with redirect_to_element('#output-footer') as s: 65 | yield s 66 | 67 | 68 | @contextlib.contextmanager 69 | def to_default_area(): 70 | """Restores output to output into default area.""" 71 | with redirect_to_element(_jsapi.output.getDefaultOutputArea()) as s: 72 | yield s 73 | -------------------------------------------------------------------------------- /google/colab/_kernel.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colab-specific kernel customizations.""" 15 | 16 | from ipykernel import ipkernel 17 | from ipykernel.jsonutil import json_clean 18 | from IPython.utils.tokenutil import token_at_cursor 19 | from google.colab import _autocomplete 20 | from google.colab import _shell 21 | from google.colab import _shell_customizations 22 | 23 | 24 | class Kernel(ipkernel.IPythonKernel): 25 | """Kernel with additional Colab-specific features.""" 26 | 27 | def __init__(self, *args, **kwargs): 28 | super(Kernel, self).__init__(*args, **kwargs) 29 | 30 | _autocomplete.enable() 31 | 32 | def _shell_class_default(self): 33 | return _shell.Shell 34 | 35 | def do_inspect(self, code, cursor_pos, detail_level=0): 36 | name = token_at_cursor(code, cursor_pos) 37 | info = self.shell.object_inspect(name) 38 | 39 | data = {} 40 | if info['found']: 41 | info_text = self.shell.object_inspect_text( 42 | name, 43 | detail_level=detail_level, 44 | ) 45 | data['text/plain'] = info_text 46 | # Provide the structured inspection information to allow the frontend to 47 | # format as desired. 48 | data['application/json'] = info 49 | 50 | reply_content = { 51 | 'status': 'ok', 52 | 'data': data, 53 | 'metadata': {}, 54 | 'found': info['found'], 55 | } 56 | 57 | return reply_content 58 | 59 | def complete_request(self, stream, ident, parent): 60 | """Colab-specific complete_request handler. 61 | 62 | Overrides the default to allow providing additional metadata in the 63 | response. 64 | 65 | Args: 66 | stream: Shell stream to send the reply on. 67 | ident: Identity of the requester. 68 | parent: Parent request message. 69 | """ 70 | 71 | content = parent['content'] 72 | code = content['code'] 73 | cursor_pos = content['cursor_pos'] 74 | 75 | matches = self.do_complete(code, cursor_pos) 76 | if parent.get('metadata', {}).get('colab_options', 77 | {}).get('include_colab_metadata'): 78 | matches['metadata'] = { 79 | 'colab_types_experimental': 80 | _shell_customizations.compute_completion_metadata( 81 | self.shell, matches['matches']), 82 | } 83 | matches = json_clean(matches) 84 | 85 | self.session.send(stream, 'complete_reply', matches, parent, ident) 86 | -------------------------------------------------------------------------------- /google/colab/_shell.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colab-specific shell customizations.""" 15 | 16 | import os 17 | import sys 18 | 19 | from ipykernel import jsonutil 20 | from ipykernel import zmqshell 21 | from IPython.core import interactiveshell 22 | from ipython_genutils import py3compat 23 | 24 | from google.colab import _shell_customizations 25 | from google.colab import _system_commands 26 | 27 | 28 | class Shell(zmqshell.ZMQInteractiveShell): 29 | """Shell with additional Colab-specific features.""" 30 | 31 | def _should_use_native_system_methods(self): 32 | return os.getenv('USE_NATIVE_IPYTHON_SYSTEM_COMMANDS', False) 33 | 34 | def getoutput(self, *args, **kwargs): 35 | if self._should_use_native_system_methods(): 36 | return super(Shell, self).getoutput(*args, **kwargs) 37 | 38 | return _system_commands._getoutput_compat(self, *args, **kwargs) # pylint:disable=protected-access 39 | 40 | def system(self, *args, **kwargs): 41 | if self._should_use_native_system_methods(): 42 | return super(Shell, self).system(*args, **kwargs) 43 | 44 | return _system_commands._system_compat(self, *args, **kwargs) # pylint:disable=protected-access 45 | 46 | def _send_error(self, exc_content): 47 | topic = (self.displayhook.topic.replace(b'execute_result', b'err') if 48 | self.displayhook.topic else None) 49 | self.displayhook.session.send( 50 | self.displayhook.pub_socket, 51 | u'error', 52 | jsonutil.json_clean(exc_content), 53 | self.displayhook.parent_header, 54 | ident=topic) 55 | 56 | def _showtraceback(self, etype, evalue, stb): 57 | # This override is largely the same as the base implementation with special 58 | # handling to provide error_details in the response if a ColabErrorDetails 59 | # item was passed along. 60 | sys.stdout.flush() 61 | sys.stderr.flush() 62 | 63 | error_details = None 64 | if isinstance(stb, _shell_customizations.ColabTraceback): 65 | colab_tb = stb 66 | error_details = colab_tb.error_details 67 | stb = colab_tb.stb 68 | 69 | exc_content = { 70 | 'traceback': stb, 71 | 'ename': py3compat.unicode_type(etype.__name__), 72 | 'evalue': py3compat.safe_unicode(evalue), 73 | } 74 | 75 | if error_details: 76 | exc_content['error_details'] = error_details 77 | self._send_error(exc_content) 78 | self._last_traceback = stb 79 | 80 | 81 | interactiveshell.InteractiveShellABC.register(Shell) 82 | -------------------------------------------------------------------------------- /google/colab/_serverextension/_resource_monitor.py: -------------------------------------------------------------------------------- 1 | """Methods for tracking resource consumption of Colab kernels.""" 2 | import csv 3 | import os 4 | import subprocess 5 | from distutils import spawn 6 | 7 | try: 8 | # pylint: disable=g-import-not-at-top 9 | import psutil 10 | except ImportError: 11 | psutil = None 12 | 13 | 14 | def get_gpu_usage(): 15 | """Reports total and per-kernel GPU memory usage. 16 | 17 | Returns: 18 | A dict of the form { 19 | usage: int, 20 | limit: int, 21 | kernels: A dict mapping kernel UUIDs to ints (memory usage in bytes), 22 | } 23 | """ 24 | gpu_memory_path = '/var/colab/gpu-memory' 25 | kernels = {} 26 | usage = 0 27 | limit = 0 28 | if os.path.exists(gpu_memory_path): 29 | with open(gpu_memory_path) as f: 30 | reader = csv.DictReader(f.readlines(), delimiter=' ') 31 | for row in reader: 32 | kernels[row['kernel_id']] = int(row['gpu_mem(MiB)']) * 1024 * 1024 33 | if spawn.find_executable('nvidia-smi') is not None: 34 | ns = subprocess.check_output([ 35 | '/usr/bin/timeout', '-sKILL', '1s', 'nvidia-smi', 36 | '--query-gpu=memory.used,memory.total', '--format=csv,nounits,noheader' 37 | ]).decode('utf-8') 38 | r = csv.reader([ns]) 39 | row = next(r) 40 | usage = int(row[0]) * 1024 * 1024 41 | limit = int(row[1]) * 1024 * 1024 42 | return {'usage': usage, 'limit': limit, 'kernels': kernels} 43 | 44 | 45 | def get_ram_usage(kernel_manager): 46 | """Reports total and per-kernel RAM usage. 47 | 48 | Arguments: 49 | kernel_manager: an IPython MultiKernelManager that owns child kernel 50 | processes 51 | 52 | Returns: 53 | A dict of the form { 54 | usage: int, 55 | limit: int, 56 | kernels: A dict mapping kernel UUIDs to ints (memory usage in bytes), 57 | } 58 | """ 59 | free, limit = 0, 0 60 | with open('/proc/meminfo', 'r') as f: 61 | lines = f.readlines() 62 | line = [x for x in lines if 'MemAvailable:' in x] 63 | if line: 64 | free = int(line[0].split()[1]) * 1024 65 | line = [x for x in lines if 'MemTotal:' in x] 66 | if line: 67 | limit = int(line[0].split()[1]) * 1024 68 | usage = limit - free 69 | pids_to_kernel_ids = dict([(str( 70 | kernel_manager.get_kernel(kernel_id).kernel.pid), kernel_id) 71 | for kernel_id in kernel_manager.list_kernel_ids()]) 72 | kernels = {} 73 | ps = subprocess.check_output([ 74 | 'ps', '-q', ','.join(pids_to_kernel_ids.keys()), '-wwo', 'pid rss', 75 | '--no-header' 76 | ]).decode('utf-8') 77 | for proc in ps.split('\n')[:-1]: 78 | proc = proc.strip().split(' ', 1) 79 | if len(proc) != 2: 80 | continue 81 | kernels[pids_to_kernel_ids[proc[0]]] = int(proc[1]) * 1024 82 | return {'usage': usage, 'limit': limit, 'kernels': kernels} 83 | 84 | 85 | def get_disk_usage(): 86 | """Reports total disk usage. 87 | 88 | Returns: 89 | A dict of the form { 90 | usage: int, 91 | limit: int, 92 | } 93 | """ 94 | usage = 0 95 | limit = 0 96 | if psutil is not None: 97 | disk_usage = psutil.disk_usage('/') 98 | usage = disk_usage.used 99 | limit = disk_usage.total 100 | return {'usage': usage, 'limit': limit} 101 | -------------------------------------------------------------------------------- /google/colab/output/_js.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colab helpers for interacting with JavaScript in outputframes.""" 15 | import json 16 | 17 | from google.colab import _ipython 18 | from google.colab import _message 19 | 20 | _json_decoder = json.JSONDecoder() 21 | 22 | 23 | def eval_js(script, ignore_result=False): 24 | """Evaluates the Javascript within the context of the current cell. 25 | 26 | Args: 27 | script: The javascript string to be evaluated 28 | ignore_result: If true, will return immediately 29 | and result from javascript side will be ignored. 30 | 31 | Returns: 32 | Result of the Javascript evaluation or None if ignore_result. 33 | """ 34 | args = ['cell_javascript_eval', {'script': script}] 35 | kernel = _ipython.get_kernel() 36 | request_id = _message.send_request(*args, parent=kernel.shell.parent_header) 37 | if ignore_result: 38 | return 39 | return _message.read_reply_from_input(request_id) 40 | 41 | 42 | _functions = {} 43 | 44 | 45 | def register_callback(function_name, callback): 46 | """Registers a function as a target invokable by Javacript in outputs. 47 | 48 | This exposes the Python function as a target which may be invoked by 49 | Javascript executing in Colab output frames. 50 | 51 | This callback can be called from javascript side using: 52 | colab.kernel.invokeFunction(function_name, [1, 2, 3], {'hi':'bye'}) 53 | then it will invoke callback(1, 2, 3, hi="bye") 54 | 55 | Args: 56 | function_name: string 57 | callback: function that possibly takes positional and keyword arguments 58 | that will be passed via invokeFunction() 59 | """ 60 | _functions[function_name] = callback 61 | 62 | 63 | def _invoke_function(function_name, json_args, json_kwargs): 64 | """Invokes callback with given function_name. 65 | 66 | This function is meant to be used by frontend when proxying 67 | data from secure iframe into kernel. For example: 68 | 69 | _invoke_function(fn_name, "'''" + JSON.stringify(data) + "'''") 70 | 71 | Note the triple quotes: valid JSON cannot contain triple quotes, 72 | so this is a valid literal. 73 | 74 | Args: 75 | function_name: string 76 | json_args: string containing valid json, provided by user. 77 | json_kwargs: string containing valid json, provided by user. 78 | 79 | Returns: 80 | The value returned by the callback. 81 | 82 | Raises: 83 | ValueError: if the registered function cannot be found. 84 | """ 85 | args = _json_decoder.decode(json_args) 86 | kwargs = _json_decoder.decode(json_kwargs) 87 | 88 | callback = _functions.get(function_name, None) 89 | if not callback: 90 | raise ValueError('Function not found: {function_name}'.format( 91 | function_name=function_name)) 92 | 93 | return callback(*args, **kwargs) 94 | -------------------------------------------------------------------------------- /tests/test_files_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for the google.colab._files_handler package.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import logging 21 | import os 22 | import shutil 23 | import tempfile 24 | 25 | from six.moves import http_client 26 | 27 | from tornado import testing 28 | from tornado import web 29 | 30 | from google.colab import _files_handler 31 | 32 | 33 | class FakeNotebookServer(object): 34 | 35 | def __init__(self, app): 36 | self.web_app = app 37 | self.log = logging.getLogger('fake_notebook_server_logger') 38 | 39 | 40 | class ColabAuthenticatedFileHandler(testing.AsyncHTTPTestCase): 41 | """Tests for ColabAuthenticatedFileHandler.""" 42 | 43 | def get_app(self): 44 | """Setup code required by testing.AsyncHTTP[S]TestCase.""" 45 | self.temp_dir = tempfile.mkdtemp() 46 | settings = { 47 | 'base_url': '/', 48 | # The underlying ipaddress library sometimes doesn't think that 49 | # 127.0.0.1 is a proper loopback device. 50 | 'local_hostnames': ['127.0.0.1'], 51 | } 52 | app = web.Application([], **settings) 53 | app.add_handlers('.*$', [ 54 | ('/files/(.*)', _files_handler.ColabAuthenticatedFileHandler, { 55 | 'path': self.temp_dir + '/' 56 | }), 57 | ]) 58 | 59 | return app 60 | 61 | def tearDown(self): 62 | super(ColabAuthenticatedFileHandler, self).tearDown() 63 | shutil.rmtree(self.temp_dir) 64 | 65 | def testNonExistentFile(self): 66 | response = self.fetch('/files/in/some/dir/foo.py') 67 | self.assertEqual(http_client.NOT_FOUND, response.code) 68 | 69 | def testExistingDirectory(self): 70 | os.makedirs(os.path.join(self.temp_dir, 'some/existing/dir')) 71 | response = self.fetch('/files/some/existing/dir') 72 | self.assertEqual(http_client.FORBIDDEN, response.code) 73 | 74 | def testExistingFile(self): 75 | file_dir = os.path.join(self.temp_dir, 'some/existing/dir') 76 | os.makedirs(file_dir) 77 | with open(os.path.join(file_dir, 'foo.txt'), 'wb') as f: 78 | f.write(b'Some content') 79 | 80 | response = self.fetch('/files/some/existing/dir/foo.txt') 81 | self.assertEqual(http_client.OK, response.code) 82 | # Body is the raw file contents. 83 | self.assertEqual(b'Some content', response.body) 84 | self.assertEqual(len(b'Some content'), int(response.headers['X-File-Size'])) 85 | self.assertIn('text/plain', response.headers['Content-Type']) 86 | 87 | def testDoesNotAllowRequestsOutsideOfRootDir(self): 88 | # Based on existing tests: 89 | # https://github.com/jupyter/notebook/blob/f5fa0c180e92d35b4cbfa1cc20b41e9d1d9dfabe/notebook/services/contents/tests/test_manager.py#L173 90 | with open(os.path.join(self.temp_dir, '..', 'foo'), 'w') as f: 91 | f.write('foo') 92 | with open(os.path.join(self.temp_dir, '..', 'bar'), 'w') as f: 93 | f.write('bar') 94 | 95 | response = self.fetch('/files/../foo') 96 | self.assertEqual(http_client.FORBIDDEN, response.code) 97 | response = self.fetch('/files/foo/../../../bar') 98 | self.assertEqual(http_client.FORBIDDEN, response.code) 99 | response = self.fetch('/files/foo/../../bar') 100 | self.assertEqual(http_client.FORBIDDEN, response.code) 101 | -------------------------------------------------------------------------------- /google/colab/html/templates/_element.mustache: -------------------------------------------------------------------------------- 1 | {{#src}} 2 | {{#script}} 3 | 4 | {{/script}} 5 | {{#module}} 6 | 9 | {{/module}} 10 | {{#html}} 11 | 12 | {{/html}} 13 | {{/src}} 14 | <{{tag}} id="{{guid}}"> 15 | {{#children}} 16 | {{{.}}} 17 | {{/children}} 18 | 19 | 146 | -------------------------------------------------------------------------------- /google/colab/_autocomplete/_dictionary.py: -------------------------------------------------------------------------------- 1 | """Provides autocomplete for dict-like structures (e.g. Dataframe).""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import ast 8 | 9 | import IPython 10 | 11 | import pandas as pd 12 | import six 13 | 14 | from google.colab._autocomplete import _inference 15 | from google.colab._autocomplete import _splitter 16 | 17 | 18 | def _is_of_literal_type(x): 19 | try: 20 | ast.literal_eval(repr(x)) 21 | return True 22 | except: # pylint: disable=bare-except 23 | return False 24 | 25 | 26 | _UNICODE_ESCAPE_MAP = { 27 | ord(u'"'): u'\\\"', 28 | ord(u'\''): u'\\\'', 29 | ord(u'\\'): u'\\\\', 30 | } 31 | for i in range(32): 32 | _UNICODE_ESCAPE_MAP[i] = six.text_type(chr(i).encode('unicode-escape')) 33 | 34 | 35 | def _escape_quote(x, quote): 36 | return x.replace(quote, '\\' + quote) 37 | 38 | 39 | def _unicode_escape_special(x): 40 | fmt = u'%s' 41 | if isinstance(x, six.binary_type): 42 | x = x.decode('utf8') 43 | if not isinstance(x, six.text_type): 44 | raise ValueError('Only str or unicode are supported, got %r ' % (x,)) 45 | return fmt % (x.translate(_UNICODE_ESCAPE_MAP)) 46 | 47 | 48 | def _unicode_repr(x): 49 | if _is_str_like(x): 50 | return u"'%s'" % _unicode_escape_special(x) 51 | else: 52 | # Have utf-8 character will use u"..." for autocompletion 53 | return u"u'%s'" % _unicode_escape_special(x) 54 | 55 | 56 | def _unicode(x): 57 | """Returns unicode representation of x.""" 58 | if isinstance(x, six.string_types): 59 | return _unicode_repr(x) 60 | return u'%r' % (x,) 61 | 62 | 63 | def _is_str_like(x): 64 | if isinstance(x, six.binary_type): 65 | return True 66 | if isinstance(x, six.text_type) and len(x.encode('utf8')) == len(x): 67 | return True 68 | return False 69 | 70 | 71 | def _item_autocomplete(shell, event): 72 | """Returns autocomplete options for a dictionary if it is a dictionary. 73 | 74 | Args: 75 | shell: unused 76 | event: contains ipython autocomplete event, most importantly it must 77 | expose text_until_cursor field 78 | Returns: 79 | List of possible completions. 80 | """ 81 | txt = event.text_until_cursor.rstrip() 82 | lines = txt.rsplit('\n', 1) 83 | if not lines: 84 | return 85 | # Fast return if the last line doesn't contain '[' or quotes 86 | if all(s not in lines[-1] for s in '\'"['): 87 | return 88 | 89 | token = '' 90 | if not txt.endswith(('[', "['", '["')): 91 | # See if removing the last token will bring us to the beginning 92 | # of open bracket. 93 | token = _splitter.split(txt) 94 | if not token: 95 | return 96 | txt = txt[:-len(token)] 97 | # If we have something[foo - we can't provide autocompletion 98 | # we can only do something["foo, or something['foo 99 | if not txt.endswith(('\'', '"')): 100 | return 101 | 102 | selector = None 103 | if txt.endswith('['): 104 | # For [ we need to provide ['foo'] autocompletion that includes both 105 | # the bracket and full repr of the key. For autocompletions when 106 | # user already typed quote, we only need to provide foo']. 107 | 108 | # This is caused 109 | # by the fact pecularities of ipython processing which requires the 110 | # autocompletion to match the token it expects (which happens to be empty 111 | # for ' and being "[" for bracket. 112 | 113 | # Represent this actual unicode string. 114 | fmt = lambda x: '[' + _unicode(x) 115 | txt = txt[:-1] 116 | selector = _is_of_literal_type 117 | elif txt.endswith(('["', "['")): 118 | fmt = _unicode_escape_special 119 | txt = txt[:-2] 120 | # only give completions on string keys, (note: ignore unicode as well) 121 | selector = _is_str_like 122 | 123 | if not selector: 124 | return 125 | 126 | symbol = _splitter.split(txt) 127 | v = _inference.infer_expression_result(symbol, shell.user_global_ns, 128 | shell.user_ns) 129 | 130 | # we only support keys of length at most 256 and return at most 100 of them 131 | if isinstance(v, (dict, pd.DataFrame)): 132 | return [(fmt(k)) 133 | for k in v.keys() 134 | if selector(k) and len(fmt(k)) < 256 and fmt(k).startswith(token) 135 | ][0:100] 136 | return 137 | 138 | 139 | def enable(): 140 | IPython.get_ipython().set_hook( 141 | 'complete_command', _item_autocomplete, re_key='.*') 142 | 143 | 144 | # Note: IPython does not provide ability to cleanly remove hooks, so we 145 | # don't provide one here. 146 | -------------------------------------------------------------------------------- /google/colab/auth.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colab-specific authentication helpers.""" 15 | 16 | # TODO(b/113878301): Test that imported modules do not appear in autocomplete. 17 | from __future__ import absolute_import as _ 18 | from __future__ import division as _ 19 | from __future__ import print_function as _ 20 | 21 | import contextlib as _contextlib 22 | import getpass as _getpass 23 | import logging as _logging 24 | import os as _os 25 | import sqlite3 as _sqlite3 # pylint: disable=g-bad-import-order 26 | import subprocess as _subprocess 27 | import tempfile as _tempfile 28 | import time as _time 29 | 30 | import google.auth as _google_auth 31 | import google.auth.transport.requests as _auth_requests 32 | from google.colab import errors as _errors 33 | from google.colab import output as _output 34 | 35 | __all__ = ['authenticate_user'] 36 | 37 | 38 | def _check_adc(): 39 | """Return whether the application default credential exists and is valid.""" 40 | try: 41 | creds, _ = _google_auth.default() 42 | except _google_auth.exceptions.DefaultCredentialsError: 43 | return False 44 | transport = _auth_requests.Request() 45 | try: 46 | creds.refresh(transport) 47 | except Exception as e: # pylint:disable=broad-except 48 | _logging.info('Failure refreshing credentials: %s', e) 49 | return creds.valid 50 | 51 | 52 | def _gcloud_login(): 53 | """Call `gcloud auth login` with custom input handling.""" 54 | # We want to run gcloud and provide user input on stdin; in order to do this, 55 | # we explicitly buffer the gcloud output and print it ourselves. 56 | gcloud_command = [ 57 | 'gcloud', 58 | 'auth', 59 | 'login', 60 | '--enable-gdrive-access', 61 | '--no-launch-browser', 62 | '--quiet', 63 | ] 64 | f, name = _tempfile.mkstemp() 65 | gcloud_process = _subprocess.Popen( 66 | gcloud_command, 67 | stdin=_subprocess.PIPE, 68 | stdout=f, 69 | stderr=_subprocess.STDOUT, 70 | universal_newlines=True) 71 | try: 72 | while True: 73 | _time.sleep(0.2) 74 | _os.fsync(f) 75 | prompt = open(name).read() 76 | if 'https' in prompt: 77 | break 78 | 79 | # Combine the URL with the verification prompt to work around 80 | # https://github.com/jupyter/notebook/issues/3159 81 | prompt = prompt.rstrip() 82 | code = _getpass.getpass(prompt + '\n\nEnter verification code: ') 83 | gcloud_process.communicate(code.strip()) 84 | finally: 85 | _os.close(f) 86 | _os.remove(name) 87 | if gcloud_process.returncode: 88 | raise _errors.AuthorizationError('Error fetching credentials') 89 | 90 | 91 | def _get_adc_path(): 92 | return _os.path.join(_os.environ.get('DATALAB_ROOT', '/'), 'content/adc.json') 93 | 94 | 95 | def _install_adc(): 96 | """Install the gcloud token as the Application Default Credential.""" 97 | gcloud_db_path = _os.path.join( 98 | _os.environ.get('DATALAB_ROOT', '/'), 'content/.config/credentials.db') 99 | db = _sqlite3.connect(gcloud_db_path) 100 | c = db.cursor() 101 | ls = list(c.execute('SELECT * FROM credentials;')) 102 | adc_path = _get_adc_path() 103 | with open(adc_path, 'w') as f: 104 | f.write(ls[0][1]) 105 | 106 | 107 | @_contextlib.contextmanager 108 | def _noop(): 109 | """Null context manager, like contextlib.nullcontext in python 3.7+.""" 110 | yield 111 | 112 | 113 | # pylint:disable=line-too-long 114 | def authenticate_user(clear_output=True): 115 | """Ensures that the given user is authenticated. 116 | 117 | Currently, this ensures that the Application Default Credentials 118 | (https://developers.google.com/identity/protocols/application-default-credentials) 119 | are available and valid. 120 | 121 | Args: 122 | clear_output: (optional, default: True) If True, clear any output related to 123 | the authorization process if it completes successfully. Any errors will 124 | remain (for debugging purposes). 125 | 126 | Returns: 127 | None. 128 | 129 | Raises: 130 | errors.AuthorizationError: If authorization fails. 131 | """ 132 | if _check_adc(): 133 | return 134 | _os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = _get_adc_path() 135 | if not _check_adc(): 136 | context_manager = _output.temporary if clear_output else _noop 137 | with context_manager(): 138 | _gcloud_login() 139 | _install_adc() 140 | if _check_adc(): 141 | return 142 | raise _errors.AuthorizationError('Failed to fetch user credentials') 143 | -------------------------------------------------------------------------------- /google/colab/html/_background_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """WSGI server utilities to run in thread. WSGI chosen for easier interop.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import socket 21 | import threading 22 | import wsgiref.simple_server 23 | import portpicker 24 | 25 | 26 | def _build_server(started, stopped, stopping, timeout): 27 | """Closure to build the server function to be passed to the thread. 28 | 29 | Args: 30 | started: Threading event to notify when started. 31 | stopped: Threading event to notify when stopped. 32 | stopping: Threading event to notify when stopping. 33 | timeout: Http timeout in seconds. 34 | Returns: 35 | A function that function that takes a port and WSGI app and notifies 36 | about its status via the threading events provided. 37 | """ 38 | 39 | def server(port, wsgi_app): 40 | """Serve a WSGI application until stopped. 41 | 42 | Args: 43 | port: Port number to serve on. 44 | wsgi_app: WSGI application to serve. 45 | """ 46 | host = '' # Bind to all. 47 | try: 48 | httpd = wsgiref.simple_server.make_server( 49 | host, port, wsgi_app, handler_class=SilentWSGIRequestHandler) 50 | except socket.error: 51 | # Try IPv6 52 | httpd = wsgiref.simple_server.make_server( 53 | host, 54 | port, 55 | wsgi_app, 56 | server_class=_WSGIServerIPv6, 57 | handler_class=SilentWSGIRequestHandler) 58 | started.set() 59 | httpd.timeout = timeout 60 | while not stopping.is_set(): 61 | httpd.handle_request() 62 | stopped.set() 63 | 64 | return server 65 | 66 | 67 | class SilentWSGIRequestHandler(wsgiref.simple_server.WSGIRequestHandler): 68 | """WSGIRequestHandler that generates no logging output.""" 69 | 70 | def log_message(self, format, *args): # pylint: disable=redefined-builtin 71 | pass 72 | 73 | 74 | class _WSGIServerIPv6(wsgiref.simple_server.WSGIServer): 75 | """IPv6 based extension of the simple WSGIServer.""" 76 | 77 | address_family = socket.AF_INET6 78 | 79 | 80 | class _WsgiServer(object): 81 | """Wsgi server.""" 82 | 83 | def __init__(self, wsgi_app): 84 | """Initialize the WsgiServer. 85 | 86 | Args: 87 | wsgi_app: WSGI pep-333 application to run. 88 | """ 89 | self._app = wsgi_app 90 | self._server_thread = None 91 | # Threading.Event objects used to communicate about the status 92 | # of the server running in the background thread. 93 | # These will be initialized after building the server. 94 | self._stopped = None 95 | self._stopping = None 96 | 97 | @property 98 | def wsgi_app(self): 99 | """Returns the wsgi app instance.""" 100 | return self._app 101 | 102 | @property 103 | def port(self): 104 | """Returns the current port or error if the server is not started. 105 | 106 | Raises: 107 | RuntimeError: If server has not been started yet. 108 | Returns: 109 | The port being used by the server. 110 | """ 111 | if self._server_thread is None: 112 | raise RuntimeError('Server not running.') 113 | return self._port 114 | 115 | def stop(self): 116 | """Stops the server thread.""" 117 | if self._server_thread is None: 118 | return 119 | self._stopping.set() 120 | self._server_thread = None 121 | self._stopped.wait() 122 | 123 | def start(self, port=None, timeout=1): 124 | """Starts a server in a thread using the WSGI application provided. 125 | 126 | Will wait until the thread has started calling with an already serving 127 | application will simple return. 128 | 129 | Args: 130 | port: Number of the port to use for the application, will find an open 131 | port if one is not provided. 132 | timeout: Http timeout in seconds. 133 | """ 134 | if self._server_thread is not None: 135 | return 136 | started = threading.Event() 137 | self._stopped = threading.Event() 138 | self._stopping = threading.Event() 139 | 140 | wsgi_app = self.wsgi_app 141 | server = _build_server(started, self._stopped, self._stopping, timeout) 142 | if port is None: 143 | self._port = portpicker.pick_unused_port() 144 | else: 145 | self._port = port 146 | server_thread = threading.Thread(target=server, args=(self._port, wsgi_app)) 147 | self._server_thread = server_thread 148 | 149 | server_thread.start() 150 | started.wait() 151 | -------------------------------------------------------------------------------- /google/colab/widgets/_tabbar.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Provides a tabbar widget that redirects output into separate tabs.""" 15 | import contextlib 16 | import six 17 | 18 | from google.colab.output import _publish 19 | from google.colab.output import _js_builder as js 20 | from google.colab.widgets import _widget 21 | 22 | 23 | class TabBar(_widget.OutputAreaWidget): 24 | """UI Widget to generate tab bars in the output. 25 | 26 | Sample usage: 27 | tab = TabBar(['evening', 'morning']) 28 | with tab.output_to('evening'): 29 | print 'hi' 30 | 31 | with tab.output_to('morning'): 32 | print 'bye' 33 | """ 34 | TABBAR_JS = '/nbextensions/google.colab/tabbar_main.min.js' 35 | TAB_CSS = '/nbextensions/google.colab/tabbar.css' 36 | 37 | def __init__(self, tab_names, location='top'): 38 | """Constructor. 39 | 40 | Args: 41 | tab_names: list of strings for the tab names. 42 | 43 | location: location of tabs. 44 | Acceptable values: 45 | 'top', 46 | 'start' (left of the text, for left-to-right text) 47 | 'end' (right of the text for left-to-right text) 48 | 'bottom' 49 | Raises: 50 | ValueError: if location is not valid 51 | """ 52 | if location not in ('top', 'bottom', 'start', 'end'): 53 | raise ValueError('Invalid value for location: %r', location) 54 | self.tab_names = tab_names 55 | self._location = location 56 | self._active = 0 57 | super(TabBar, self).__init__() 58 | 59 | def _tab_id(self, index): 60 | return '%s_%d' % (self._content_div, index) 61 | 62 | def _html_repr(self): 63 | """Generates html representation for this tab. 64 | 65 | Returns: 66 | string - the html representation 67 | """ 68 | return """
""" % { 69 | 'id': self._id, 70 | } 71 | 72 | def _get_tab_id(self, index_or_name): 73 | if isinstance(index_or_name, six.string_types): 74 | names = tuple(self.tab_names) 75 | index = names.index(index_or_name) 76 | if index_or_name in names[index + 1:]: 77 | raise ValueError('Ambiguous tab name: %s ' % index_or_name) 78 | else: 79 | index = index_or_name 80 | tabid = self._tab_id(index) 81 | return tabid, index 82 | 83 | def _prepare_component_for_output(self, index, select): 84 | if select: 85 | js.js_global[self._id].setSelectedTabIndex(index) 86 | 87 | @contextlib.contextmanager 88 | def output_to(self, tab, select=True): 89 | """Sets current output tab. 90 | 91 | Args: 92 | tab: the tab's name matching one of the tab_names provided in 93 | the constructor. Alternatively a 0-based index can be used. 94 | Note: if tab_names contains duplicates, they can only 95 | be accessed via index. Trying to access by name will trigger 96 | ValueError 97 | 98 | select: if True this will also select the tab, otherwise 99 | the tab will be updated in the background 100 | """ 101 | if not self._published: 102 | self._publish() 103 | tabid, index = self._get_tab_id(tab) 104 | with self._active_component( 105 | tabid, init_params={ 106 | 'select': select, 107 | 'index': index 108 | }): 109 | yield 110 | 111 | def clear_tab(self, tab=None): 112 | """Clears tabs. 113 | 114 | Args: 115 | tab: if None clears current tab otherwise clears the corresponding tab. 116 | Tab could be the tab's name (if it is unique), or 0-based index. 117 | """ 118 | if tab is not None: 119 | tabid, _ = self._get_tab_id(tab) 120 | else: 121 | tabid = None 122 | self._clear_component(tabid) 123 | 124 | def __iter__(self): 125 | """Iterates over tabs. Allows quick population of tab in a loop. 126 | 127 | Yields: 128 | current tab index 129 | """ 130 | self._publish() 131 | for i, _ in enumerate(self.tab_names): 132 | with self.output_to(i): 133 | yield i 134 | 135 | def _publish(self): 136 | """Publishes this tab bar in the given cell. 137 | 138 | Note: this function is idempotent. 139 | """ 140 | if self._published: 141 | return 142 | content_height = 'initial', 143 | content_border = '0px', 144 | border_color = '#a7a7a7', 145 | self._content_div = self._id + '_content' 146 | super(TabBar, self)._publish() 147 | with self._output_in_widget(): 148 | _publish.css(url=self.TAB_CSS) 149 | _publish.javascript(url=self.TABBAR_JS) 150 | _publish.html(self._html_repr()) 151 | 152 | js.js_global.colab_lib.createTabBar({ 153 | 'location': self._location, 154 | 'elementId': self._id, 155 | 'tabNames': self.tab_names, 156 | 'initialSelection': self._active, 157 | 'contentBorder': content_border, 158 | 'contentHeight': content_height, 159 | 'borderColor': border_color, 160 | }) 161 | # Note: publish() will only be called once, thus this will never change 162 | # already visible tab. 163 | js.js_global[self._id].setSelectedTabIndex(0) 164 | -------------------------------------------------------------------------------- /google/colab/widgets/_widget.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language govestylerning permissions and 13 | # limitations under the License. 14 | """Base widget for interactive elements.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import contextlib 21 | 22 | from google.colab import errors 23 | from google.colab import output 24 | from google.colab.output import _tags 25 | from google.colab.output import _util 26 | 27 | # pylint: disable=invalid-name 28 | WidgetException = errors.WidgetException 29 | 30 | 31 | class OutputAreaWidget(object): 32 | """Base widget that redirects output into UI elements, e.g. table.""" 33 | 34 | def __init__(self): 35 | self._published = False 36 | self._id = _util.get_locally_unique_id() 37 | self._saved_output_area = 'output_area_' + self._id 38 | self._tag = 'outputarea_' + self._id 39 | self._publish() 40 | 41 | def _publish(self): 42 | """Publishes this widget. 43 | 44 | Default does nothing but saves the current output area. 45 | """ 46 | _util.flush_all() 47 | self._published = True 48 | # We save active output tags, and use that for all output inside any of the 49 | # subcomponents, even if they are no longer active. 50 | # This is done so that if the table # is deleted via tags that are 51 | # currently active, all its content 52 | # is also cleaned up. 53 | self._saved_output_tags = _tags.get_active_tags() 54 | self._current_component = None 55 | self._output_tags = self._saved_output_tags.union([self._tag]) 56 | 57 | def remove(self, wait=False): 58 | """Removes the widget from the document. 59 | 60 | Args: 61 | wait: if true the actual deletion doesn't happen until the next output. 62 | """ 63 | _util.flush_all() 64 | _tags.clear(wait, self._output_tags) 65 | 66 | @contextlib.contextmanager 67 | def _active_component(self, component_id, init_params=None): 68 | """Sets active subcomponent.""" 69 | if init_params is None: 70 | init_params = {} 71 | if not self._published: 72 | self._publish() 73 | if self._current_component is not None: 74 | raise WidgetException('Already inside a component') 75 | self._current_component = component_id 76 | _util.flush_all() 77 | with self._output_in_widget(): 78 | with output.use_tags(self._current_component): 79 | with output.redirect_to_element('#' + component_id): 80 | # 81 | self._prepare_component_for_output(**init_params) 82 | with output.use_tags('user_output'): 83 | try: 84 | yield 85 | finally: 86 | _util.flush_all() 87 | self._current_component = None 88 | 89 | def _prepare_component_for_output(self, **kwargs): 90 | """Initialization code that's called when component is to become active. 91 | 92 | This function will be called to produce additional browser-side outputs 93 | that needs to be added before adding output to the component. 94 | 95 | For example it can make a tab visible, change styling etc. The important 96 | property is that output produced by this function will be preserved 97 | whenever clear_component is called from *within* this component. 98 | 99 | This function will often remain empty in the implementation. 100 | 101 | Args: 102 | **kwargs: any extra arguments passed by the implementing class 103 | to _active_component 104 | """ 105 | 106 | @contextlib.contextmanager 107 | def _output_in_widget(self): 108 | with output.use_tags(self._output_tags): 109 | try: 110 | yield 111 | finally: 112 | _util.flush_all() 113 | 114 | def _clear_component(self, component_id=None, wait=False): 115 | """Clears component. 116 | 117 | If component_id is None, it will clear currently active component, 118 | otherwise it will clear one with given id. 119 | 120 | NOTE FOR SUBCLASS IMPLEMENTTERS: 121 | 122 | When _clear_output is called it will remove all outputs that were created 123 | within context of _active_component. 124 | 125 | This might produce subtle errors in situations where user clears component 126 | he is currently producing output for as it will destroy any output that 127 | is in the context of _active_component. Therefore if your widget 128 | needs javascript to setup the component for output it should always 129 | be produced by overloading _prepare_component_for_output. 130 | 131 | Args: 132 | component_id: which component to clear. 133 | wait: if True, the output won't be cleared until the next user output. 134 | See colab.output.clear for full details. 135 | 136 | Raises: 137 | WidgetException: if component_id and no active element is 138 | selected 139 | """ 140 | _util.flush_all() 141 | if component_id is None: 142 | if self._current_component is None: 143 | raise WidgetException('No active component selected') 144 | component_id = self._current_component 145 | if component_id == self._current_component: 146 | # Do not clear the part that sets current active element. 147 | # If we did, this would have made all consecutive output to stream 148 | # to wrong outputarea on reload. 149 | output.clear(wait, output_tags=[component_id] + ['user_output']) 150 | else: 151 | output.clear(wait, output_tags=[component_id]) 152 | -------------------------------------------------------------------------------- /google/colab/_message.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colab-specific messaging helpers.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import time 21 | 22 | import zmq 23 | 24 | from google.colab import _ipython as ipython 25 | from google.colab import errors 26 | 27 | _NOT_READY = object() 28 | 29 | 30 | class MessageError(errors.Error): 31 | """Thrown on error response from frontend.""" 32 | 33 | 34 | def _read_next_input_message(): 35 | """Reads the next message from stdin_socket. 36 | 37 | Returns: 38 | _NOT_READY if input is not available. 39 | """ 40 | kernel = ipython.get_kernel() 41 | stdin_socket = kernel.stdin_socket 42 | 43 | reply = None 44 | try: 45 | _, reply = kernel.session.recv(stdin_socket, zmq.NOBLOCK) 46 | except Exception: # pylint: disable=broad-except 47 | # We treat invalid messages as empty replies. 48 | pass 49 | if reply is None: 50 | return _NOT_READY 51 | 52 | # We want to return '' even if reply is malformed. 53 | return reply.get('content', {}).get('value', '') 54 | 55 | 56 | def _read_stdin_message(): 57 | """Reads a stdin message. 58 | 59 | This discards any colab messaging replies that may arrive on the stdin_socket. 60 | 61 | Returns: 62 | The input message or None if input is not available. 63 | """ 64 | while True: 65 | value = _read_next_input_message() 66 | if value == _NOT_READY: 67 | return None 68 | 69 | # Skip any colab responses. 70 | if isinstance(value, dict) and value.get('type') == 'colab_reply': 71 | continue 72 | 73 | return value 74 | 75 | 76 | def read_reply_from_input(message_id, timeout_sec=None): 77 | """Reads a reply to the message from the stdin channel. 78 | 79 | Any extraneous input or messages received on the stdin channel while waiting 80 | for the reply are discarded. This blocks until the reply has been 81 | received. 82 | 83 | Args: 84 | message_id: the ID of the request for which this is blocking. 85 | timeout_sec: blocks for that many seconds. 86 | 87 | Returns: 88 | If a reply of type `colab_reply` for `message_id` is returned before the 89 | timeout, we return reply['data'] or None. 90 | If a timeout is provided and no reply is received, we return None. 91 | 92 | Raises: 93 | MessageError: if a reply is returned to us with an error. 94 | """ 95 | deadline = None 96 | if timeout_sec: 97 | deadline = time.time() + timeout_sec 98 | while not deadline or time.time() < deadline: 99 | reply = _read_next_input_message() 100 | if reply == _NOT_READY or not isinstance(reply, dict): 101 | time.sleep(0.025) 102 | continue 103 | if (reply.get('type') == 'colab_reply' and 104 | reply.get('colab_msg_id') == message_id): 105 | if 'error' in reply: 106 | raise MessageError(reply['error']) 107 | return reply.get('data', None) 108 | 109 | # Global counter for message id. 110 | # Note: this is not thread safe, if we want to make this 111 | # thread sfe we should replace this with thread safe counter 112 | # And add appropriate thread handling logic to read_reply_from_input 113 | _msg_id = 0 114 | 115 | 116 | def send_request(request_type, request_body, parent=None): 117 | """Sends the given message to the frontend without waiting for a reply.""" 118 | 119 | instance = ipython.get_kernelapp() 120 | global _msg_id 121 | _msg_id += 1 122 | request_id = _msg_id 123 | 124 | metadata = { 125 | 'colab_msg_id': request_id, 126 | 'colab_request_type': request_type, 127 | } 128 | 129 | content = { 130 | 'request': request_body, 131 | } 132 | 133 | # If there's no parent message, add in the session header to route to the 134 | # appropriate frontend. 135 | if parent is None: 136 | parent_header = instance.kernel.shell.parent_header 137 | if parent_header: 138 | parent = { 139 | 'header': { 140 | # Only specifying the session if it is not a cell-related message. 141 | 'session': parent_header['header']['session'] 142 | }, 143 | } 144 | 145 | msg = instance.session.msg( 146 | 'colab_request', content=content, metadata=metadata, parent=parent) 147 | instance.session.send(instance.iopub_socket, msg) 148 | 149 | return request_id 150 | 151 | 152 | def blocking_request(request_type, request='', timeout_sec=5, parent=None): 153 | """Calls the front end with a request, and blocks until a reply is received. 154 | 155 | Note: this function is not thread safe, e.g. if two threads 156 | send blocking_request they will likely race with each other and consume 157 | each other responses leaving another thread deadlocked. 158 | 159 | Args: 160 | request_type: type of request being made 161 | request: Jsonable object to send to front end as the request. 162 | timeout_sec: max number of seconds to block, None, for no timeout. 163 | parent: Parent message, for routing. 164 | Returns: 165 | Reply by front end (Json'able object), or None if the timeout occurs. 166 | """ 167 | # If we want this thread safe we can make read_reply_from_input to 168 | # not discard messages with unknown msg ids as well as making msg_ids globally 169 | # unique. 170 | request_id = send_request(request_type, request, parent=parent) 171 | return read_reply_from_input(request_id, timeout_sec) 172 | -------------------------------------------------------------------------------- /google/colab/_autocomplete/_splitter.py: -------------------------------------------------------------------------------- 1 | """Contains drop in replacement for IPython.CompletionSplitter class.""" 2 | import re 3 | import tokenize 4 | 5 | # List of positional params as they arrive from the callback 6 | _TOKEN_TYPE = 0 7 | _TOKEN = 1 8 | _TOKEN_START = 2 9 | _TOKEN_END = 3 10 | 11 | _brackets = {']': '[', ')': '(', '.': '.'} 12 | 13 | 14 | def _quittable(context, token_type, token, *unused_args): 15 | """Returns true if given token might be the end of the completion token.""" 16 | 17 | if token in _brackets: 18 | return False 19 | if (token_type in {tokenize.NAME, tokenize.STRING, tokenize.NUMBER} and 20 | context.maybe_followed_by_name): 21 | return False 22 | 23 | return True 24 | 25 | 26 | def _push_token(context, unused_token_type, token, *unused_args): 27 | """Pushes token into a stack as needed.""" 28 | if token in _brackets.values(): 29 | context.maybe_followed_by_name = True 30 | else: 31 | context.maybe_followed_by_name = False 32 | 33 | stack = context.stack 34 | # Remove the previous period first, since it is just there to ensure 35 | # that we get the hold on to next token 36 | if stack and stack[-1] == '.': 37 | stack.pop() 38 | 39 | if token in _brackets: 40 | stack.append(_brackets[token]) 41 | 42 | elif token == '.': 43 | stack.append('.') 44 | elif not stack: 45 | return 46 | elif stack[-1] == token: 47 | stack.pop() 48 | 49 | 50 | class _Context(object): 51 | """Describes current parsing context as we parse from right to left.""" 52 | 53 | def __init__(self): 54 | self.stack = [] 55 | self.maybe_followed_by_name = False 56 | 57 | 58 | def _find_expression_start(tokens): 59 | """Finds the start of the expression that needs autocompletion. 60 | 61 | Args: 62 | tokens: list of tokens, where each token is 5-tuple as arrived from 63 | 'tokenize' 64 | Returns: 65 | index of the first token that should be included in completion. 66 | """ 67 | if not tokens: 68 | return 0 69 | # Last token is eof, ignore 70 | i = len(tokens) - 1 71 | while tokens[i][_TOKEN_TYPE] in {tokenize.ENDMARKER, tokenize.DEDENT}: 72 | i -= 1 73 | context = _Context() 74 | context.maybe_followed_by_name = True 75 | while i >= 0 and (context.stack or not _quittable(context, *tokens[i])): 76 | _push_token(context, *tokens[i]) 77 | i -= 1 78 | 79 | return i + 1 80 | 81 | 82 | def _last_real_token(tokens): 83 | i = len(tokens) - 1 84 | while i >= 0 and tokens[i][_TOKEN_TYPE] == tokenize.ENDMARKER: 85 | i -= 1 86 | if i < 0: 87 | return '' 88 | return tokens[i][_TOKEN] 89 | 90 | 91 | def split(s): 92 | """Splits one last token that needs to be autocompleted.""" 93 | # Treat magics specially, since they don't follow python syntax 94 | # and require '%%' symbols to be preserved 95 | magic_match = re.search(r'%%?\w+$', s) 96 | if magic_match: 97 | return magic_match.group(0) 98 | 99 | s2 = s.rstrip() 100 | if s != s2: 101 | # If there is whitespace at the end of the string 102 | # the completion token is empty 103 | return '' 104 | tokens = [] 105 | 106 | # Remove front whitespace, somehow it confuses tokenizer 107 | s = s.lstrip() 108 | 109 | # accumulates all arguments in to the array 110 | accumulate = lambda *args: tokens.append(args) 111 | 112 | try: 113 | # Convert input into readline analog 114 | lines = s.split('\n') 115 | # Add '\n to all lines except last one. 116 | lines[:-1] = [line + '\n' for line in lines[:-1]] 117 | readline = (e for e in lines).next 118 | tokenize.tokenize(readline, accumulate) 119 | except tokenize.TokenError: 120 | # Tokenizer failed, usually an indication of not-terminated strings. 121 | # Remove all quotes and return the last sequence of not-spaces 122 | if not tokens: 123 | s = s.replace('"', ' ').replace("'", ' ').split() 124 | return s[-1] if s else '' 125 | except Exception: # pylint: disable=broad-except 126 | # If all else fails, use poor's man tokenizer 127 | s = s.split() 128 | return s[-1] if s else '' 129 | 130 | # First we check if there is unfished quoted string 131 | for each in reversed(tokens): 132 | if each[_TOKEN_TYPE] == tokenize.ERRORTOKEN and each[_TOKEN] in { 133 | "'", '"', '"""', "'''" 134 | }: 135 | line = each[_TOKEN_END][0] - 1 136 | col = each[_TOKEN_END][1] 137 | return lines[line][col:] 138 | 139 | start_token = _find_expression_start(tokens) 140 | 141 | if start_token >= len(tokens): 142 | # This prevents us from generating random completions when there is 143 | # no completion to be generated 144 | return _last_real_token(tokens) 145 | 146 | start_pos = tokens[start_token][_TOKEN_START] 147 | 148 | first_line_index = start_pos[0] - 1 149 | if first_line_index >= len(lines): 150 | return _last_real_token(tokens) 151 | 152 | first_line = lines[first_line_index][start_pos[1]:] 153 | result = first_line + ''.join(lines[first_line_index + 1:]) 154 | return result 155 | 156 | 157 | class _CompletionSplitter(object): 158 | """Drop-in replacement for IPython CompletionSplitter replacement.""" 159 | 160 | def __init__(self): 161 | # Used by ipython 162 | self.delims = '' 163 | 164 | def split_line(self, line, cursor_pos=None): 165 | """Split a line of text with a cursor at the given position.""" 166 | l = line if cursor_pos is None else line[:cursor_pos] 167 | result = split(l) 168 | return result 169 | 170 | 171 | _original_splitter = None 172 | 173 | 174 | def enable(ip): 175 | """Installs completion splitter into IPython instance.""" 176 | global _original_splitter 177 | if isinstance(ip.Completer.splitter, _CompletionSplitter): 178 | # Already installed 179 | return 180 | _original_splitter = ip.Completer.splitter 181 | ip.Completer.splitter = _CompletionSplitter() 182 | 183 | 184 | def disable(ip): 185 | global _original_splitter 186 | if _original_splitter is None: 187 | return 188 | 189 | ip.Completer.splitter = _original_splitter 190 | _original_splitter = None 191 | -------------------------------------------------------------------------------- /google/colab/drive.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colab-specific Google Drive integration.""" 15 | 16 | from __future__ import absolute_import as _ 17 | from __future__ import division as _ 18 | from __future__ import print_function as _ 19 | 20 | import getpass as _getpass 21 | import os as _os 22 | import re as _re 23 | import socket as _socket 24 | import sys as _sys 25 | import uuid as _uuid 26 | 27 | import pexpect as _pexpect 28 | 29 | __all__ = ['mount'] 30 | 31 | 32 | def mount(mountpoint, force_remount=False): 33 | """Mount your Google Drive at the specified mountpoint path.""" 34 | 35 | mountpoint = _os.path.expanduser(mountpoint) 36 | # If we've already mounted drive at the specified mountpoint, exit now. 37 | already_mounted = _os.path.isdir(_os.path.join(mountpoint, 'My Drive')) 38 | if not force_remount and already_mounted: 39 | print('Drive already mounted at {}; to attempt to forcibly remount, ' 40 | 'call drive.mount("{}", force_remount=True).'.format( 41 | mountpoint, mountpoint)) 42 | return 43 | home = _os.environ['HOME'] 44 | root_dir = _os.path.realpath( 45 | _os.path.join(_os.environ['CLOUDSDK_CONFIG'], '../..')) 46 | inet_family = 'IPV4_ONLY' 47 | dev = '/dev/fuse' 48 | path = '/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:.' 49 | if len(root_dir) > 1: 50 | home = _os.path.join(root_dir, home) 51 | inet_family = 'IPV6_ONLY' 52 | fum = _os.environ['HOME'].split('mount')[0] + '/mount/alloc/fusermount' 53 | dev = fum + '/dev/fuse' 54 | path = path + ':' + fum + '/bin' 55 | config_dir = _os.path.join(home, '.config', 'Google') 56 | try: 57 | _os.makedirs(config_dir) 58 | except OSError: 59 | if not _os.path.isdir(config_dir): 60 | raise ValueError('{} must be a directory if present'.format(config_dir)) 61 | 62 | # Launch an intermediate bash inside of which drive is launched, so that 63 | # after auth is done we can daemonize drive with its stdout/err no longer 64 | # being captured by pexpect. Otherwise buffers will eventually fill up and 65 | # drive may hang, because pexpect doesn't have a .startDiscardingOutput() 66 | # call (https://github.com/pexpect/pexpect/issues/54). 67 | prompt = u'root@{}-{}: '.format(_socket.gethostname(), _uuid.uuid4().hex) 68 | d = _pexpect.spawn( 69 | '/bin/bash', 70 | args=['--noediting'], 71 | timeout=120, 72 | maxread=int(1e6), 73 | encoding='utf-8', 74 | env={ 75 | 'HOME': home, 76 | 'FUSE_DEV_NAME': dev, 77 | 'PATH': path 78 | }) 79 | if mount._DEBUG: # pylint:disable=protected-access 80 | d.logfile_read = _sys.stdout 81 | d.sendline('export PS1="{}"'.format(prompt)) 82 | d.expect(prompt) # The echoed input above. 83 | d.expect(prompt) # The new prompt. 84 | # Robustify to previously-running copies of drive. Don't only [pkill -9] 85 | # because that leaves enough cruft behind in the mount table that future 86 | # operations fail with "Transport endpoint is not connected". 87 | d.sendline('umount -f {mnt} || umount {mnt}; pkill -9 -x drive'.format( 88 | mnt=mountpoint)) 89 | # Wait for above to be received, using the next prompt. 90 | d.expect(u'pkill') # Echoed command. 91 | d.expect(prompt) 92 | # Only check the mountpoint after potentially unmounting/pkill'ing above. 93 | try: 94 | if _os.path.islink(mountpoint): 95 | raise ValueError('Mountpoint must not be a symlink') 96 | if _os.path.isdir(mountpoint) and _os.listdir(mountpoint): 97 | raise ValueError('Mountpoint must not already contain files') 98 | if not _os.path.isdir(mountpoint) and _os.path.exists(mountpoint): 99 | raise ValueError('Mountpoint must either be a directory or not exist') 100 | except: 101 | d.terminate(force=True) 102 | raise 103 | 104 | # Watch for success. 105 | success = u'google.colab.drive MOUNTED' 106 | success_watcher = ( 107 | '( while `sleep 0.5`; do if [[ -d "{m}" && "$(ls -A {m})" != "" ]]; ' 108 | 'then echo "{s}"; break; fi; done ) &').format( 109 | m=mountpoint, s=success) 110 | d.sendline(success_watcher) 111 | d.expect(prompt) # Eat the match of the input command above being echoed. 112 | drive_dir = _os.path.join(root_dir, 'opt/google/drive') 113 | d.sendline(('{d}/drive --features=virtual_folders:true ' 114 | '--inet_family=' + inet_family + ' ' 115 | '--preferences=trusted_root_certs_file_path:' 116 | '{d}/roots.pem,mount_point_path:{mnt} --console_auth').format( 117 | d=drive_dir, mnt=mountpoint)) 118 | 119 | while True: 120 | case = d.expect([ 121 | success, prompt, 122 | _re.compile(u'(Go to this URL in a browser: https://.*)\r\n') 123 | ]) 124 | if case == 0: 125 | break 126 | elif case == 1: 127 | # Prompt appearing here means something went wrong with the drive binary. 128 | d.terminate(force=True) 129 | raise ValueError('mount failed') 130 | elif case == 2: 131 | # Not already authorized, so do the authorization dance. 132 | prompt = d.match.group(1) + '\n\nEnter your authorization code:\n' 133 | d.send(_getpass.getpass(prompt) + '\n') 134 | d.sendcontrol('z') 135 | d.expect(u'Stopped') 136 | d.sendline('bg; disown; exit') 137 | d.expect(_pexpect.EOF) 138 | assert not d.isalive() 139 | assert d.exitstatus == 0 140 | print('Mounted at {}'.format(mountpoint)) 141 | 142 | 143 | mount._DEBUG = False # pylint:disable=protected-access 144 | -------------------------------------------------------------------------------- /google/colab/resources/files.js: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /** 16 | * @fileoverview Helpers for google.colab Python module. 17 | */ 18 | (function(scope) { 19 | function span(text, styleAttributes = {}) { 20 | const element = document.createElement('span'); 21 | element.textContent = text; 22 | for (const key of Object.keys(styleAttributes)) { 23 | element.style[key] = styleAttributes[key]; 24 | } 25 | return element; 26 | } 27 | 28 | // Max number of bytes which will be uploaded at a time. 29 | const MAX_PAYLOAD_SIZE = 100 * 1024; 30 | // Max amount of time to block waiting for the user. 31 | const FILE_CHANGE_TIMEOUT_MS = 30 * 1000; 32 | 33 | function _uploadFiles(inputId, outputId) { 34 | const steps = uploadFilesStep(inputId, outputId); 35 | const outputElement = document.getElementById(outputId); 36 | // Cache steps on the outputElement to make it available for the next call 37 | // to uploadFilesContinue from Python. 38 | outputElement.steps = steps; 39 | 40 | return _uploadFilesContinue(outputId); 41 | } 42 | 43 | // This is roughly an async generator (not supported in the browser yet), 44 | // where there are multiple asynchronous steps and the Python side is going 45 | // to poll for completion of each step. 46 | // This uses a Promise to block the python side on completion of each step, 47 | // then passes the result of the previous step as the input to the next step. 48 | function _uploadFilesContinue(outputId) { 49 | const outputElement = document.getElementById(outputId); 50 | const steps = outputElement.steps; 51 | 52 | const next = steps.next(outputElement.lastPromiseValue); 53 | return Promise.resolve(next.value.promise).then((value) => { 54 | // Cache the last promise value to make it available to the next 55 | // step of the generator. 56 | outputElement.lastPromiseValue = value; 57 | return next.value.response; 58 | }); 59 | } 60 | 61 | /** 62 | * Generator function which is called between each async step of the upload 63 | * process. 64 | * @param {string} inputId Element ID of the input file picker element. 65 | * @param {string} outputId Element ID of the output display. 66 | * @return {!Iterable} Iterable of next steps. 67 | */ 68 | function* uploadFilesStep(inputId, outputId) { 69 | const inputElement = document.getElementById(inputId); 70 | inputElement.disabled = false; 71 | 72 | const outputElement = document.getElementById(outputId); 73 | outputElement.innerHTML = ''; 74 | 75 | const pickedPromise = new Promise((resolve) => { 76 | inputElement.addEventListener('change', (e) => { 77 | resolve(e.target.files); 78 | }); 79 | }); 80 | 81 | const cancel = document.createElement('button'); 82 | inputElement.parentElement.appendChild(cancel); 83 | cancel.textContent = 'Cancel upload'; 84 | const cancelPromise = new Promise((resolve) => { 85 | cancel.onclick = () => { 86 | resolve(null); 87 | }; 88 | }); 89 | 90 | // Cancel upload if user hasn't picked anything in timeout. 91 | const timeoutPromise = new Promise((resolve) => { 92 | setTimeout(() => { 93 | resolve(null); 94 | }, FILE_CHANGE_TIMEOUT_MS); 95 | }); 96 | 97 | // Wait for the user to pick the files. 98 | const files = yield { 99 | promise: Promise.race([pickedPromise, timeoutPromise, cancelPromise]), 100 | response: { 101 | action: 'starting', 102 | } 103 | }; 104 | 105 | if (!files) { 106 | return { 107 | response: { 108 | action: 'complete', 109 | } 110 | }; 111 | } 112 | 113 | cancel.remove(); 114 | 115 | // Disable the input element since further picks are not allowed. 116 | inputElement.disabled = true; 117 | 118 | for (const file of files) { 119 | const li = document.createElement('li'); 120 | li.append(span(file.name, {fontWeight: 'bold'})); 121 | li.append(span( 122 | `(${file.type || 'n/a'}) - ${file.size} bytes, ` + 123 | `last modified: ${ 124 | file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() : 125 | 'n/a'} - `)); 126 | const percent = span('0% done'); 127 | li.appendChild(percent); 128 | 129 | outputElement.appendChild(li); 130 | 131 | const fileDataPromise = new Promise((resolve) => { 132 | const reader = new FileReader(); 133 | reader.onload = (e) => { 134 | resolve(e.target.result); 135 | }; 136 | reader.readAsArrayBuffer(file); 137 | }); 138 | // Wait for the data to be ready. 139 | let fileData = yield { 140 | promise: fileDataPromise, 141 | response: { 142 | action: 'continue', 143 | } 144 | }; 145 | 146 | // Use a chunked sending to avoid message size limits. See b/62115660. 147 | let position = 0; 148 | while (position < fileData.byteLength) { 149 | const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE); 150 | const chunk = new Uint8Array(fileData, position, length); 151 | position += length; 152 | 153 | const base64 = btoa(String.fromCharCode.apply(null, chunk)); 154 | yield { 155 | response: { 156 | action: 'append', 157 | file: file.name, 158 | data: base64, 159 | }, 160 | }; 161 | percent.textContent = 162 | `${Math.round((position / fileData.byteLength) * 100)}% done`; 163 | } 164 | } 165 | 166 | // All done. 167 | yield { 168 | response: { 169 | action: 'complete', 170 | } 171 | }; 172 | } 173 | 174 | scope.google = scope.google || {}; 175 | scope.google.colab = scope.google.colab || {}; 176 | scope.google.colab._files = { 177 | _uploadFiles, 178 | _uploadFilesContinue, 179 | }; 180 | })(self); 181 | -------------------------------------------------------------------------------- /google/colab/files.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colab-specific file helpers.""" 15 | 16 | from __future__ import absolute_import as _ 17 | from __future__ import division as _ 18 | from __future__ import print_function as _ 19 | 20 | import base64 as _base64 21 | import collections as _collections 22 | import os as _os 23 | import socket as _socket 24 | import threading as _threading 25 | import uuid as _uuid 26 | 27 | import IPython as _IPython 28 | import portpicker as _portpicker 29 | import six as _six 30 | from six.moves import SimpleHTTPServer as _SimpleHTTPServer 31 | from six.moves import socketserver as _socketserver 32 | from six.moves import urllib as _urllib 33 | 34 | from google.colab import output as _output 35 | 36 | __all__ = ['upload', 'download'] 37 | 38 | 39 | def upload(): 40 | """Renders widget to upload local (to the browser) files to the kernel. 41 | 42 | Blocks until the files are available. 43 | 44 | Returns: 45 | A map of the form {: } for all uploaded files. 46 | """ 47 | upload_id = str(_uuid.uuid4()) 48 | input_id = 'files-' + upload_id 49 | output_id = 'result-' + upload_id 50 | 51 | _IPython.display.display( 52 | _IPython.core.display.HTML(""" 53 | 54 | 55 | Upload widget is only available when the cell has been executed in the 56 | current browser session. Please rerun this cell to enable. 57 | 58 | """.format( 59 | input_id=input_id, output_id=output_id))) 60 | 61 | # First result is always an indication that the file picker has completed. 62 | result = _output.eval_js( 63 | 'google.colab._files._uploadFiles("{input_id}", "{output_id}")'.format( 64 | input_id=input_id, output_id=output_id)) 65 | files = _collections.defaultdict(_six.binary_type) 66 | # Mapping from original filename to filename as saved locally. 67 | local_filenames = dict() 68 | 69 | while result['action'] != 'complete': 70 | result = _output.eval_js( 71 | 'google.colab._files._uploadFilesContinue("{output_id}")'.format( 72 | output_id=output_id)) 73 | if result['action'] != 'append': 74 | # JS side uses a generator of promises to process all of the files- some 75 | # steps may not produce data for the Python side, so just proceed onto the 76 | # next message. 77 | continue 78 | data = _base64.b64decode(result['data']) 79 | filename = result['file'] 80 | 81 | files[filename] += data 82 | local_filename = local_filenames.get(filename) 83 | if not local_filename: 84 | local_filename = _get_unique_filename(filename) 85 | local_filenames[filename] = local_filename 86 | print('Saving {filename} to {local_filename}'.format( 87 | filename=filename, local_filename=local_filename)) 88 | with open(local_filename, 'ab') as f: 89 | f.write(data) 90 | 91 | return dict(files) 92 | 93 | 94 | def _get_unique_filename(filename): 95 | if not _os.path.lexists(filename): 96 | return filename 97 | counter = 1 98 | while True: 99 | path, ext = _os.path.splitext(filename) 100 | new_filename = '{} ({}){}'.format(path, counter, ext) 101 | if not _os.path.lexists(new_filename): 102 | return new_filename 103 | counter += 1 104 | 105 | 106 | class _V6Server(_socketserver.TCPServer): 107 | address_family = _socket.AF_INET6 108 | 109 | 110 | class _FileHandler(_SimpleHTTPServer.SimpleHTTPRequestHandler): 111 | """SimpleHTTPRequestHandler with a couple tweaks.""" 112 | 113 | def translate_path(self, path): 114 | # Client specifies absolute paths. 115 | # TODO(b/79760241): Remove this spurious lint warning. 116 | return _urllib.parse.unquote(path) # pylint:disable=too-many-function-args 117 | 118 | def log_message(self, fmt, *args): 119 | # Suppress logging since it's on the background. Any errors will be reported 120 | # via the handler. 121 | pass 122 | 123 | def end_headers(self): 124 | # Do not cache the response in the notebook, since it may be quite large. 125 | self.send_header('x-colab-notebook-cache-control', 'no-cache') 126 | _SimpleHTTPServer.SimpleHTTPRequestHandler.end_headers(self) 127 | 128 | 129 | def download(filename): 130 | """Downloads the file to the user's local disk via a browser download action. 131 | 132 | Args: 133 | filename: Name of the file on disk to be downloaded. 134 | 135 | Raises: 136 | OSError: if the file cannot be found. 137 | """ 138 | 139 | if not _os.path.exists(filename): 140 | msg = 'Cannot find file: {}'.format(filename) 141 | if _six.PY2: 142 | raise OSError(msg) 143 | else: 144 | raise FileNotFoundError(msg) # pylint: disable=undefined-variable 145 | 146 | started = _threading.Event() 147 | port = _portpicker.pick_unused_port() 148 | 149 | def server_entry(): 150 | httpd = _V6Server(('::', port), _FileHandler) 151 | started.set() 152 | # Handle a single request then exit the thread. 153 | httpd.handle_request() 154 | 155 | thread = _threading.Thread(target=server_entry) 156 | thread.start() 157 | started.wait() 158 | 159 | _output.eval_js( 160 | """ 161 | (async function() { 162 | const response = await fetch('https://localhost:%(port)d%(path)s'); 163 | if (!response.ok) { 164 | throw new Error('Failed to download: ' + response.statusText); 165 | } 166 | const blob = await response.blob(); 167 | 168 | const a = document.createElement('a'); 169 | a.href = window.URL.createObjectURL(blob); 170 | a.download = '%(name)s'; 171 | document.body.appendChild(a); 172 | a.click(); 173 | a.remove(); 174 | })(); 175 | """ % { 176 | 'port': port, 177 | 'path': _os.path.abspath(filename), 178 | 'name': _os.path.basename(filename), 179 | }) 180 | -------------------------------------------------------------------------------- /google/colab/output/_tags.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """This module provides support for tagging and selectively deleting outputs.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | import contextlib 20 | 21 | import sys 22 | import threading 23 | import uuid 24 | 25 | import IPython 26 | from IPython import display 27 | import six 28 | from google.colab import _ipython as ipython 29 | 30 | 31 | def _add_or_remove_tags(tags_to_add=(), tags_to_remove=()): 32 | """Adds or removes tags from the frontend.""" 33 | # Clear tags when this cell is done. 34 | output_tags = _get_or_create_tags() 35 | tags_to_add = tuple(tags_to_add) 36 | tags_to_remove = tuple(tags_to_remove) 37 | 38 | output_tags.update(tags_to_add) 39 | output_tags.difference_update(tags_to_remove) 40 | 41 | sys.stdout.flush() 42 | sys.stderr.flush() 43 | 44 | metadata = { 45 | 'outputarea': { 46 | 'nodisplay': True, 47 | 'add_tags': tags_to_add, 48 | 'remove_tags': tags_to_remove 49 | } 50 | } 51 | 52 | if ipython.in_ipython(): 53 | if IPython.version_info[0] > 2: 54 | display.publish_display_data({}, metadata=metadata) 55 | else: 56 | display.publish_display_data('display', {}, metadata=metadata) 57 | 58 | return output_tags 59 | 60 | 61 | def _convert_string_to_list(v): 62 | return [v] if isinstance(v, six.string_types) else v 63 | 64 | 65 | # Thread local storage for tags 66 | _tls = threading.local() 67 | 68 | 69 | def _get_or_create_tags(create=True): 70 | if not hasattr(_tls, 'tags') and create: 71 | _tls.tags = set() 72 | return getattr(_tls, 'tags', None) 73 | 74 | 75 | def reset_tags(): 76 | """Resets output tags in the runtime. 77 | 78 | This function is an escape hatch in case runtime ends up in 79 | inconsistent state between frontend (where tags are cell local) 80 | and runtime (which mantains global state). 81 | """ 82 | if hasattr(_tls, 'tags'): 83 | del _tls.tags 84 | 85 | 86 | def get_active_tags(): 87 | return set(_get_or_create_tags(create=False) or ()) 88 | 89 | 90 | @contextlib.contextmanager 91 | def use_tags(tags, append=True): 92 | """Will add `tags` to all outputs within this context. 93 | 94 | Tags allow user to mark output (such as one produce by print statments, 95 | images and any other output ), and later delete a subset of it 96 | that have given set of tags. 97 | 98 | Note 1: the set of tags will be restored even if underlying code 99 | throws exception. 100 | 101 | Note 2: This function is not thread safe. 102 | If this function is accessed from non-ui thread, it might lead to 103 | racing behaviors unless special care is taken of clean ups, otherwise 104 | tags added from different threads will interfere with each other on frontend. 105 | 106 | Note 3: Using this function outside of context manager is not supported and 107 | is undefined behavior. Specifically if __exit__ is not called across 108 | single run_cell, this might lead to broken output in future cell runs. 109 | 110 | Args: 111 | tags: one or more tags to attach to outputs within this context 112 | append: if true, the set of tag will be added to currently 113 | active, otherwise it will replace the set. 114 | 115 | Yields: 116 | set of current tags. 117 | """ 118 | tags = _convert_string_to_list(tags) 119 | try: 120 | current_tags = set(_get_or_create_tags()) 121 | # remove all tags which were not in the current_tags 122 | added_tags = set(tags).difference(current_tags) 123 | remove_tags = [] if append else set(current_tags).difference(tags) 124 | tags = _add_or_remove_tags( 125 | tags_to_add=added_tags, tags_to_remove=remove_tags) 126 | yield tags 127 | finally: 128 | _add_or_remove_tags(tags_to_add=remove_tags, tags_to_remove=added_tags) 129 | 130 | 131 | def clear(wait=False, output_tags=()): 132 | """Clears all output marked with a superset of a given output_tags. 133 | 134 | For example: 135 | from google.colab import output 136 | with output.use_tag('conversation'): 137 | with output.use_tag('introduction'): 138 | print 'Hello' 139 | print 'Bye' 140 | 141 | # This will remove 'hello' from the output 142 | output.clear(output_tags='introduction') 143 | # This will remove bye from the output 144 | output.clear(output_tags='conversation') 145 | 146 | If wait is true, the operation will be deferred 147 | until next output statement. For example: 148 | 149 | print "hello" 150 | output.clear(wait=True) 151 | time.sleep(10) 152 | print "bye" 153 | 154 | will have "Hello" printed for 10 seconds, then replace it with "bye". 155 | 156 | Args: 157 | wait: whether to wait until the next output before clearing output. 158 | 159 | output_tags: string or iterable over strings. If provided, only 160 | outputs that are marked with superset of output_tags will be cleared. 161 | """ 162 | output_tags = _convert_string_to_list(output_tags) 163 | content = dict(wait=wait, output_tags=tuple(output_tags)) 164 | 165 | # In contrast with ipython we don't send any extraneous symbols to 166 | # stdin/stdout. 167 | sys.stdout.flush() 168 | sys.stderr.flush() 169 | 170 | ip = IPython.get_ipython() 171 | if not ip or not hasattr(ip, 'kernel'): 172 | return 173 | 174 | if not hasattr(ip.kernel, 'shell'): 175 | return 176 | display_pub = ip.kernel.shell.display_pub 177 | if not hasattr(display_pub, 'pub_socket'): 178 | return 179 | session = ip.kernel.session 180 | session.send( 181 | display_pub.pub_socket, 182 | u'clear_output', 183 | content, 184 | parent=display_pub.parent_header, 185 | ident=display_pub.topic) 186 | 187 | 188 | @contextlib.contextmanager 189 | def temporary(): 190 | """Outputs produced within this context will be cleared upon exiting. 191 | 192 | Note: if context throws exception no output will be cleared. 193 | 194 | Yields: 195 | None 196 | """ 197 | temptag = str(uuid.uuid4()) 198 | with use_tags(temptag): 199 | yield temptag 200 | clear(output_tags=temptag) 201 | -------------------------------------------------------------------------------- /tests/serverextension/test_handlers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for the google.colab._handlers package.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import collections 21 | import logging 22 | import subprocess 23 | import sys 24 | from distutils import spawn 25 | 26 | import psutil 27 | 28 | from tornado import escape 29 | from tornado import testing 30 | from tornado import web 31 | 32 | from google.colab import _serverextension 33 | from google.colab._serverextension import _handlers 34 | 35 | # pylint:disable=g-import-not-at-top 36 | try: 37 | from unittest import mock 38 | except ImportError: 39 | import mock 40 | # pylint:enable=g-import-not-at-top 41 | 42 | 43 | class FakeNotebookServer(object): 44 | 45 | def __init__(self, app): 46 | self.web_app = app 47 | self.log = logging.getLogger('fake_notebook_server_logger') 48 | 49 | 50 | class FakeUsage(object): 51 | """Provides methods that fake memory usage shell invocations.""" 52 | kernel_ids = [ 53 | '3a7c6914-ce88-4ae8-a37a-9ecf7a21bbef', 54 | 'cfb901b0-c55a-4536-94f8-4d9357265a7a' 55 | ] 56 | 57 | pids = [1, 2] 58 | 59 | usage = [135180, 143736] 60 | 61 | gpu_usage = {'usage': 841, 'limit': 4035} 62 | 63 | disk_usage = psutil._common.sdiskusage( # pylint: disable=protected-access 64 | total=10, used=1, free=9, percent=10) 65 | 66 | _ps_output = '{} {}\n{} {}\n'.format(pids[0], usage[0], pids[1], usage[1]) 67 | 68 | _nvidia_smi_output = '{}, {}\n'.format(gpu_usage['usage'], gpu_usage['limit']) 69 | 70 | @staticmethod 71 | def fake_check_output(cmdline): 72 | output = '' 73 | if cmdline[0] == 'ps': 74 | output = FakeUsage._ps_output 75 | elif 'nvidia-smi' in cmdline: 76 | output = FakeUsage._nvidia_smi_output 77 | if sys.version_info[0] == 3: # returns bytes in py3, string in py2 78 | return bytes(output.encode('utf-8')) 79 | return output 80 | 81 | 82 | class FakeKernelManager(object): 83 | """Provides methods faking an IPython MultiKernelManager.""" 84 | _kernel_factory = collections.namedtuple('FakeKernelFactory', ['kernel']) 85 | _popen = collections.namedtuple('FakePOpen', ['pid']) 86 | 87 | def list_kernel_ids(self): 88 | return FakeUsage.kernel_ids 89 | 90 | def get_kernel(self, kernel_id): 91 | if kernel_id == FakeUsage.kernel_ids[0]: 92 | pid = FakeUsage.pids[0] 93 | elif kernel_id == FakeUsage.kernel_ids[1]: 94 | pid = FakeUsage.pids[1] 95 | else: 96 | raise KeyError(kernel_id) 97 | return FakeKernelManager._kernel_factory(FakeKernelManager._popen(pid)) 98 | 99 | 100 | class ColabResourcesHandlerTest(testing.AsyncHTTPTestCase): 101 | """Tests for ChunkedFileDownloadHandler.""" 102 | 103 | def get_app(self): 104 | """Setup code required by testing.AsyncHTTP[S]TestCase.""" 105 | settings = { 106 | 'base_url': '/', 107 | # The underyling ipaddress library sometimes doesn't think that 108 | # 127.0.0.1 is a proper loopback device. 109 | 'local_hostnames': ['127.0.0.1'], 110 | 'kernel_manager': FakeKernelManager(), 111 | } 112 | app = web.Application([], **settings) 113 | nb_server_app = FakeNotebookServer(app) 114 | _serverextension.load_jupyter_server_extension(nb_server_app) 115 | return app 116 | 117 | @mock.patch.object( 118 | subprocess, 119 | 'check_output', 120 | # Use canned ps output. 121 | side_effect=FakeUsage.fake_check_output, 122 | ) 123 | def testColabResources(self, mock_check_output): 124 | response = self.fetch('/api/colab/resources') 125 | self.assertEqual(response.code, 200) 126 | # Body is a JSON response. 127 | json_response = escape.json_decode( 128 | response.body[len(_handlers._XSSI_PREFIX):]) # pylint: disable=protected-access 129 | self.assertGreater(json_response['ram']['limit'], 0) 130 | 131 | @mock.patch.object( 132 | subprocess, 133 | 'check_output', 134 | # Use canned ps output. 135 | side_effect=FakeUsage.fake_check_output, 136 | ) 137 | def testColabResourcesFakeRam(self, mock_check_output): 138 | response = self.fetch('/api/colab/resources') 139 | self.assertEqual(response.code, 200) 140 | # Body is a JSON response. 141 | json_response = escape.json_decode( 142 | response.body[len(_handlers._XSSI_PREFIX):]) # pylint: disable=protected-access 143 | self.assertEqual(json_response['ram']['kernels'][FakeUsage.kernel_ids[0]], 144 | FakeUsage.usage[0] * 1024) 145 | self.assertEqual(json_response['ram']['kernels'][FakeUsage.kernel_ids[1]], 146 | FakeUsage.usage[1] * 1024) 147 | 148 | @mock.patch.object( 149 | spawn, 150 | 'find_executable', 151 | # Pretend there is nvidia-smi. 152 | return_value=True) 153 | @mock.patch.object( 154 | subprocess, 155 | 'check_output', 156 | # Use canned ps and nvidia-smi output. 157 | side_effect=FakeUsage.fake_check_output, 158 | ) 159 | def testColabResourcesFakeGPU(self, mock_check_output, mock_find_executable): 160 | response = self.fetch('/api/colab/resources') 161 | self.assertEqual(response.code, 200) 162 | # Body is a JSON response. 163 | json_response = escape.json_decode( 164 | response.body[len(_handlers._XSSI_PREFIX):]) # pylint: disable=protected-access 165 | self.assertEqual(json_response['gpu']['usage'], 166 | FakeUsage.gpu_usage['usage'] * 1024 * 1024) 167 | self.assertEqual(json_response['gpu']['limit'], 168 | FakeUsage.gpu_usage['limit'] * 1024 * 1024) 169 | 170 | @mock.patch.object( 171 | psutil, 172 | 'disk_usage', 173 | # Return fake usage data. 174 | return_value=FakeUsage.disk_usage) 175 | @mock.patch.object( 176 | subprocess, 177 | 'check_output', 178 | # Use canned ps output. 179 | side_effect=FakeUsage.fake_check_output, 180 | ) 181 | def testColabResourcesFakeDisk(self, mock_check_output, mock_disk_usage): 182 | response = self.fetch('/api/colab/resources') 183 | self.assertEqual(response.code, 200) 184 | # Body is a JSON response. 185 | json_response = escape.json_decode( 186 | response.body[len(_handlers._XSSI_PREFIX):]) # pylint: disable=protected-access 187 | self.assertEqual(json_response['disk']['usage'], FakeUsage.disk_usage.used) 188 | self.assertEqual(json_response['disk']['limit'], FakeUsage.disk_usage.total) 189 | -------------------------------------------------------------------------------- /google/colab/_shell_customizations.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colab-specific shell customizations.""" 15 | 16 | import re 17 | import textwrap 18 | 19 | from IPython.utils import coloransi 20 | from google.colab import _ipython as ipython 21 | 22 | 23 | _GREEN = coloransi.TermColors.Green 24 | _RED = coloransi.TermColors.Red 25 | _NORMAL = coloransi.TermColors.Normal 26 | _SEP = _RED + '-' * 75 27 | 28 | # Set of modules that have snippets explaining how they can be installed. Any 29 | # ImportErrors for modules in this set will show a custom error message pointing 30 | # to the snippet. 31 | SNIPPET_MODULES = set([ 32 | 'cartopy', 33 | 'libarchive', 34 | 'pydot', 35 | 'torch', 36 | ]) 37 | 38 | 39 | def initialize(): 40 | ip = ipython.get_ipython() 41 | if ip: 42 | _CustomErrorHandlers(ip) 43 | 44 | 45 | class ColabTraceback(object): 46 | 47 | def __init__(self, stb, error_details): 48 | self.stb = stb 49 | self.error_details = error_details 50 | 51 | 52 | class FormattedTracebackError(Exception): 53 | 54 | def __init__(self, message, stb, details): 55 | super(FormattedTracebackError, self).__init__(message) 56 | self._colab_traceback = ColabTraceback(stb, details) 57 | 58 | def _render_traceback_(self): 59 | return self._colab_traceback 60 | 61 | 62 | class _CustomErrorHandlers(object): 63 | """Custom error handler for the IPython shell. 64 | 65 | Allows us to add custom messaging for certain error types (i.e. ImportError). 66 | """ 67 | 68 | def __init__(self, shell): 69 | # The values for this map are functions which return 70 | # (custom_message, additional error details). 71 | self.custom_error_handlers = { 72 | ImportError: _CustomErrorHandlers.import_message, 73 | } 74 | shell.set_custom_exc( 75 | tuple(self.custom_error_handlers.keys()), self.handle_error) 76 | 77 | def _get_error_handler(self, etype): 78 | for handled_type in self.custom_error_handlers: 79 | if issubclass(etype, handled_type): 80 | return self.custom_error_handlers[handled_type] 81 | return None 82 | 83 | def handle_error(self, shell, etype, exception, tb, tb_offset=None): 84 | """Invoked when the shell catches an error in custom_message_getters.""" 85 | handler = self._get_error_handler(etype) 86 | if not handler: 87 | return shell.showtraceback() 88 | 89 | result = handler(exception) 90 | if result: 91 | custom_message, details = result 92 | structured_traceback = shell.InteractiveTB.structured_traceback( 93 | etype, exception, tb, tb_offset=tb_offset) 94 | # Ensure a blank line appears between the standard traceback and custom 95 | # error messaging. 96 | structured_traceback += ['', custom_message] 97 | wrapped = FormattedTracebackError( 98 | str(exception), structured_traceback, details) 99 | return shell.showtraceback(exc_tuple=(etype, wrapped, tb)) 100 | 101 | @staticmethod 102 | def import_message(error): 103 | """Return a helpful message for failed imports.""" 104 | # Python 3 ModuleNotFoundErrors have a "name" attribute. Preferring this 105 | # over regex matching if the attribute is available. 106 | module_name = getattr(error, 'name', None) 107 | if not module_name: 108 | match = re.search(r'No module named \'?(?P[a-zA-Z0-9_\.]+)\'?', 109 | str(error)) 110 | module_name = match.groupdict()['name'].split('.')[0] if match else None 111 | 112 | if module_name in SNIPPET_MODULES: 113 | msg = textwrap.dedent("""\ 114 | {sep}{green} 115 | NOTE: If your import is failing due to a missing package, you can 116 | manually install dependencies using either !pip or !apt. 117 | 118 | To install {snippet}, click the button below. 119 | {sep}{normal}\n""".format( 120 | sep=_SEP, green=_GREEN, normal=_NORMAL, snippet=module_name)) 121 | details = { 122 | 'actions': [ 123 | { 124 | 'action': 'open_snippet', 125 | 'action_text': 'Install {}'.format(module_name), 126 | # Snippets for installing a custom library always end with 127 | # an import of the library itself. 128 | 'snippet_filter': 'import {}'.format(module_name), 129 | }, 130 | ], 131 | } 132 | return msg, details 133 | 134 | msg = textwrap.dedent("""\ 135 | {sep}{green} 136 | NOTE: If your import is failing due to a missing package, you can 137 | manually install dependencies using either !pip or !apt. 138 | 139 | To view examples of installing some common dependencies, click the 140 | "Open Examples" button below. 141 | {sep}{normal}\n""".format(sep=_SEP, green=_GREEN, normal=_NORMAL)) 142 | 143 | details = { 144 | 'actions': [{ 145 | 'action': 'open_url', 146 | 'action_text': 'Open Examples', 147 | 'url': '/notebooks/snippets/importing_libraries.ipynb', 148 | },], 149 | } 150 | return msg, details 151 | 152 | 153 | def compute_completion_metadata(shell, matches): 154 | """Computes completion item metadata. 155 | 156 | Args: 157 | shell: IPython shell 158 | matches: List of string completion matches. 159 | 160 | Returns: 161 | Metadata for each of the matches. 162 | """ 163 | 164 | # We want to temporarily change the default level of detail returned by the 165 | # inspector, to avoid slow completions (cf b/112153563). 166 | old_str_detail_level = shell.inspector.str_detail_level 167 | shell.inspector.str_detail_level = 1 168 | try: 169 | infos = [] 170 | for match in matches: 171 | info = {} 172 | if '#' in match: 173 | # Runtime type information added by customization._add_type_information. 174 | info['type_name'] = match.split('#')[1] 175 | else: 176 | inspect_results = shell.object_inspect(match) 177 | # Use object_inspect to find the type and filter to only what is needed 178 | # since there can be a lot of completions to send. 179 | info['type_name'] = inspect_results['type_name'] 180 | if inspect_results['definition']: 181 | info['definition'] = inspect_results['definition'] 182 | elif inspect_results['init_definition']: 183 | info['definition'] = inspect_results['init_definition'] 184 | infos.append(info) 185 | return infos 186 | finally: 187 | shell.inspector.str_detail_level = old_str_detail_level 188 | -------------------------------------------------------------------------------- /notebooks/colab-github-demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "colab_type": "text", 7 | "id": "view-in-github" 8 | }, 9 | "source": [ 10 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/googlecolab/colabtools/blob/master/notebooks/colab-github-demo.ipynb)\n" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "colab_type": "text", 17 | "id": "-pVhOfzLx9us" 18 | }, 19 | "source": [ 20 | "# Using Google Colab with GitHub\n", 21 | "\n" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "metadata": { 27 | "colab_type": "text", 28 | "id": "wKJ4bd5rt1wy" 29 | }, 30 | "source": [ 31 | "\n", 32 | "[Google Colaboratory](http://colab.research.google.com) is designed to integrate cleanly with GitHub, allowing both loading notebooks from github and saving notebooks to github." 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": { 38 | "colab_type": "text", 39 | "id": "K-NVg7RjyeTk" 40 | }, 41 | "source": [ 42 | "## Loading Public Notebooks Directly from GitHub\n", 43 | "\n", 44 | "Colab can load public github notebooks directly, with no required authorization step.\n", 45 | "\n", 46 | "For example, consider the notebook at this address: https://github.com/googlecolab/colabtools/blob/master/notebooks/colab-github-demo.ipynb.\n", 47 | "\n", 48 | "The direct colab link to this notebook is: https://colab.research.google.com/github/googlecolab/colabtools/blob/master/notebooks/colab-github-demo.ipynb." 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": { 54 | "colab_type": "text", 55 | "id": "WzIRIt9d2huC" 56 | }, 57 | "source": [ 58 | "## Browsing GitHub Repositories from Colab\n", 59 | "\n", 60 | "Colab also supports special URLs that link directly to a GitHub browser for any user/organization, repository, or branch. For example:\n", 61 | "\n", 62 | "- http://colab.research.google.com/github will give you a general github browser, where you can search for any github organization or username.\n", 63 | "- http://colab.research.google.com/github/googlecolab/ will open the repository browser for the ``googlecolab`` organization. Replace ``googlecolab`` with any other github org or user to see their repositories.\n", 64 | "- http://colab.research.google.com/github/googlecolab/colabtools/ will flet you browse the main branch of the ``colabtools`` repository within the ``googlecolab`` organization. Substitute any user/org and repository to see its contents.\n", 65 | "- http://colab.research.google.com/github/googlecolab/colabtools/blob/master will let you browse ``master`` branch of the ``colabtools`` repository within the ``googlecolab`` organization. (don't forget the ``blob`` here!) You can specify any valid branch for any valid repository." 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": { 71 | "colab_type": "text", 72 | "id": "Rmai0dD30XzL" 73 | }, 74 | "source": [ 75 | "## Loading Private Notebooks\n", 76 | "\n", 77 | "Loading a notebook from a private GitHub repository is possible, but requires an additional step to allow Colab to access your files.\n", 78 | "Do the following:\n", 79 | "\n", 80 | "1. Navigate to http://colab.research.google.com/github.\n", 81 | "2. Click the \"Include Private Repos\" checkbox.\n", 82 | "3. In the popup window, sign-in to your Github account and authorize Colab to read the private files.\n", 83 | "4. Your private repositories and notebooks will now be available via the github navigation pane." 84 | ] 85 | }, 86 | { 87 | "cell_type": "markdown", 88 | "metadata": { 89 | "colab_type": "text", 90 | "id": "8J3NBxtZpPcK" 91 | }, 92 | "source": [ 93 | "## Saving Notebooks To GitHub or Drive\n", 94 | "\n", 95 | "Any time you open a GitHub hosted notebook in Colab, it opens a new editable view of the notebook. You can run and modify the notebook without worrying about overwriting the source.\n", 96 | "\n", 97 | "If you would like to save your changes from within Colab, you can use the File menu to save the modified notebook either to Google Drive or back to GitHub. Choose **File→Save a copy in Drive** or **File→Save a copy to GitHub** and follow the resulting prompts. To save a Colab notebook to GitHub requires giving Colab permission to push the commit to your repository." 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "metadata": { 103 | "colab_type": "text", 104 | "id": "8QAWNjizy_3O" 105 | }, 106 | "source": [ 107 | "## Open In Colab Badge\n", 108 | "\n", 109 | "Anybody can open a copy of any github-hosted notebook within Colab. To make it easier to give people access to live views of GitHub-hosted notebooks,\n", 110 | "colab provides a [shields.io](http://shields.io/)-style badge, which appears as follows:\n", 111 | "\n", 112 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/googlecolab/colabtools/blob/master/notebooks/colab-github-demo.ipynb)\n", 113 | "\n", 114 | "The markdown for the above badge is the following:\n", 115 | "\n", 116 | "```markdown\n", 117 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/googlecolab/colabtools/blob/master/notebooks/colab-github-demo.ipynb)\n", 118 | "```\n", 119 | "\n", 120 | "The HTML equivalent is:\n", 121 | "\n", 122 | "```HTML\n", 123 | "\u003ca href=\"https://colab.research.google.com/github/googlecolab/colabtools/blob/master/notebooks/colab-github-demo.ipynb\"\u003e\n", 124 | " \u003cimg src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/\u003e\n", 125 | "\u003c/a\u003e\n", 126 | "```\n", 127 | "\n", 128 | "Remember to replace the notebook URL in this template with the notebook you want to link to." 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 0, 134 | "metadata": { 135 | "colab": {}, 136 | "colab_type": "code", 137 | "id": "3VQqVi-3ScBC" 138 | }, 139 | "outputs": [], 140 | "source": [ 141 | "" 142 | ] 143 | } 144 | ], 145 | "metadata": { 146 | "colab": { 147 | "collapsed_sections": [], 148 | "name": "colab-github-demo.ipynb", 149 | "provenance": [], 150 | "version": "0.3.2" 151 | }, 152 | "kernelspec": { 153 | "display_name": "Python 3", 154 | "name": "python3" 155 | } 156 | }, 157 | "nbformat": 4, 158 | "nbformat_minor": 0 159 | } 160 | -------------------------------------------------------------------------------- /google/colab/html/_provide.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Helper to provide resources via the colab service worker. 16 | 17 | Example inside Colab: 18 | 19 | # First, create a resource in Python 20 | from google.colab import html 21 | ref = html.create_resource(content='hello world') 22 | 23 | # Next, execute javascript code that uses that resource 24 | from IPython.display import display, Javascript 25 | 26 | display(Javascript(''' 27 | var content = {ref.content!r}; 28 | var url = {ref.url!r}; 29 | 30 | fetch(url) 31 | .then(r => r.text()) 32 | .then(c => console.log("Content matches:", c === content)) 33 | '''.format(ref=ref))) 34 | """ 35 | 36 | from __future__ import absolute_import 37 | from __future__ import division 38 | from __future__ import print_function 39 | 40 | import abc 41 | import collections 42 | import mimetypes 43 | import uuid 44 | import weakref 45 | 46 | import six 47 | import tornado.web 48 | import tornado.wsgi 49 | 50 | from google.colab.html import _background_server 51 | 52 | 53 | @six.add_metaclass(abc.ABCMeta) 54 | class _Resource(object): 55 | """Abstract resource class to handle content to colab.""" 56 | 57 | def __init__(self, provider, headers, extension, route): 58 | if not isinstance(headers, collections.Mapping): 59 | raise ValueError('headers must be a dict') 60 | if route and extension: 61 | raise ValueError('Should only provide one of route or extension.') 62 | self.headers = headers 63 | self._route = route 64 | if route: 65 | self._guid = route 66 | else: 67 | self._guid = str(uuid.uuid4()) 68 | if extension: 69 | self._guid += '.' + extension 70 | self._provider = provider 71 | 72 | @abc.abstractmethod 73 | def get(self, handler): 74 | """Gets the resource using the tornado handler passed in. 75 | 76 | Args: 77 | handler: Tornado handler to be used. 78 | """ 79 | for key, value in self.headers.items(): 80 | handler.set_header(key, value) 81 | 82 | @property 83 | def guid(self): 84 | """Unique id used to serve and reference the resource.""" 85 | return self._guid 86 | 87 | @property 88 | def url(self): 89 | """Url to fetch the resource at.""" 90 | return 'https://localhost:{}/{}'.format(self._provider.port, self._guid) 91 | 92 | 93 | class _ContentResource(_Resource): 94 | 95 | def __init__(self, content, *args, **kwargs): 96 | self.content = content 97 | super(_ContentResource, self).__init__(*args, **kwargs) 98 | 99 | def get(self, handler): 100 | super(_ContentResource, self).get(handler) 101 | handler.write(self.content) 102 | 103 | 104 | class _FileResource(_Resource): 105 | """Handle file resources.""" 106 | 107 | def __init__(self, filepath, *args, **kwargs): 108 | self.filepath = filepath 109 | super(_FileResource, self).__init__(*args, **kwargs) 110 | 111 | def get(self, handler): 112 | super(_FileResource, self).get(handler) 113 | with open(self.filepath) as f: 114 | data = f.read() 115 | handler.write(data) 116 | 117 | 118 | class _HandlerResource(_Resource): 119 | 120 | def __init__(self, func, *args, **kwargs): 121 | self.func = func 122 | super(_HandlerResource, self).__init__(*args, **kwargs) 123 | 124 | def get(self, handler): 125 | super(_HandlerResource, self).get(handler) 126 | content = self.func() 127 | handler.write(content) 128 | 129 | 130 | class _Provider(_background_server._WsgiServer): # pylint: disable=protected-access 131 | """Background server which can provide a set of resources.""" 132 | 133 | def __init__(self): 134 | """Initialize the server with a ResourceHandler script.""" 135 | resources = weakref.WeakValueDictionary() 136 | self._resources = resources 137 | 138 | class ResourceHandler(tornado.web.RequestHandler): 139 | """Serves the `_Resource` objects.""" 140 | 141 | def get(self): 142 | path = self.request.path 143 | resource = resources.get(path.lstrip('/')) 144 | if not resource: 145 | self.set_status(404) 146 | return 147 | content_type, _ = mimetypes.guess_type(path) 148 | if content_type: 149 | self.set_header('Content-Type', content_type) 150 | resource.get(self) 151 | 152 | app = tornado.wsgi.WSGIApplication([ 153 | (r'.*', ResourceHandler), 154 | ]) 155 | 156 | super(_Provider, self).__init__(app) 157 | 158 | def create(self, 159 | content=None, 160 | filepath=None, 161 | handler=None, 162 | headers=None, 163 | extension=None, 164 | route=None): 165 | """Creates and provides a new resource to be served. 166 | 167 | Can only provide one of content, path, or handler. 168 | 169 | Args: 170 | content: The string or byte content to return. 171 | filepath: The filepath to a file whose contents should be returned. 172 | handler: A function which will be executed and returned on each request. 173 | headers: A dict of header values to return. 174 | extension: Optional extension to add to the url. 175 | route: Optional route to serve on. 176 | Returns: 177 | The the `_Resource` object which will be served and will provide its url. 178 | Raises: 179 | ValueError: If you don't provide one of content, filepath, or handler. 180 | """ 181 | sources = sum(map(bool, (content, filepath, handler))) 182 | if sources != 1: 183 | raise ValueError( 184 | 'Must provide exactly one of content, filepath, or handler') 185 | 186 | if not headers: 187 | headers = {} 188 | 189 | if route: 190 | route = route.lstrip('/') 191 | 192 | if content: 193 | resource = _ContentResource( 194 | content, 195 | headers=headers, 196 | extension=extension, 197 | provider=self, 198 | route=route) 199 | elif filepath: 200 | resource = _FileResource( 201 | filepath, 202 | headers=headers, 203 | extension=extension, 204 | provider=self, 205 | route=route) 206 | elif handler: 207 | resource = _HandlerResource( 208 | handler, 209 | headers=headers, 210 | extension=extension, 211 | provider=self, 212 | route=route) 213 | else: 214 | raise ValueError('Must provide one of content, filepath, or handler.') 215 | 216 | self._resources[resource.guid] = resource 217 | self.start() 218 | return resource 219 | 220 | 221 | # Global singleton instance 222 | _global_provider = _Provider() 223 | 224 | create = _global_provider.create 225 | -------------------------------------------------------------------------------- /google/colab/_autocomplete/_inference.py: -------------------------------------------------------------------------------- 1 | """Provides support for type inference during autocompletion. 2 | 3 | Provides decorators, that can be used to enable autocomplete on results 4 | of function calls. See below for annotations that you can use to mark 5 | your functions safe to eval, and/or return specific types, etc. 6 | """ 7 | import ast 8 | import functools 9 | import inspect 10 | import math 11 | import tempfile 12 | import types 13 | 14 | 15 | class _FuncTransformer(ast.NodeTransformer): 16 | """Replaces all function calls x(y) with _infertype(x, y).""" 17 | 18 | # Overrides function of ast.NodeTransformer 19 | # pylint: disable=invalid-name 20 | def visit_Call(self, node): 21 | """Overrides calls in ast.""" 22 | func = node.func 23 | node.func = ast.Name( 24 | id='_autocomplete_infertype', 25 | lineno=node.lineno, 26 | col_offset=node.col_offset, 27 | ctx=node.func.ctx) 28 | node.args[:0] = [func] 29 | return self.generic_visit(node) 30 | 31 | 32 | _BuiltinDescriptorType = type(str.split) 33 | 34 | # Maps type(f) to unique descriptor describing given function. 35 | # we want all foo.bar() to map to the same function for all foo of the same 36 | # type. 37 | _PROTOTYPES = { 38 | # buitlin bound instance method like ''.str, also matches global 39 | # functions, in which case f.__self__ is None. 40 | types.BuiltinMethodType: 41 | lambda f: (type(f.__self__), f.__name__), 42 | 43 | # either unbound or bound method like foo.bar(), we don't need im_class 44 | # since im_func is actually unique. This is contrast with builtins where 45 | # we can only get their string name. 46 | types.MethodType: 47 | lambda f: (f.im_func), 48 | 49 | # str.split() 50 | _BuiltinDescriptorType: 51 | lambda f: (f.__objclass__, f.__name__), 52 | 53 | # Lambda and regular functions return self for indexing 54 | types.LambdaType: (lambda f: f), 55 | types.FunctionType: (lambda f: f), 56 | } 57 | 58 | _type_map = {} 59 | _safe_to_eval_classes = set() 60 | 61 | # Contains the last exception error during autocompletion 62 | _last_autocompletion_error = None 63 | 64 | 65 | def _get_prototype(func): 66 | """Returns a prototype of a function. 67 | 68 | This function returns essentially a normalized version of the function that 69 | is independent of attached instance and only depends on the actual code 70 | that will be executed when this function is called. 71 | 72 | Args: 73 | func: a callable 74 | 75 | Returns: 76 | A signature that is the same for all functions that are essentially the 77 | same. 78 | 79 | """ 80 | t = type(func) 81 | if t in _PROTOTYPES: 82 | return _PROTOTYPES[t](func) 83 | 84 | # Note: if we are here, we know that func is actually not a function 85 | # but it could be either a class (e.g. this is a constructor call) 86 | # or an instance. (E.g. this is an instance w/defined __call__) 87 | # 88 | # If object has __call__ and not a class (e.g. instance), 89 | # it is indexed by class/__call__ 90 | if callable(func) and not inspect.isclass(func): 91 | return type(func), func.__call__ 92 | 93 | return func 94 | 95 | 96 | def _infertype(*args, **kwargs): 97 | func = args[0] 98 | prototype = _get_prototype(func) 99 | inferrer = _type_map.get(prototype, None) 100 | if inferrer: 101 | return inferrer(func, *args[1:], **kwargs) 102 | else: 103 | return None 104 | 105 | 106 | def safe_to_eval(func): 107 | """Indicates that 'func' is safe to eval during autocomplete. 108 | 109 | Can be used as annotation on arbitrary functions, e.g. 110 | 111 | @autocomplete.safe_to_eval 112 | def pure_func(x, y): 113 | return x + y 114 | 115 | Args: 116 | func: function 117 | 118 | Returns: 119 | func. 120 | 121 | """ 122 | _type_map[_get_prototype(func)] = ( 123 | lambda f, *args, **kwargs: f(*args, **kwargs)) 124 | return func 125 | 126 | 127 | def is_class_safe_to_eval(t): 128 | """Returns whether type t was previously marked as safe_to_eval.""" 129 | return t in _safe_to_eval_classes 130 | 131 | 132 | def _annotate_with_type_inferrer(type_inferrer, func): 133 | """Registers type_inferrer to be used to infer the result type of func. 134 | 135 | See use_inferrer for public API. 136 | 137 | Args: 138 | type_inferrer: function that takes as an argument a func and all arguments 139 | func: function that will be applied to. 140 | Returns: 141 | func 142 | """ 143 | _type_map[_get_prototype(func)] = type_inferrer 144 | return func 145 | 146 | 147 | def use_inferrer(inferrer): 148 | """Allows to annotate given function with return type inferrer. 149 | 150 | Example: 151 | def inferrer(f, arg): 152 | '''Does result inference for complex_function depending on arg''' 153 | return '' if arg is None else arg 154 | 155 | @use_inferrer(inferrer) 156 | def complex_function(arg): 157 | ... do something complex 158 | 159 | For the purposes of autocompletion, complex_function(arg) will be assumed 160 | to have type 'string' if arg is None, and arg, if it is not. 161 | 162 | This is the most generic way of annotating a function, it can use 163 | both the function and the arguments that were passed in to compute what 164 | the resulting type/object should be. 165 | 166 | Args: 167 | inferrer: is a function that takes function as a first argument 168 | followed by args/kwargs and returns the desired mock return instance that 169 | will be used for autocompletion 170 | 171 | Returns: 172 | a decorator that can be applied to any function. 173 | """ 174 | return functools.partial(_annotate_with_type_inferrer, inferrer) 175 | 176 | 177 | def returns_instance(instance): 178 | """Indicates that given function always returns instance. 179 | 180 | Use as annotation e.g. 181 | 182 | @returns_instance('') 183 | def this_method_returns_string(): LoadStringFromBigtable(...) 184 | 185 | Args: 186 | instance: instance to return 187 | 188 | Returns: 189 | Annotation function. 190 | """ 191 | return use_inferrer(lambda f, *argv, **kwargs: instance) 192 | 193 | 194 | def returns_arg(arg): 195 | """Annotates function to return argument for the purposes of autocomplete.""" 196 | return use_inferrer(lambda f, *argv, **kwargs: argv[arg]) 197 | 198 | 199 | def returns_kwarg(arg): 200 | """Annotates function to return kwarg for the purposes of autocomplete.""" 201 | return use_inferrer(lambda f, *argv, **kwargs: kwargs[arg]) 202 | 203 | 204 | def infer_expression_result(expr, global_ns, local_ns): 205 | """Returns the result of the expression, or None if not successful. 206 | 207 | Functions are not evaluated and instead infertype is used. 208 | 209 | Args: 210 | expr: string containing expression 211 | global_ns: global namespace 212 | local_ns: local namespace 213 | 214 | Returns: 215 | the result or None 216 | """ 217 | tree = ast.parse(expr, mode='eval') 218 | _FuncTransformer().visit(tree) 219 | local_ns = dict(local_ns) if local_ns else {} 220 | local_ns['_autocomplete_infertype'] = _infertype 221 | 222 | try: 223 | return eval( # pylint: disable=eval-used 224 | compile(tree, '', 'eval'), global_ns, local_ns) 225 | except Exception as e: # pylint: disable=broad-except 226 | global _last_autocompletion_error 227 | _last_autocompletion_error = e 228 | 229 | return None 230 | 231 | 232 | def mark_class_safe_to_eval(m): 233 | _safe_to_eval_classes.add(m) 234 | for each in dir(m): 235 | if each.startswith('_'): 236 | continue 237 | safe_to_eval(getattr(m, each)) 238 | 239 | 240 | def annotate_builtins(): 241 | mark_class_safe_to_eval(str) 242 | mark_class_safe_to_eval(unicode) 243 | mark_class_safe_to_eval(math) 244 | returns_instance(tempfile.mktemp())(open) 245 | -------------------------------------------------------------------------------- /google/colab/widgets/_grid.py: -------------------------------------------------------------------------------- 1 | """Grid widget - allows to create a table, where each cell be printed into.""" 2 | import contextlib 3 | 4 | from IPython import display 5 | from google.colab.output import _publish 6 | 7 | from google.colab.output import _util 8 | from google.colab.widgets import _widget 9 | 10 | 11 | class Grid(_widget.OutputAreaWidget): 12 | """Grid widget allows organing outputs into NxM of individual cells. 13 | 14 | Each cell in this grid can be populated independently using standard 15 | ipython output functionality (such as print/display.HTML/matplotlib) 16 | 17 | Example: 18 | 19 | t = Grid(3, 4) 20 | with t.output_to(0, 0): 21 | print(1) # will print into cell (0, 0) 22 | 23 | t.clear_cell(0, 0) # clears cell 0, 0 24 | 25 | with t.output_to(0, 1): 26 | print(1) # will print into cell (0, 1) 27 | 28 | with t.output_to(1,1): 29 | print(2) 30 | pylab.plot([1,2,3]) 31 | 32 | etc... 33 | """ 34 | 35 | def __init__(self, 36 | rows, 37 | columns, 38 | header_row=False, 39 | header_column=False, 40 | style=''): 41 | """Creates a new grid object. 42 | 43 | Args: 44 | rows: number of rows 45 | columns: number of columns 46 | header_row: if true will include header row (th) 47 | header_column: if true will include header column. 48 | style: a css string containing style for this grid. 49 | """ 50 | self.rows = rows 51 | self.columns = columns 52 | self.header_row = header_row 53 | self.header_column = header_column 54 | self._id = _util.get_locally_unique_id() 55 | self._style = style 56 | super(Grid, self).__init__() 57 | 58 | def clear_cell(self, row=None, col=None): 59 | """Clears given cell. If row/col are None clears active cell.""" 60 | if row is not None: 61 | if row < 0 or row >= self.rows: 62 | raise ValueError('%d is not a valid row' % row) 63 | if col < 0 or col >= self.columns: 64 | raise ValueError('%d is not a valid column' % col) 65 | cellid = self._get_cell_id(row, col) 66 | else: 67 | cellid = None 68 | self._clear_component(cellid) 69 | 70 | def _get_cell_id(self, row, col): 71 | return '%s-%s-%s' % (self._id, row, col) 72 | 73 | def __iter__(self): 74 | if not self._published: 75 | self._publish() 76 | for i in range(self.rows): 77 | for j in range(self.columns): 78 | with self.output_to(i, j): 79 | yield (i, j) 80 | 81 | def _populate(self, row_data, col_data, render, header_render=None): 82 | """Populate the grid with a cross product of row and cols.""" 83 | 84 | def display_one_cell(render_func, *args): 85 | result = render_func(*args) 86 | if result is not None: 87 | display.display(result) 88 | 89 | header_render = header_render or display.display 90 | rows = list(row_data) 91 | cols = list(col_data) 92 | row_offset = 1 if self.header_row else 0 93 | col_offset = 1 if self.header_column else 0 94 | 95 | if (row_offset + len(rows) > self.rows or 96 | col_offset + len(cols) > self.columns): 97 | raise _widget.WidgetException('Can not fit %dx%d data into %dx%d grid. ' % 98 | (len(rows), len(cols), self.rows, 99 | self.columns)) 100 | for row, col in iter(self): 101 | row -= row_offset 102 | col -= col_offset 103 | if row >= len(rows): 104 | continue 105 | if col >= len(cols): 106 | continue 107 | 108 | if row < 0 and col < 0: 109 | continue 110 | if row < 0: 111 | display_one_cell(header_render, cols[col]) 112 | continue 113 | if col < 0: 114 | display_one_cell(header_render, rows[row]) 115 | continue 116 | display_one_cell(render, rows[row], cols[col]) 117 | return self 118 | 119 | def _html_repr(self): 120 | """Returns html representation of this grid.""" 121 | html = '' % (self._id,) 122 | 123 | for row in range(self.rows): 124 | html += '' 125 | for col in range(self.columns): 126 | if row == 0 and self.header_row or col == 0 and self.header_column: 127 | tag = 'th' 128 | else: 129 | tag = 'td' 130 | html += '<%(tag)s id=%(id)s>' % { 131 | 'tag': tag, 132 | 'id': self._get_cell_id(row, col) 133 | } 134 | html += '' 135 | html += '
' 136 | return html 137 | 138 | def _publish(self): 139 | """Publishes the grid. 140 | 141 | Grid will publish automatically on first call to output_to. 142 | """ 143 | 144 | if self._published: 145 | return 146 | super(Grid, self)._publish() 147 | with self._output_in_widget(): 148 | _publish.css(""" 149 | table#%(id)s, #%(id)s > tbody > tr > th, #%(id)s > tbody > tr > td { 150 | border: 1px solid lightgray; 151 | border-collapse:collapse; 152 | %(userstyle)s 153 | }""" % { 154 | 'id': self._id, 155 | 'userstyle': self._style 156 | }) 157 | 158 | _publish.html(self._html_repr()) 159 | 160 | @contextlib.contextmanager 161 | def output_to(self, row, column): 162 | """Redirects output to the corresponding cell of this grid. 163 | 164 | Args: 165 | row: 0 based row 166 | column: 0 based column 167 | 168 | Yields: 169 | nothing 170 | """ 171 | if row < 0 or column < 0 or row >= self.rows or column >= self.columns: 172 | raise _widget.WidgetException( 173 | 'Cell (%d, %d) is outside of boundaries of %dx%d grid' % 174 | (row, column, self.rows, self.columns)) 175 | component_id = self._get_cell_id(row, column) 176 | with self._active_component(component_id): 177 | yield 178 | 179 | 180 | def create_grid(row_data, 181 | col_data, 182 | render, 183 | header_render=None, 184 | header_row=True, 185 | header_column=True): 186 | """Creates Grid using cross product of rows and cols. 187 | 188 | 189 | This function populates the grid using row_data and col_data. 190 | In addition, if header_row/header_col are truthy it renders row[i] in 1st 191 | column and i-th row, as a header for row i, and, and col[i] in 1-st row 192 | and column j. 193 | 194 | Examples: 195 | 196 | Distance matrix between two vectors use 197 | create_grid(x, y, render=lambda x, y: np.linalg.norm(x-y)) 198 | 199 | To render row-based data you can do: 200 | create_grid(data, range(10), render=lambda x, y: x[y])) 201 | 202 | For column based: 203 | create_grid(range(10), columns, render=lambda x, y: y[x])) 204 | 205 | Args: 206 | row_data: an iterable returning a generating element for each row 207 | col_data: an iterable returning a generating element for each col 208 | render: element display function. It should accept two arguments 209 | (corresponding row and column element). 210 | This function can produce output using either or both of two ways. 211 | 1. It can use print statement, or any other display functions - 212 | such as pylab.show(), or display_html. 213 | 2. It can return not None object to be rendered via IPython.display. 214 | This will produce repr(..) for vanilla python object and rich outputs 215 | for things like display.html. 216 | 217 | header_render: header display function that accepts one element to 218 | display. If no header rendering function is provided, each header 219 | is displayed using Python.display() function. 220 | header_row: if True the header row will be created 221 | header_column: if True the header column will be created. 222 | 223 | Raises: 224 | OutputWidgetException: if the grid size is not enough to 225 | render row/cols with their headers as requested. 226 | 227 | Returns: 228 | Filled grid for chaining. 229 | """ 230 | rows = list(row_data) 231 | cols = list(col_data) 232 | t = Grid( 233 | len(rows) + header_row, 234 | len(cols) + header_column, 235 | header_row=header_row, 236 | header_column=header_column) 237 | # pylint: disable=protected-access 238 | t._populate(rows, cols, render, header_render) 239 | return t 240 | -------------------------------------------------------------------------------- /google/colab/html/_html.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """HTML renderable element in notebooks.""" 17 | 18 | import base64 19 | import json 20 | import uuid 21 | import IPython 22 | import pystache 23 | 24 | import six 25 | 26 | from google.colab import output 27 | from google.colab.html import _resources 28 | 29 | _MSG_CHUNK_SIZE = 1 * 1024 * 1024 30 | 31 | 32 | def _to_html_str(obj): 33 | """Renders an object as html string on a best effort basis. 34 | 35 | IPython allows for registering of formatters. This 36 | tries to format the object using that registered text/html 37 | formater method. If it cannot and it is a string it returns 38 | it unchanged, otherwise it tries to serialize to json. 39 | The result should be something that can be html output 40 | for the notebook outputcell. 41 | 42 | Args: 43 | obj: An object to try to convert into HTML. 44 | Returns: 45 | An html string representation of the object. 46 | """ 47 | ip = IPython.get_ipython() 48 | formatter = ip.display_formatter.formatters['text/html'] 49 | try: 50 | render = formatter.lookup(obj) 51 | return render(obj) 52 | except KeyError: # No html formatter exists 53 | pass 54 | if hasattr(obj, '_repr_html_'): 55 | html = obj._repr_html_() # pylint: disable=protected-access 56 | if html: 57 | return html 58 | elif isinstance(obj, six.string_types): 59 | return obj 60 | else: 61 | try: 62 | return json.dumps(obj) 63 | except TypeError: # Not json serializable 64 | pass 65 | return str(obj) 66 | 67 | 68 | def _call_js_function(js_function, *args): 69 | """Evalutes a javascript string with arguments and returns its value.""" 70 | serialized = json.dumps(args) 71 | if len(serialized) < _MSG_CHUNK_SIZE: 72 | return output.eval_js('({})(...{})'.format(js_function, serialized)) 73 | 74 | name = str(uuid.uuid4()) 75 | for i in range(0, len(serialized), _MSG_CHUNK_SIZE): 76 | chunk = serialized[i:i + _MSG_CHUNK_SIZE] 77 | output.eval_js( 78 | """window["{name}"] = (window["{name}"] || "") + atob("{b64_chunk}"); 79 | """.format(name=name, b64_chunk=base64.b64encode(chunk)), 80 | ignore_result=True) 81 | return output.eval_js(""" 82 | (function() {{ 83 | const msg = JSON.parse(window["{name}"]); 84 | delete window["{name}"]; 85 | return ({js_function})(...msg); 86 | }})(); 87 | """.format(name=name, js_function=js_function)) 88 | 89 | 90 | def _proxy(guid, msg): 91 | """Makes a proxy call on an element.""" 92 | template = _resources.get_data(__name__, 'js/_proxy.js') 93 | return _call_js_function(template, guid, msg) 94 | 95 | 96 | def _exists(guid): 97 | """Checks if an element with the given guid exists.""" 98 | template = _resources.get_data(__name__, 'js/_proxy.js') 99 | return _call_js_function(template, guid, {'method': 'exists'}, False) 100 | 101 | 102 | class _ElementView(object): 103 | """View container used for rendering element template in mustache.""" 104 | 105 | def __init__(self, element): 106 | # pylint: disable=protected-access 107 | self.guid = element._guid 108 | self.tag = element._tag 109 | self.src = element._src 110 | self.attributes = [{ 111 | 'name': k, 112 | 'value': v 113 | } for k, v in element._attributes.items()] 114 | self.properties = [{ 115 | 'name': k, 116 | 'value': json.dumps(v) 117 | } for k, v in element._properties.items()] 118 | self.js_listeners = [{ 119 | 'name': k, 120 | 'value': json.dumps(c) 121 | } for k, v in element._js_listeners.items() for c in v.values()] 122 | self.py_listeners = [{ 123 | 'name': k, 124 | 'value': c 125 | } for k, v in element._py_listeners.items() for c in v.values()] 126 | self.children = [_to_html_str(c) for c in element._children] 127 | # pylint: enable=protected-access 128 | 129 | 130 | class Element(object): 131 | """Create an object which will render as an html element in output cell.""" 132 | 133 | def __init__(self, tag, attributes=None, properties=None, src=None): 134 | """Initialize the element. 135 | 136 | Args: 137 | tag: Custom element tag name. 138 | attributes: Initial attributes to set. 139 | properties: Initial properties to set. 140 | src: Entry point url of source for element. Should be a dict 141 | containing one of the following keys script, html, module. 142 | For example: {"script": "data:application/javascript;,"} 143 | Raises: 144 | ValueError: If invalid deps, attributes, or properites. 145 | """ 146 | if src: 147 | if not ('script' in src or 'module' in src or 'html' in src): 148 | raise ValueError('Must provide a valid src.') 149 | self._src = src 150 | if attributes and not isinstance(attributes, dict): 151 | raise ValueError('attributes must be a dict.') 152 | if properties and not isinstance(properties, dict): 153 | raise ValueError('properties must be a dict.') 154 | self._tag = tag 155 | self._guid = str(uuid.uuid4()) 156 | self._attributes = attributes or {} 157 | self._properties = properties or {} 158 | self._children = [] 159 | self._js_listeners = {} 160 | self._py_listeners = {} 161 | self._parent = None 162 | self._could_exist = False 163 | 164 | def _exists(self): 165 | if not self._could_exist: 166 | return False 167 | return _exists(self._guid) 168 | 169 | def get_attribute(self, name): 170 | if not self._exists(): 171 | return self._attributes.get(name) 172 | return _proxy(self._guid, {'method': 'getAttribute', 'name': name}) 173 | 174 | def set_attribute(self, name, value): 175 | if not isinstance(value, six.string_types): 176 | raise ValueError('Attribute value must be a string') 177 | if not self._exists(): 178 | self._attributes[name] = value 179 | else: 180 | _proxy(self._guid, { 181 | 'method': 'setAttribute', 182 | 'value': value, 183 | 'name': name 184 | }) 185 | 186 | def get_property(self, name): 187 | if not self._exists(): 188 | return self._properties.get(name) 189 | return _proxy(self._guid, {'method': 'getProperty', 'name': name}) 190 | 191 | def set_property(self, name, value): 192 | if not self._exists(): 193 | self._properties[name] = value 194 | else: 195 | _proxy(self._guid, { 196 | 'method': 'setProperty', 197 | 'value': value, 198 | 'name': name 199 | }) 200 | 201 | def call(self, method, *args): 202 | if not self._exists(): 203 | raise ValueError('Cannot call method on undisplayed element.') 204 | return _proxy(self._guid, {'method': 'call', 'value': args, 'name': method}) 205 | 206 | def add_event_listener(self, name, callback): 207 | """Adds an event listener to the element. 208 | 209 | Args: 210 | name: Name of the event. 211 | callback: The python function or js string to evaluate when event occurs. 212 | Raises: 213 | ValueError: If callback is not valid. 214 | """ 215 | msg = {'name': name} 216 | if isinstance(callback, six.string_types): 217 | callbacks = self._js_listeners.get(name, {}) 218 | if callback in callbacks: 219 | raise ValueError('Callback is already added.') 220 | callbacks[callback] = callback 221 | self._js_listeners[name] = callbacks 222 | msg['method'] = 'addJsEventListener' 223 | msg['value'] = callback 224 | elif callable(callback): 225 | callbacks = self._py_listeners.get(name, {}) 226 | if callback in callbacks: 227 | raise ValueError('Callback is already added.') 228 | callback_name = str(uuid.uuid4()) 229 | output.register_callback(callback_name, callback) 230 | callbacks[callback] = callback_name 231 | self._py_listeners[name] = callbacks 232 | msg['method'] = 'addPythonEventListener' 233 | msg['value'] = callback_name 234 | else: 235 | raise ValueError('callback must be a js string or callable python') 236 | if self._exists(): 237 | _proxy(self._guid, msg) 238 | 239 | def remove_event_listener(self, name, callback): 240 | """Removes an event listener from the element. 241 | 242 | Args: 243 | name: String of the event. 244 | callback: The callback passed into add_event_listener previously. 245 | Raises: 246 | ValueError: If the callback was not added previously. 247 | """ 248 | if isinstance(callback, six.string_types): 249 | listener_map = self._js_listeners 250 | else: 251 | listener_map = self._py_listeners 252 | if name not in listener_map: 253 | raise ValueError('listener does not exist') 254 | callbacks = listener_map[name] 255 | if callback not in callbacks: 256 | raise ValueError('listener does not exist') 257 | callback_name = callbacks[callback] 258 | del callbacks[callback] 259 | if not callbacks: 260 | del listener_map[name] 261 | if self._exists(): 262 | _proxy(self._guid, { 263 | 'method': 'removeEventListener', 264 | 'name': name, 265 | 'value': callback_name 266 | }) 267 | 268 | def append_child(self, child): 269 | """Append child to Element.""" 270 | # Child could be anything that can be converted to html. 271 | if isinstance(child, Element): 272 | child.remove() 273 | child._parent = self # pylint: disable=protected-access 274 | self._children.append(child) 275 | 276 | def remove_child(self, child): 277 | """Remove child from Element.""" 278 | if isinstance(child, Element): 279 | if child._parent != self: # pylint: disable=protected-access 280 | raise ValueError('Child parent must match.') 281 | child._parent = None # pylint: disable=protected-access 282 | self._children = [c for c in self._children if c is not child] 283 | 284 | def remove(self): 285 | parent = self._parent 286 | if not parent: 287 | return 288 | parent.remove_child(self) 289 | 290 | def _repr_html_(self): 291 | """Converts element to HTML string.""" 292 | self._could_exist = True 293 | view = _ElementView(self) 294 | template = _resources.get_data(__name__, 'templates/_element.mustache') 295 | return pystache.render(template, view) 296 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2017 Google LLC 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /google/colab/output/_js_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Building javascript in python.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import datetime 21 | import functools 22 | import json 23 | import time 24 | import uuid 25 | from google.colab.output import _js 26 | from google.colab.output import _publish 27 | 28 | 29 | # Different ways of creating javascript 30 | # PERSISTENT: Will result in javascript being saved in the ipynb permanently 31 | # and reexecuting on reload 32 | PERSISTENT = 'persistent' 33 | 34 | # EVAL: javascript will be evaled and never saved in ipynb file. 35 | EVAL = 'eval' 36 | 37 | 38 | class JsException(Exception): 39 | pass 40 | 41 | 42 | class Js(object): 43 | """Base class to execute javascript using python chaining. 44 | 45 | Basic usage is like this: 46 | 47 | alert = Js('alert') 48 | 49 | # equivalent to alert("Hello World") on javascript console 50 | alert("Hello world") 51 | 52 | jQuery = Js('$') # Assumes jquery is loaded 53 | jQuery("#my-id").css("background", "green") 54 | # The result is Js() object, whose properties could be further accessed. 55 | # For example 56 | my_dom_el = jQuery("#my-id") # equivalent to my_dom_el = $("#my-id") 57 | my_dom_el.css("background", "black) # my_dom_el.css(...) 58 | # It could also be passed as parameter: 59 | jQuery(my_dom_el) # equivalent to $(my_dom_el), etc. 60 | 61 | # Also properties can be accessed normally: 62 | global = Js() 63 | global.console.log("hi") # equivalent to console.log("hi") 64 | global.setTimeout(global.alert, 3000) # setTimeout(alert, 3000) 65 | 66 | If the result of javascript computation is jsonable object one can access 67 | access its value via eval(). 68 | 69 | Supports all basic python types, including L{datetime.datetime} objects. 70 | """ 71 | 72 | def __init__(self, expr=None, mode=PERSISTENT): 73 | """Constructor. 74 | 75 | win = Js() # Will create an opaque reference to global context 76 | win.console.log('Test') # will log to javascript console 77 | 78 | js_func = Js('function() { ...}') will create a function that could 79 | be later called like a regular function. 80 | Args: 81 | expr: if provided will create an opaque representatoin 82 | of this javascript expression and it could be used to directly 83 | operate on it. If None: will assume a global context. 84 | mode: how to run javascript, one of (PERSISTENT or EVAL) 85 | """ 86 | self._attr_map = {} 87 | self._context = expr 88 | self._mode = mode 89 | self._builder = functools.partial(type(self), mode=mode) 90 | self._run_js = self._get_javascript_runner(mode) 91 | 92 | def _get_javascript_runner(self, mode): 93 | """Returns an appropriate function that executes the given javascript.""" 94 | if mode == PERSISTENT: 95 | # Note: want lazy binding, particularly for tests to be able 96 | # to inject custom handling. 97 | # pylint: disable=unnecessary-lambda 98 | return lambda x: _publish.javascript(x) 99 | elif mode == EVAL: 100 | # Note: we don't want javascript value on python side 101 | # unless user specifically requests it using .eval(). 102 | # This allows us to properly chain function and javascript class 103 | # (since those are not serializable) and eliminates the need to wait 104 | # for frontend to return. 105 | return lambda x: _js.eval_js('(()=>{' + x + '})()', ignore_result=True) 106 | else: 107 | raise JsException('Invalid mode: %r.' % mode) 108 | 109 | def __repr__(self): 110 | return 'Js(%s)' % self._context 111 | 112 | def __call__(self, *args, **kwargs): 113 | """Sends args into javascript call for this context. 114 | 115 | Args: 116 | *args: list of arguments to pass to a javascript function. 117 | **kwargs: optional keyword args. Currently only supports result_name 118 | to store the result of the call in the named JS variable. If omitted, 119 | the result is stored in a window member named after a UUID generated for 120 | this call. 121 | 122 | Returns: 123 | A Js object that could be used in arguments or for further chaining. 124 | 125 | Raises: 126 | JsException: if this object has no context (e.g. js_global) 127 | ValueError: if an unexpected kwargs name is specified 128 | """ 129 | result_name = kwargs.pop('result_name', None) 130 | if kwargs: 131 | raise ValueError('Unexpected kwargs: {}'.format(kwargs)) 132 | return self._get_expr_result(self._call_expr(args), result_name=result_name) 133 | 134 | def _call_expr(self, args): 135 | """Construct javascript call on current context with args.""" 136 | if self._context is None: 137 | raise JsException('Cannot call a function with empty context.') 138 | # Generates argument list without surrounding '[' and ']' 139 | arg_json = json.dumps(args, cls=_JavascriptEncoder)[1:-1] 140 | return '%s(%s)' % (self._js_value(), arg_json) 141 | 142 | def _get_expr_result(self, expr, result_name=None): 143 | result_name = result_name or uuid.uuid1() 144 | js_result = self._js_value(result_name) 145 | self._run_js('%s = %s;' % (js_result, expr)) 146 | return self._builder(result_name) 147 | 148 | def _join(self, context, name): 149 | if context: 150 | return context + '.' + name 151 | return name 152 | 153 | def _js_value_as_object(self): 154 | return self._js_value() or 'window' 155 | 156 | def __getitem__(self, name): 157 | return self._builder( 158 | self._js_value_as_object() + '[' + json.dumps(name) + ']') 159 | 160 | def __setitem__(self, name, value): 161 | """Enables setting properties on javascript object. 162 | 163 | For example: 164 | output.js_global['foo'] = 'bar' # global variable named foo with value bar 165 | output.js_global[output.js_global.foo] = 'hi' # 'bar' have value hi. 166 | 167 | Args: 168 | name: name of the item 169 | value: the value to set it to - should be json-like data, or JS object 170 | """ 171 | v = json.dumps(value, cls=_JavascriptEncoder) 172 | name = json.dumps(name, cls=_JavascriptEncoder) 173 | self._run_js('%s[%s] = %s;' % (self._js_value_as_object(), name, v)) 174 | 175 | def __setattr__(self, name, value): 176 | """Allows to do variable assignment. 177 | 178 | Note this doesn't allow to assign to variables starting with '_'. 179 | Args: 180 | name: The name of the attribute/variable 181 | value: json-like value, or JS object 182 | """ 183 | if name.startswith('_'): 184 | object.__setattr__(self, name, value) 185 | return 186 | v = json.dumps(value, cls=_JavascriptEncoder) 187 | name = json.dumps(name, cls=_JavascriptEncoder)[1:-1] 188 | result = '%s = %s;' % (self._join(self._js_value(), name), v) 189 | self._run_js(result) 190 | 191 | def __getattr__(self, name): 192 | """Returns a JS object pointing to context.name. 193 | 194 | The result could be used for chaining, as an argument 195 | or as a function. 196 | 197 | Args: 198 | name: name of the attribute to look up. 199 | Returns: 200 | The named attribute (as a Js object). 201 | Raises: 202 | AttributeError: if name is invalid 203 | """ 204 | # Don't try to evaluate special python functions. 205 | if name.startswith('__') and name.endswith('__'): 206 | raise AttributeError('%s not found' % name) 207 | val = self._attr_map.get(name, None) 208 | if val is None: 209 | val = self._builder(self._join(self._js_value(), name)) 210 | self._attr_map[name] = val 211 | return val 212 | 213 | def _js_value(self, name=None): 214 | """Return a string representing this object javascript value.""" 215 | if name is None: 216 | name = self._context 217 | # Running in global context 218 | if name is None: 219 | return '' 220 | # Context is compound object 221 | if not isinstance(name, uuid.UUID): 222 | return name 223 | # Context is a global variable or artificial uuid 224 | return 'window["%s"]' % name 225 | 226 | def eval(self): 227 | """Evals the content on javascript side and returns result. 228 | 229 | Note: if the result of this javascript computation is not 230 | json serializable (e.g. it is a function or class) this will fail. 231 | 232 | This function does not affect the underlying ipynb. 233 | 234 | Usage example: 235 | # this is executed on javascript side, x is opaque reference to 236 | # the result of my_function computation. 237 | x = output.js_global.my_class.my_function(1, 2, 3) 238 | # This gets the value to python 239 | print x.eval() 240 | 241 | This works with any javascript mode. 242 | Returns: 243 | evaled javascript. 244 | """ 245 | return _js.eval_js(self._js_value()) 246 | 247 | def trait_names(self): 248 | """IPython expects this function, otherwise getattr() is called .""" 249 | return [] 250 | 251 | # pylint: disable=invalid-name 252 | def _getAttributeNames(self): 253 | """Same as trait_names.""" 254 | return self.__dir__() 255 | 256 | def __add__(self, other): 257 | return self._get_expr_result('%s + %s' % self._arith_args(other)) 258 | 259 | def __sub__(self, other): 260 | return self._get_expr_result('%s - %s' % self._arith_args(other)) 261 | 262 | def __mul__(self, other): 263 | return self._get_expr_result('%s * %s' % self._arith_args(other)) 264 | 265 | def __div__(self, other): 266 | return self._get_expr_result('%s / %s' % self._arith_args(other)) 267 | 268 | def __mod__(self, other): 269 | return self._get_expr_result('%s % %s' % self._arith_args(other)) 270 | 271 | def __radd__(self, other): 272 | return self._get_expr_result('%s + %s' % self._arith_args(other)[::-1]) 273 | 274 | def __rsub__(self, other): 275 | return self._get_expr_result('%s - %s' % self._arith_args(other)[::-1]) 276 | 277 | def __rmul__(self, other): 278 | return self._get_expr_result('%s * %s' % self._arith_args(other)[::-1]) 279 | 280 | def __rdiv__(self, other): 281 | return self._get_expr_result('%s / %s' % self._arith_args(other)[::-1]) 282 | 283 | def __rmod__(self, other): 284 | return self._get_expr_result('%s % %s' % self._arith_args(other)[::-1]) 285 | 286 | def _arith_args(self, other): 287 | """Helper for arithmetic support.""" 288 | if self._context is None: 289 | raise JsException('Cannot do arithmetic on empty context.') 290 | s = self._js_value() 291 | o = json.dumps(other, cls=_JavascriptEncoder) 292 | if not o: 293 | raise JsException('Cannot do arithmetic on empty operand.') 294 | return s, o 295 | 296 | def new_object(self, *args): 297 | """Assuming self describes a type, constructs a new object of that type. 298 | 299 | Example usage: 300 | THREE = Js('THREE') 301 | // v now points to new THREE.Vector3(1, 2, 3) 302 | v = THREE.Vector3.new_object(1, 2, 3) 303 | 304 | Args: 305 | *args: list of arguments to pass to a javascript constructor. 306 | 307 | Returns: 308 | A Js object holding the constructed instance. 309 | 310 | Raises: 311 | JsException: if this object has no context (i.e. not a constructor) 312 | """ 313 | return self._get_expr_result('new ' + self._call_expr(args)) 314 | 315 | 316 | def _py_datetime_to_js_date(o): 317 | return Js('new Date(%d)' % (1000 * time.mktime(o.timetuple()))) 318 | 319 | 320 | TYPE_CONVERSION_MAP = { 321 | datetime.datetime: _py_datetime_to_js_date, 322 | } 323 | 324 | 325 | class _JavascriptEncoder(json.JSONEncoder): 326 | """Provides json-esque enconding for Js objects and python standard types. 327 | 328 | Note: while the name suggests JSON, this is not necessarily a json, 329 | instead it produces valid javascript that looks like json for trivial types, 330 | but might otherwise contain function calls, new Date() objects etc. 331 | """ 332 | 333 | def __init__(self, *args, **kwargs): 334 | kwargs['allow_nan'] = False 335 | json.JSONEncoder.__init__(self, *args, **kwargs) 336 | self._replacement_map = {} 337 | 338 | def default(self, o): 339 | if isinstance(o, Js): 340 | key = uuid.uuid4().hex 341 | # pylint: disable=protected-access 342 | self._replacement_map[key] = o._js_value() 343 | return key 344 | if hasattr(o, '__javascript__'): 345 | return Js(o.__javascript__()) 346 | # Get a list of ancestors of kls for new type classes 347 | # for old-style we don't support inheritance. 348 | kls = type(o) 349 | # Note: only new-style classes have mro method. 350 | bases = kls.mro() if issubclass(kls, object) else [kls] 351 | 352 | # Walk up the inheritance tree (or classes that participate in method 353 | # resolution), until we find something in conversion map 354 | # that's an this class is an instance of. This way we are guaranteed 355 | # to go from more specific classes to least specific in resolving 356 | # which class to use for conversion. 357 | for each_type in bases: 358 | if each_type in TYPE_CONVERSION_MAP: 359 | return TYPE_CONVERSION_MAP[each_type](o) 360 | return json.JSONEncoder.default(self, o) 361 | 362 | def encode(self, o): 363 | try: 364 | result = json.JSONEncoder.encode(self, o) 365 | except ValueError: 366 | # If NaN or +/-Infinity are part of the input, we need custom logic, 367 | # since they're not officially handled as part of JSON. Our solution is 368 | # the following, involving an extra round-trip through JSON: 369 | # * first, convert the input to JSON, allowing NaNs, 370 | # * second, convert *back* to a Python object, but explicitly converting 371 | # NaN and +/-Infinity into strings, and 372 | # * finally, convert our now-NaN-free object to JSON. 373 | # 374 | # We do it this way because Python only provides custom hooks for NaN 375 | # handling at *deserialization* time. 376 | # 377 | # Note that we're doing this in the context of a custom JSON serializer, 378 | # so it's important that we preserve that serializer for the first step, 379 | # which we do by passing `self.default` as the `default` arg to 380 | # `json.dumps`. 381 | nan_free_object = json.loads( 382 | json.dumps(o, default=self.default), 383 | parse_constant=lambda constant: constant) 384 | result = json.dumps(nan_free_object) 385 | # Why is this correct? Well, it is invalid to have anywhere 386 | # outside of quotes. (won't be a valid javascript) And it is invalid 387 | # to have it in quotes, because, browser parser. 388 | # This fixes the latter issue. It keeps the former invaild. 389 | result = result.replace('', r'<\/script>') 390 | for k, v in self._replacement_map.items(): 391 | result = result.replace('"%s"' % (k,), v) 392 | return result 393 | 394 | # global context 395 | js_global = Js() 396 | -------------------------------------------------------------------------------- /google/colab/_system_commands.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Colab-specific system command helpers.""" 15 | 16 | from __future__ import absolute_import 17 | from __future__ import division 18 | from __future__ import print_function 19 | 20 | import codecs 21 | import contextlib 22 | import locale 23 | import os 24 | import pty 25 | import select 26 | import signal 27 | import subprocess 28 | import sys 29 | import termios 30 | import time 31 | 32 | from IPython.core import magic_arguments 33 | from IPython.utils import text 34 | import six 35 | from google.colab import _ipython 36 | from google.colab import _message 37 | from google.colab.output import _tags 38 | 39 | # Linux read(2) limits to 0x7ffff000 so stay under that for clarity. 40 | _PTY_READ_MAX_BYTES_FOR_TEST = 2**20 # 1MB 41 | 42 | _ENCODING = 'UTF-8' 43 | 44 | 45 | def _shell_line_magic(line): 46 | """Runs a shell command, allowing input to be provided. 47 | 48 | This is similar to Jupyter's `!` magic, but additionally allows input to be 49 | provided to the subprocess. If the subprocess returns a non-zero exit code 50 | a `subprocess.CalledProcessError` is raised. The provided command is run 51 | within a bash shell. 52 | 53 | Also available as a cell magic. 54 | 55 | Usage: 56 | # Returns a ShellResult. 57 | f = %shell echo "hello" 58 | 59 | Args: 60 | line: The shell command to execute. 61 | 62 | Returns: 63 | ShellResult containing the results of the executed command. 64 | 65 | Raises: 66 | subprocess.CalledProcessError: If the subprocess exited with a non-zero 67 | exit code. 68 | """ 69 | result = _run_command(line, clear_streamed_output=False) 70 | result.check_returncode() 71 | return result 72 | 73 | 74 | @magic_arguments.magic_arguments() 75 | @magic_arguments.argument( 76 | '--ignore-errors', 77 | dest='ignore_errors', 78 | action='store_true', 79 | help=('Don\'t raise a `subprocess.CalledProcessError` when the ' 80 | 'subprocess returns a non-0 exit code.')) 81 | def _shell_cell_magic(args, cmd): 82 | """Run the cell via a shell command, allowing input to be provided. 83 | 84 | Also available as a line magic. 85 | 86 | Usage: 87 | # Returns a ShellResult. 88 | %%shell 89 | echo "hello" 90 | 91 | This is similar to Jupyter's `!` magic, but additionally allows input to be 92 | provided to the subprocess. By default, if the subprocess returns a non-zero 93 | exit code a `subprocess.CalledProcessError` is raised. The provided command 94 | is run within a bash shell. 95 | 96 | Args: 97 | args: Optional arguments. 98 | cmd: The shell command to execute. 99 | 100 | Returns: 101 | ShellResult containing the results of the executed command. 102 | 103 | Raises: 104 | subprocess.CalledProcessError: If the subprocess exited with a non-zero 105 | exit code and the `ignore_errors` argument wasn't provided. 106 | """ 107 | 108 | parsed_args = magic_arguments.parse_argstring(_shell_cell_magic, args) 109 | 110 | result = _run_command(cmd, clear_streamed_output=False) 111 | if not parsed_args.ignore_errors: 112 | result.check_returncode() 113 | return result 114 | 115 | 116 | class ShellResult(object): 117 | """Result of an invocation of the shell magic. 118 | 119 | Note: This is intended to mimic subprocess.CompletedProcess, but has slightly 120 | different characteristics, including: 121 | * CompletedProcess has separate stdout/stderr properties. A ShellResult 122 | has a single property containing the merged stdout/stderr stream, 123 | providing compatibility with the existing "!" shell magic (which this is 124 | intended to provide an alternative to). 125 | * A custom __repr__ method that returns output. When the magic is invoked as 126 | the only statement in the cell, Python prints the string representation by 127 | default. The existing "!" shell magic also returns output. 128 | """ 129 | 130 | def __init__(self, args, returncode, command_output): 131 | self.args = args 132 | self.returncode = returncode 133 | self.output = command_output 134 | 135 | def check_returncode(self): 136 | if self.returncode: 137 | raise subprocess.CalledProcessError( 138 | returncode=self.returncode, cmd=self.args, output=self.output) 139 | 140 | def _repr_pretty_(self, p, cycle): # pylint:disable=unused-argument 141 | # Note: When invoking the magic and not assigning the result 142 | # (e.g. %shell echo "foo"), Python's default semantics will be used and 143 | # print the string representation of the object. By default, this will 144 | # display the __repr__ of ShellResult. Suppress this representation since 145 | # the output of the command has already been displayed to the output window. 146 | if cycle: 147 | raise NotImplementedError 148 | 149 | 150 | def _configure_term_settings(pty_fd): 151 | term_settings = termios.tcgetattr(pty_fd) 152 | # ONLCR transforms NL to CR-NL, which is undesirable. Ensure this is disabled. 153 | # http://man7.org/linux/man-pages/man3/termios.3.html 154 | term_settings[1] &= ~termios.ONLCR 155 | 156 | # ECHOCTL echoes control characters, which is undesirable. 157 | term_settings[3] &= ~termios.ECHOCTL 158 | 159 | termios.tcsetattr(pty_fd, termios.TCSANOW, term_settings) 160 | 161 | 162 | def _run_command(cmd, clear_streamed_output): 163 | """Calls the shell command, forwarding input received on the stdin_socket.""" 164 | locale_encoding = locale.getpreferredencoding() 165 | if locale_encoding != _ENCODING: 166 | raise NotImplementedError( 167 | 'A UTF-8 locale is required. Got {}'.format(locale_encoding)) 168 | 169 | parent_pty, child_pty = pty.openpty() 170 | _configure_term_settings(child_pty) 171 | 172 | epoll = select.epoll() 173 | epoll.register( 174 | parent_pty, 175 | (select.EPOLLIN | select.EPOLLOUT | select.EPOLLHUP | select.EPOLLERR)) 176 | 177 | try: 178 | temporary_clearer = _tags.temporary if clear_streamed_output else _no_op 179 | 180 | with temporary_clearer(), _display_stdin_widget( 181 | delay_millis=500) as update_stdin_widget: 182 | # TODO(b/115531839): Ensure that subprocesses are terminated upon 183 | # interrupt. 184 | p = subprocess.Popen( 185 | cmd, 186 | shell=True, 187 | executable='/bin/bash', 188 | stdout=child_pty, 189 | stdin=child_pty, 190 | stderr=child_pty, 191 | close_fds=True) 192 | # The child PTY is only needed by the spawned process. 193 | os.close(child_pty) 194 | 195 | return _monitor_process(parent_pty, epoll, p, cmd, update_stdin_widget) 196 | finally: 197 | epoll.close() 198 | os.close(parent_pty) 199 | 200 | 201 | class _MonitorProcessState(object): 202 | 203 | def __init__(self): 204 | self.process_output = six.StringIO() 205 | self.is_pty_still_connected = True 206 | 207 | 208 | def _monitor_process(parent_pty, epoll, p, cmd, update_stdin_widget): 209 | """Monitors the given subprocess until it terminates.""" 210 | state = _MonitorProcessState() 211 | 212 | # A single UTF-8 character can span multiple bytes. os.read returns bytes and 213 | # could return a partial byte sequence for a UTF-8 character. Using an 214 | # incremental decoder is incrementally fed input bytes and emits UTF-8 215 | # characters. 216 | decoder = codecs.getincrementaldecoder(_ENCODING)() 217 | 218 | num_interrupts = 0 219 | echo_status = None 220 | while True: 221 | try: 222 | result = _poll_process(parent_pty, epoll, p, cmd, decoder, state) 223 | if result is not None: 224 | return result 225 | 226 | term_settings = termios.tcgetattr(parent_pty) 227 | new_echo_status = bool(term_settings[3] & termios.ECHO) 228 | if echo_status != new_echo_status: 229 | update_stdin_widget(new_echo_status) 230 | echo_status = new_echo_status 231 | except KeyboardInterrupt: 232 | try: 233 | num_interrupts += 1 234 | if num_interrupts == 1: 235 | p.send_signal(signal.SIGINT) 236 | elif num_interrupts == 2: 237 | # Process isn't responding to SIGINT and user requested another 238 | # interrupt. Attempt to send SIGTERM followed by a SIGKILL if the 239 | # process doesn't respond. 240 | p.send_signal(signal.SIGTERM) 241 | time.sleep(0.5) 242 | if p.poll() is None: 243 | p.send_signal(signal.SIGKILL) 244 | except KeyboardInterrupt: 245 | # Any interrupts that occur during shutdown should not propagate. 246 | pass 247 | 248 | if num_interrupts > 2: 249 | # In practice, this shouldn't be possible since 250 | # SIGKILL is quite effective. 251 | raise 252 | 253 | 254 | def _poll_process(parent_pty, epoll, p, cmd, decoder, state): 255 | """Polls the process and captures / forwards input and output.""" 256 | 257 | terminated = p.poll() is not None 258 | if terminated: 259 | termios.tcdrain(parent_pty) 260 | # We're no longer interested in write events and only want to consume any 261 | # remaining output from the terminated process. Continuing to watch write 262 | # events may cause early termination of the loop if no output was 263 | # available but the pty was ready for writing. 264 | epoll.modify(parent_pty, 265 | (select.EPOLLIN | select.EPOLLHUP | select.EPOLLERR)) 266 | 267 | output_available = False 268 | 269 | events = epoll.poll() 270 | input_events = [] 271 | for _, event in events: 272 | if event & select.EPOLLIN: 273 | output_available = True 274 | raw_contents = os.read(parent_pty, _PTY_READ_MAX_BYTES_FOR_TEST) 275 | decoded_contents = decoder.decode(raw_contents) 276 | 277 | sys.stdout.write(decoded_contents) 278 | state.process_output.write(decoded_contents) 279 | 280 | if event & select.EPOLLOUT: 281 | # Queue polling for inputs behind processing output events. 282 | input_events.append(event) 283 | 284 | # PTY was disconnected or encountered a connection error. In either case, 285 | # no new output should be made available. 286 | if (event & select.EPOLLHUP) or (event & select.EPOLLERR): 287 | state.is_pty_still_connected = False 288 | 289 | for event in input_events: 290 | # Check to see if there is any input on the stdin socket. 291 | # pylint: disable=protected-access 292 | input_line = _message._read_stdin_message() 293 | # pylint: enable=protected-access 294 | if input_line is not None: 295 | # If a very large input or sequence of inputs is available, it's 296 | # possible that the PTY buffer could be filled and this write call 297 | # would block. To work around this, non-blocking writes and keeping 298 | # a list of to-be-written inputs could be used. Empirically, the 299 | # buffer limit is ~12K, which shouldn't be a problem in most 300 | # scenarios. As such, optimizing for simplicity. 301 | input_bytes = bytes(input_line.encode(_ENCODING)) 302 | os.write(parent_pty, input_bytes) 303 | 304 | # Once the process is terminated, there still may be output to be read from 305 | # the PTY. Wait until the PTY has been disconnected and no more data is 306 | # available for read. Simply waiting for disconnect may be insufficient if 307 | # there is more data made available on the PTY than we consume in a single 308 | # read call. 309 | if terminated and not state.is_pty_still_connected and not output_available: 310 | sys.stdout.flush() 311 | command_output = state.process_output.getvalue() 312 | return ShellResult(cmd, p.returncode, command_output) 313 | 314 | if not output_available: 315 | # The PTY is almost continuously available for reading input to provide 316 | # to the underlying subprocess. This means that the polling loop could 317 | # effectively become a tight loop and use a large amount of CPU. Add a 318 | # slight delay to give resources back to the system while monitoring the 319 | # process. 320 | # Skip this delay if we read output in the previous loop so that a partial 321 | # read doesn't unnecessarily sleep before reading more output. 322 | # TODO(b/115527726): Rather than sleep, poll for incoming messages from 323 | # the frontend in the same poll as for the output. 324 | time.sleep(0.1) 325 | 326 | 327 | @contextlib.contextmanager 328 | def _display_stdin_widget(delay_millis=0): 329 | """Context manager that displays a stdin UI widget and hides it upon exit. 330 | 331 | Args: 332 | delay_millis: Duration (in milliseconds) to delay showing the widget within 333 | the UI. 334 | 335 | Yields: 336 | A callback that can be invoked with a single argument indicating whether 337 | echo is enabled. 338 | """ 339 | shell = _ipython.get_ipython() 340 | display_args = ['cell_display_stdin', {'delayMillis': delay_millis}] 341 | _message.blocking_request(*display_args, parent=shell.parent_header) 342 | 343 | def echo_updater(new_echo_status): 344 | # Note: Updating the echo status uses colab_request / colab_reply on the 345 | # stdin socket. Input provided by the user also sends messages on this 346 | # socket. If user input is provided while the blocking_request call is still 347 | # waiting for a colab_reply, the input will be dropped per 348 | # https://github.com/googlecolab/colabtools/blob/56e4dbec7c4fa09fad51b60feb5c786c69d688c6/google/colab/_message.py#L100. 349 | update_args = ['cell_update_stdin', {'echo': new_echo_status}] 350 | _message.blocking_request(*update_args, parent=shell.parent_header) 351 | 352 | yield echo_updater 353 | 354 | hide_args = ['cell_remove_stdin', {}] 355 | _message.blocking_request(*hide_args, parent=shell.parent_header) 356 | 357 | 358 | @contextlib.contextmanager 359 | def _no_op(): 360 | yield 361 | 362 | 363 | def _register_magics(ip): 364 | ip.register_magic_function( 365 | _shell_line_magic, magic_kind='line', magic_name='shell') 366 | ip.register_magic_function( 367 | _shell_cell_magic, magic_kind='cell', magic_name='shell') 368 | 369 | 370 | _INTERRUPTED_SIGNALS = ( 371 | signal.SIGINT, 372 | signal.SIGTERM, 373 | signal.SIGKILL, 374 | ) 375 | 376 | 377 | def _getoutput_compat(shell, cmd, split=True, depth=0): 378 | """Compatibility function for IPython's built-in getoutput command. 379 | 380 | The getoutput command has the following semantics: 381 | * Returns a SList containing an array of output 382 | * SList items are of type "str". In Python 2, the str object is utf-8 383 | encoded. In Python 3, the "str" type already supports Unicode. 384 | * The _exit_code attribute is not set 385 | * If the process was interrupted, "^C" is printed. 386 | 387 | Args: 388 | shell: An InteractiveShell instance. 389 | cmd: Command to execute. This is the same as the corresponding argument to 390 | InteractiveShell.getoutput. 391 | split: Same as the corresponding argument to InteractiveShell.getoutput. 392 | depth: Same as the corresponding argument to InteractiveShell.getoutput. 393 | Returns: 394 | The output as a SList if split was true, otherwise an LSString. 395 | """ 396 | # We set a higher depth than the IPython system command since a shell object 397 | # is expected to call this function, thus adding one level of nesting to the 398 | # stack. 399 | result = _run_command( 400 | shell.var_expand(cmd, depth=depth + 2), clear_streamed_output=True) 401 | if -result.returncode in _INTERRUPTED_SIGNALS: 402 | print('^C') 403 | 404 | output = result.output 405 | if six.PY2: 406 | # Backwards compatibility. Python 2 getoutput() expects the result as a 407 | # str, not a unicode. 408 | output = output.encode(_ENCODING) 409 | 410 | if split: 411 | return text.SList(output.splitlines()) 412 | else: 413 | return text.LSString(output) 414 | 415 | 416 | def _system_compat(shell, cmd): 417 | """Compatibility function for IPython's built-in system command. 418 | 419 | The system command has the following semantics: 420 | * No return value, and thus the "_" variable is not set 421 | * Sets the _exit_code variable to the return value of the called process 422 | * Unicode characters are preserved 423 | * If the process was interrupted, "^C" is printed. 424 | 425 | Args: 426 | shell: An InteractiveShell instance. 427 | cmd: Command to execute. This is the same as the corresponding argument to 428 | InteractiveShell.system_piped. 429 | Returns: 430 | Nothing. 431 | """ 432 | # We set a higher depth than the IPython system command since a shell object 433 | # is expected to call this function, thus adding one level of nesting to the 434 | # stack. 435 | result = _run_command( 436 | shell.var_expand(cmd, depth=2), clear_streamed_output=False) 437 | shell.user_ns['_exit_code'] = result.returncode 438 | if -result.returncode in _INTERRUPTED_SIGNALS: 439 | print('^C') 440 | --------------------------------------------------------------------------------