├── docs ├── source │ ├── _templates │ │ └── .keep │ ├── changelog.rst │ ├── project │ │ ├── releasing.rst │ │ ├── contributing.rst │ │ ├── local_ci.rst │ │ ├── code_of_conduct.rst │ │ ├── python_tests.rst │ │ └── index.rst │ ├── openapi.rst │ ├── _static │ │ └── images │ │ │ ├── nteract_logo_app_icon_purple.png │ │ │ └── nteract_logo_compact_purple.png │ ├── reference │ │ ├── s3_paths.rst │ │ ├── index.rst │ │ ├── store_client.rst │ │ ├── handlers.rst │ │ ├── publish.rst │ │ ├── archive.rst │ │ ├── bookstore_config.rst │ │ ├── nb_client.rst │ │ └── clone.rst │ ├── index.rst │ ├── installation.rst │ ├── configuration.rst │ ├── usage.rst │ ├── conf.py │ └── bookstore_api.yaml ├── requirements-doc.txt ├── Makefile └── make.bat ├── bookstore ├── tests │ ├── client │ │ ├── __init__.py │ │ ├── client_fixtures.py │ │ └── test_nb_client.py │ ├── __init__.py │ ├── test_files │ │ └── EmptyNotebook.ipynb │ ├── test_version.py │ ├── test_utils.py │ ├── test_s3_paths.py │ ├── test_archive.py │ ├── test_bookstore_config.py │ ├── test_publish.py │ ├── test_handlers.py │ └── test_clone.py ├── client │ ├── __init__.py │ ├── store_client.py │ └── nb_client.py ├── __init__.py ├── utils.py ├── _version.py ├── s3_paths.py ├── clone.html ├── bookstore_config.py ├── handlers.py ├── publish.py ├── archive.py └── clone.py ├── .gitattributes ├── ci ├── sleep.js ├── token.js ├── local.sh ├── jupyter_notebook_config.py ├── utils.js ├── s3.js ├── clone_request.py ├── jupyter.js └── integration.js ├── requirements.txt ├── jupyter_config └── jupyter_notebook_config.d │ └── bookstore.json ├── pytest.ini ├── requirements-dev.txt ├── .coveragerc ├── mypy.ini ├── .readthedocs.yml ├── pyproject.toml ├── CHECKLIST-release.md ├── .flake8 ├── .travis.yml ├── jupyter_config.py.example ├── .pre-commit-config.yaml ├── package.json ├── MANIFEST.in ├── running_python_tests.md ├── setup.cfg ├── RELEASING.md ├── LICENSE ├── .gitignore ├── running_ci_locally.md ├── .circleci └── config.yml ├── tox.ini ├── setup.py ├── CODE_OF_CONDUCT.md ├── README.md ├── CONTRIBUTING.md └── CHANGELOG.md /docs/source/_templates/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bookstore/tests/client/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | bookstore/_version.py export-subst 2 | -------------------------------------------------------------------------------- /docs/source/changelog.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../../CHANGELOG.md -------------------------------------------------------------------------------- /docs/source/project/releasing.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../../../RELEASING.md -------------------------------------------------------------------------------- /bookstore/client/__init__.py: -------------------------------------------------------------------------------- 1 | from .store_client import BookstoreClient 2 | -------------------------------------------------------------------------------- /docs/source/project/contributing.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../../../CONTRIBUTING.md -------------------------------------------------------------------------------- /docs/source/project/local_ci.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../../../running_ci_locally.md -------------------------------------------------------------------------------- /docs/source/project/code_of_conduct.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../../../CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /docs/source/project/python_tests.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../../../running_python_tests.md -------------------------------------------------------------------------------- /docs/requirements-doc.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | m2r 3 | sphinxcontrib-napoleon 4 | sphinxcontrib-openapi 5 | -------------------------------------------------------------------------------- /docs/source/openapi.rst: -------------------------------------------------------------------------------- 1 | REST API 2 | ======== 3 | 4 | 5 | .. openapi:: bookstore_api.yaml 6 | -------------------------------------------------------------------------------- /bookstore/tests/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | test_dir = os.path.realpath(os.path.dirname(__file__)) 4 | -------------------------------------------------------------------------------- /ci/sleep.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | sleep: timeout => 3 | new Promise((resolve, reject) => setTimeout(resolve, timeout)) 4 | }; 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | future 2 | futures ; python_version < "3.0" 3 | ipython >= 5.0 4 | notebook 5 | tornado >= 5.1.1 6 | aioboto3 7 | aiobotocore 8 | requests 9 | -------------------------------------------------------------------------------- /docs/source/_static/images/nteract_logo_app_icon_purple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/bookstore/HEAD/docs/source/_static/images/nteract_logo_app_icon_purple.png -------------------------------------------------------------------------------- /docs/source/_static/images/nteract_logo_compact_purple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nteract/bookstore/HEAD/docs/source/_static/images/nteract_logo_compact_purple.png -------------------------------------------------------------------------------- /jupyter_config/jupyter_notebook_config.d/bookstore.json: -------------------------------------------------------------------------------- 1 | { 2 | "NotebookApp": { 3 | "nbserver_extensions": { 4 | "bookstore": true 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | python_files = test_*.py 3 | 4 | env = 5 | AWS_SECRET_ACCESS_KEY=foobar_secret 6 | AWS_ACCESS_KEY_ID=foobar_key 7 | 8 | markers = 9 | asyncio 10 | -------------------------------------------------------------------------------- /docs/source/project/index.rst: -------------------------------------------------------------------------------- 1 | Project 2 | ======= 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | contributing 8 | code_of_conduct 9 | local_ci 10 | python_tests 11 | releasing 12 | -------------------------------------------------------------------------------- /docs/source/reference/s3_paths.rst: -------------------------------------------------------------------------------- 1 | Storage 2 | ======= 3 | 4 | 5 | The ``s3_paths`` module 6 | ----------------------- 7 | 8 | .. automodule:: bookstore.s3_paths 9 | :members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | codecov 2 | coverage 3 | flake8 4 | mock 5 | mypy==0.660 # pin to avoid surprises 6 | pre-commit 7 | pytest>=3.6 8 | pytest-asyncio 9 | pytest-cov 10 | pytest-mock 11 | tox 12 | black; python_version >= '3.6' 13 | -------------------------------------------------------------------------------- /docs/source/reference/index.rst: -------------------------------------------------------------------------------- 1 | Reference 2 | ========= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | bookstore_config 8 | archive 9 | handlers 10 | s3_paths 11 | clone 12 | publish 13 | nb_client 14 | store_client 15 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = bookstore 3 | 4 | [report] 5 | exclude_lines = 6 | if self.debug: 7 | pragma: no cover 8 | raise NotImplementedError 9 | if __name__ == .__main__.: 10 | ignore_errors = True 11 | omit = 12 | */tests/* 13 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | # Global options: 2 | 3 | [mypy] 4 | python_version = 3.6 5 | warn_return_any = True 6 | warn_unused_configs = True 7 | ignore_missing_imports = True 8 | 9 | # Per-module options: 10 | 11 | [mypy-bookstore._version] 12 | ignore_errors = True 13 | 14 | -------------------------------------------------------------------------------- /docs/source/reference/store_client.rst: -------------------------------------------------------------------------------- 1 | Bookstore Client 2 | ================ 3 | 4 | The ``bookstore.client.store_client`` module 5 | -------------------------------------------- 6 | 7 | .. module:: bookstore.client.store_client 8 | 9 | ``BookstoreClient`` 10 | ~~~~~~~~~~~~~~~~~~~ 11 | 12 | .. autoclass:: BookstoreClient 13 | :members: 14 | :show-inheritance: -------------------------------------------------------------------------------- /ci/token.js: -------------------------------------------------------------------------------- 1 | const { randomBytes } = require("crypto"); 2 | 3 | function genToken(byteLength = 32) { 4 | return new Promise((resolve, reject) => { 5 | randomBytes(byteLength, (err, buffer) => { 6 | if (err) { 7 | reject(err); 8 | return; 9 | } 10 | 11 | resolve(buffer.toString("hex")); 12 | }); 13 | }); 14 | } 15 | 16 | module.exports = { 17 | genToken 18 | }; 19 | -------------------------------------------------------------------------------- /bookstore/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | PACKAGE_DIR: str = os.path.realpath(os.path.dirname(__file__)) 4 | 5 | del os 6 | 7 | from .archive import BookstoreContentsArchiver 8 | from .bookstore_config import BookstoreSettings 9 | from .handlers import load_jupyter_server_extension 10 | from ._version import __version__ 11 | from ._version import version_info 12 | 13 | 14 | def _jupyter_server_extension_paths(): 15 | return [dict(module="bookstore")] 16 | -------------------------------------------------------------------------------- /ci/local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | 6 | # Run a minio server locally in a similar manner to how we do on CI 7 | 8 | docker rm -f minio-like-ci || true # Plow forward regardless 9 | docker run -p 9000:9000 \ 10 | --name minio-like-ci \ 11 | -e MINIO_ACCESS_KEY=ONLY_ON_CIRCLE \ 12 | -e MINIO_SECRET_KEY=CAN_WE_DO_THIS \ 13 | -v /mnt/data:/data -v /mnt/config:/Users/kylek/.minio \ 14 | minio/minio server /data 15 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | sphinx: 9 | configuration: docs/source/conf.py 10 | 11 | formats: all 12 | 13 | # Optionally set the version of Python and requirements required to build your docs 14 | python: 15 | version: 3.7 16 | install: 17 | - requirements: docs/requirements-doc.txt 18 | - method: pip 19 | path: . 20 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | 4 | 5 | [tool.black] 6 | line-length = 100 7 | include = '\.pyi?$' 8 | exclude = ''' 9 | /( 10 | \.git 11 | | \.hg 12 | | \.mypy_cache 13 | | \.tox 14 | | \.venv 15 | | _build 16 | | buck-out 17 | | build 18 | | dist 19 | 20 | # The following are specific to Black, you probably don't want those. 21 | | blib2to3 22 | | tests/data 23 | | profiling 24 | )/ 25 | ''' 26 | skip-string-normalization = true 27 | -------------------------------------------------------------------------------- /docs/source/reference/handlers.rst: -------------------------------------------------------------------------------- 1 | API Handlers 2 | ============ 3 | 4 | The ``handlers`` module 5 | ----------------------- 6 | 7 | .. module:: bookstore.handlers 8 | 9 | ``BookstoreVersionHandler`` 10 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 11 | 12 | .. autoclass:: BookstoreVersionHandler 13 | :members: 14 | :show-inheritance: 15 | 16 | Jupyter Server extension 17 | ~~~~~~~~~~~~~~~~~~~~~~~~ 18 | 19 | .. autofunction:: load_jupyter_server_extension 20 | 21 | This function loads bookstore as a Jupyter Server extension. 22 | -------------------------------------------------------------------------------- /docs/source/reference/publish.rst: -------------------------------------------------------------------------------- 1 | Publishing 2 | ========== 3 | 4 | The ``publish`` module 5 | ---------------------- 6 | 7 | .. module:: bookstore.publish 8 | 9 | ``BookstorePublishAPIHandler`` 10 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 11 | 12 | .. autoclass:: BookstorePublishAPIHandler 13 | 14 | Methods 15 | ^^^^^^^ 16 | 17 | .. automethod:: BookstorePublishAPIHandler.initialize 18 | 19 | .. automethod:: BookstorePublishAPIHandler.put 20 | 21 | .. automethod:: BookstorePublishAPIHandler.validate_model 22 | 23 | .. automethod:: BookstorePublishAPIHandler.prepare_response 24 | -------------------------------------------------------------------------------- /CHECKLIST-release.md: -------------------------------------------------------------------------------- 1 | # Release checklist 2 | 3 | - [ ] Upgrade Docs prior to Release 4 | 5 | - [ ] Change log 6 | - [ ] New features documented 7 | - [ ] Update the contributor list - thank you page 8 | 9 | - [ ] Release software 10 | 11 | - [ ] Make sure 0 issues in milestone 12 | - [ ] Follow release process steps 13 | - [ ] Send builds to PyPI (Warehouse) and Conda Forge 14 | 15 | - [ ] Blog post and/or release note 16 | 17 | - [ ] Notify users of release 18 | 19 | - [ ] Tweet 20 | 21 | - [ ] Increment the version number for the next release 22 | 23 | - [ ] Update roadmap 24 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | # Ignore style and complexity 3 | # E: style errors 4 | # W: style warnings 5 | # C: complexity 6 | # F401: module imported but unused 7 | # F403: import * 8 | # F811: redefinition of unused `name` from line `N` 9 | # F821: 10 | # F841: local variable assigned but never used 11 | # E402: module level import not at top of file 12 | # I100: Import statements are in the wrong order 13 | # I101: Imported names are in the wrong order. Should be 14 | ignore = E, C, W, F401, F403, F811, F821, F841, E402, I100, I101, D400 15 | 16 | exclude = 17 | .cache, 18 | .github, 19 | docs, 20 | setup.py 21 | -------------------------------------------------------------------------------- /docs/source/reference/archive.rst: -------------------------------------------------------------------------------- 1 | Archiving 2 | ========= 3 | 4 | .. module:: bookstore.archive 5 | 6 | The ``archive`` module 7 | ---------------------- 8 | 9 | The ``archive`` module manages archival of notebooks to storage (i.e. S3) when 10 | a notebook save occurs. 11 | 12 | ``ArchiveRecord`` 13 | ~~~~~~~~~~~~~~~~~ 14 | 15 | Bookstore uses an immutable ``ArchiveRecord`` to represent a notebook file by 16 | its storage path. 17 | 18 | .. autoclass:: ArchiveRecord 19 | :members: 20 | 21 | 22 | ``BookstoreContentsArchiver`` 23 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 24 | 25 | .. autoclass:: BookstoreContentsArchiver 26 | :members: 27 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = ./source 8 | BUILDDIR = _build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 20 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | matrix: 4 | include: 5 | - python: 3.6 6 | env: TOXENV=py36 7 | - python: 3.7 8 | dist: xenial # required for Python 3.7 (travis-ci/travis-ci#9069) 9 | sudo: required # required for Python 3.7 (travis-ci/travis-ci#9069) 10 | env: TOXENV=py37 11 | - python: 3.6 12 | env: TOXENV=flake8 13 | - python: 3.6 14 | env: TOXENV=black 15 | - python: 3.6 16 | env: TOXENV=mypy 17 | - python: 3.6 18 | env: TOXENV=dist 19 | - python: 3.6 20 | env: TOXENV=docs 21 | - python: 3.6 22 | env: TOXENV=manifest 23 | 24 | install: 25 | - pip install tox 26 | script: 27 | - tox -e $TOXENV 28 | -------------------------------------------------------------------------------- /bookstore/tests/test_files/EmptyNotebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [] 9 | } 10 | ], 11 | "metadata": { 12 | "kernelspec": { 13 | "display_name": "dev", 14 | "language": "python", 15 | "name": "dev" 16 | }, 17 | "language_info": { 18 | "codemirror_mode": { 19 | "name": "ipython", 20 | "version": 3 21 | }, 22 | "file_extension": ".py", 23 | "mimetype": "text/x-python", 24 | "name": "python", 25 | "nbconvert_exporter": "python", 26 | "pygments_lexer": "ipython3", 27 | "version": "3.6.8" 28 | } 29 | }, 30 | "nbformat": 4, 31 | "nbformat_minor": 2 32 | } 33 | -------------------------------------------------------------------------------- /bookstore/tests/test_version.py: -------------------------------------------------------------------------------- 1 | """Test version checking""" 2 | import logging 3 | 4 | import pytest 5 | 6 | from .._version import _check_version 7 | 8 | 9 | @pytest.mark.parametrize( 10 | 'bookstore_version, msg', 11 | [ 12 | ('', 'Bookstore has no version header'), 13 | ('1.0.0', 'deprecated bookstore'), 14 | ('2.0.0', 'Bookstore version is'), 15 | ('2.3.1', 'Bookstore version is'), 16 | ('xxxxx', 'Invalid version'), 17 | ], 18 | ) 19 | def test_check_version(bookstore_version, msg, caplog): 20 | log = logging.getLogger() 21 | caplog.set_level(logging.DEBUG) 22 | _check_version(bookstore_version, log) 23 | record = caplog.records[0] 24 | assert msg in record.getMessage() 25 | -------------------------------------------------------------------------------- /docs/source/reference/bookstore_config.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | Bookstore may be configured by providing ``BookstoreSettings`` in the 5 | ``~/.jupyter/jupyter_notebook_config.py`` file. 6 | 7 | The ``bookstore_config`` module 8 | ------------------------------- 9 | 10 | .. module:: bookstore.bookstore_config 11 | 12 | ``BookstoreSettings`` 13 | ~~~~~~~~~~~~~~~~~~~~~ 14 | 15 | These settings are configurable by the user. Bookstore uses the traitlets 16 | library to handle the configurable options. 17 | 18 | .. autoclass:: BookstoreSettings 19 | :members: 20 | 21 | Functions 22 | ~~~~~~~~~ 23 | 24 | These functions will generally be used by developers of the bookstore application. 25 | 26 | .. autofunction:: validate_bookstore -------------------------------------------------------------------------------- /jupyter_config.py.example: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | print("Welcome to the bookstore 📚") 5 | 6 | from bookstore import BookstoreContentsArchiver, BookstoreSettings 7 | 8 | # jupyter config 9 | # At ~/.jupyter/jupyter_notebook_config.py for user installs 10 | # At __ for system installs 11 | c = get_config() 12 | 13 | c.NotebookApp.contents_manager_class = BookstoreContentsArchiver 14 | 15 | c.BookstoreSettings.workspace_prefix = "works" 16 | 17 | # If using minio for development 18 | c.BookstoreSettings.s3_endpoint_url = "http://127.0.0.1:9000" 19 | c.BookstoreSettings.s3_bucket = "bookstore" 20 | c.BookstoreSettings.s3_access_key_id = "4MR9ON7H4UNVCT2LQTFX" 21 | c.BookstoreSettings.s3_secret_access_key = "o8CnAN5G9x87P9aLxSjohoSV0EsCLjksuY6wjK9N" 22 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/ambv/black 3 | rev: 19.3b0 4 | hooks: 5 | - id: black 6 | language_version: python 7 | 8 | - repo: https://gitlab.com/pycqa/flake8 9 | rev: 3.7.7 10 | hooks: 11 | - id: flake8 12 | language_version: python3.7 13 | 14 | - repo: https://github.com/asottile/seed-isort-config 15 | rev: v1.9.0 16 | hooks: 17 | - id: seed-isort-config 18 | 19 | - repo: https://github.com/pre-commit/mirrors-isort 20 | rev: v4.3.18 21 | hooks: 22 | - id: isort 23 | language_version: python3.7 24 | 25 | - repo: https://github.com/pre-commit/pre-commit-hooks 26 | rev: v2.2.1 27 | hooks: 28 | - id: trailing-whitespace 29 | - id: end-of-file-fixer 30 | - id: debug-statements 31 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. bookstore documentation master file, created by 2 | sphinx-quickstart on Mon Oct 22 20:40:04 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | bookstore 7 | ========= 8 | 9 | Release v\ |release| (:doc:`What's new? `). 10 | 11 | **bookstore** provides tooling and workflow recommendations for storing, scheduling, and publishing notebooks. 12 | 13 | 14 | Table of Contents 15 | ================= 16 | 17 | .. toctree:: 18 | :maxdepth: 2 19 | 20 | installation 21 | configuration 22 | usage 23 | openapi 24 | reference/index 25 | project/index 26 | changelog 27 | 28 | 29 | Indices and tables 30 | ================== 31 | 32 | * :ref:`genindex` 33 | * :ref:`modindex` 34 | * :ref:`search` 35 | -------------------------------------------------------------------------------- /bookstore/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from bookstore.utils import url_path_join 4 | 5 | 6 | def test_url_join_pieces(): 7 | bucket = 'mybucket' 8 | prefix = 'yo' 9 | path = 'pickles' 10 | assert url_path_join(bucket, prefix, path) == 'mybucket/yo/pickles' 11 | 12 | 13 | def test_url_join_no_pieces(): 14 | with pytest.raises(IndexError): 15 | url_path_join() 16 | 17 | 18 | def test_url_join_one_piece(): 19 | assert url_path_join('mypath') == 'mypath' 20 | 21 | 22 | def test_url_path_join_strip_slash(): 23 | assert url_path_join('/bucket/', 'yo', 'pickles/') == '/bucket/yo/pickles/' 24 | assert url_path_join('bucket/', '/yo/', '/pickles') == 'bucket/yo/pickles' 25 | assert url_path_join('/bucket/', '/yo/', '/pickles/') == '/bucket/yo/pickles/' 26 | assert url_path_join('/', '/') == '/' 27 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=.\source 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bookstore", 3 | "version": "1.0.0", 4 | "description": "[![Documentation Status](https://readthedocs.org/projects/bookstore/badge/?version=latest)](https://bookstore.readthedocs.io/en/latest/?badge=latest)", 5 | "main": "index.js", 6 | "directories": { 7 | "doc": "docs" 8 | }, 9 | "private": "true", 10 | "scripts": { 11 | "test": "node ci/integration.js", 12 | "test:server": "./ci/local.sh" 13 | }, 14 | "repository": { 15 | "type": "git", 16 | "url": "git+https://github.com/nteract/bookstore.git" 17 | }, 18 | "author": "Kyle Kelley ", 19 | "license": "BSD-3-Clause", 20 | "bugs": { 21 | "url": "https://github.com/nteract/bookstore/issues" 22 | }, 23 | "homepage": "https://github.com/nteract/bookstore#readme", 24 | "dependencies": { 25 | "lodash": "^4.17.11", 26 | "minio": "^7.0.1", 27 | "rxjs": "^6.3.3", 28 | "xmlhttprequest": "^1.8.0" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include bookstore *.py 2 | recursive-include bookstore *.ipynb 3 | recursive-include bookstore *.json 4 | recursive-include bookstore *.yaml 5 | recursive-include bookstore *.keep 6 | recursive-include bookstore *.txt 7 | 8 | recursive-include ci *.js 9 | recursive-include ci *.py 10 | recursive-include ci *.sh 11 | 12 | include setup.py 13 | include requirements*.txt 14 | include tox.ini 15 | include pytest.ini 16 | include mypy.ini 17 | include .coveragerc 18 | include README.md 19 | include LICENSE 20 | include MANIFEST.in 21 | include *.md 22 | include *.toml 23 | include *.example 24 | include *.json 25 | include *.yaml 26 | include *.yml 27 | 28 | include bookstore/_version.py 29 | include bookstore/clone.html 30 | 31 | # Documentation 32 | graft docs 33 | # exclude build files 34 | prune docs/_build 35 | # Scripts 36 | graft scripts 37 | # Test env 38 | prune .tox 39 | prune .flake8 40 | prune .circleci 41 | prune .readthedocs.yml 42 | prune .precommit-config.yaml -------------------------------------------------------------------------------- /ci/jupyter_notebook_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | print("using CI config") 5 | 6 | from bookstore import BookstoreContentsArchiver, BookstoreSettings 7 | from bookstore.tests import test_dir 8 | 9 | 10 | # jupyter config 11 | # At ~/.jupyter/jupyter_notebook_config.py for user installs 12 | # At __ for system installs 13 | c = get_config() 14 | 15 | c.NotebookApp.contents_manager_class = BookstoreContentsArchiver 16 | 17 | c.BookstoreSettings.workspace_prefix = "ci-workspace" 18 | c.BookstoreSettings.published_prefix = "ci-published" 19 | 20 | # If using minio for development 21 | c.BookstoreSettings.s3_endpoint_url = "http://localhost:9000" 22 | c.BookstoreSettings.s3_bucket = "bookstore" 23 | 24 | # Straight out of `circleci/config.yml` 25 | c.BookstoreSettings.s3_access_key_id = "ONLY_ON_CIRCLE" 26 | c.BookstoreSettings.s3_secret_access_key = "CAN_WE_DO_THIS" 27 | 28 | # Local filesystem cloning 29 | c.BookstoreSettings.fs_cloning_basedir = test_dir 30 | -------------------------------------------------------------------------------- /ci/utils.js: -------------------------------------------------------------------------------- 1 | const url_path_join = function (...pieces) { 2 | // """Join components of url into a relative url 3 | // Use to prevent double slash when joining subpaths. 4 | // This will leave the initial and final / in place 5 | // 6 | // url_path_join("http://127.0.0.1:9988", "mybaseUrl/ipynb", "/api/contents//") => "http://127.0.0.1:9988/mybaseUrl/ipynb/api/contents/" 7 | // 8 | // They will be readded in between words, and at the beginning and end if they were 9 | // """ 10 | const initial = pieces[0].startsWith("/"); 11 | const final = pieces[pieces.length - 1].endsWith("/"); 12 | let result = pieces 13 | .filter(el => el !== "") 14 | .map(el => el.replace(/(^[ /]+)|([/]+$)/g, "")) 15 | .join("/"); 16 | if (initial) { 17 | result = "/" + result; 18 | } 19 | if (final) { 20 | result = result + "/"; 21 | } 22 | if (result == "//") { 23 | result = "/"; 24 | } 25 | return result; 26 | }; 27 | 28 | module.exports = { 29 | url_path_join 30 | }; 31 | -------------------------------------------------------------------------------- /running_python_tests.md: -------------------------------------------------------------------------------- 1 | # Running Python Tests 2 | 3 | The project uses pytest to run Python tests and tox as a tool for running 4 | tests in different environments. 5 | 6 | ## Setup Local development system 7 | 8 | Using Python 3.6+, install the dev requirements: 9 | 10 | ```bash 11 | pip install -r requirements-dev.txt 12 | ``` 13 | 14 | ## Run Python tests 15 | 16 | **Important:** We recommend using tox for running tests locally. 17 | Please deactivate any conda environments before running 18 | tests using tox. Failure to do so may corrupt your virtual environments. 19 | 20 | To run tests for a particular Python version (3.6 or 3.7): 21 | 22 | ```bash 23 | tox -e py36 # or py37 24 | ``` 25 | 26 | This will run the tests and display coverage information. 27 | 28 | ## Run linters 29 | 30 | ```bash 31 | tox -e flake8 32 | tox -e black 33 | ``` 34 | 35 | ## Run type checking 36 | 37 | ```bash 38 | tox -e mypy 39 | ``` 40 | 41 | ## Run All Tests and Checks 42 | 43 | ```bash 44 | tox 45 | ``` 46 | -------------------------------------------------------------------------------- /docs/source/reference/nb_client.rst: -------------------------------------------------------------------------------- 1 | Notebook Client 2 | =============== 3 | 4 | The ``bookstore.client.nb_client`` module 5 | ----------------------------------------- 6 | 7 | .. module:: bookstore.client.nb_client 8 | 9 | ``NotebookClient`` 10 | ~~~~~~~~~~~~~~~~~~ 11 | 12 | .. autoclass:: NotebookClient 13 | :members: 14 | 15 | ``NotebookClientCollection`` 16 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 17 | 18 | .. autoclass:: NotebookClientCollection 19 | :members: 20 | 21 | ``CurrentNotebookClient`` 22 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 23 | 24 | .. autoclass:: CurrentNotebookClient 25 | :members: 26 | 27 | 28 | ``LiveNotebookRecord`` 29 | ~~~~~~~~~~~~~~~~~~~~~~ 30 | 31 | .. autoclass:: LiveNotebookRecord 32 | :members: 33 | 34 | ``KernelInfo`` 35 | ~~~~~~~~~~~~~~ 36 | 37 | .. autoclass:: KernelInfo 38 | :members: 39 | 40 | ``NotebookSession`` 41 | ~~~~~~~~~~~~~~~~~~~ 42 | 43 | .. autoclass:: NotebookSession 44 | :members: 45 | 46 | Helper Function 47 | ~~~~~~~~~~~~~~~ 48 | 49 | .. autofunction:: extract_kernel_id -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | # References: 3 | # https://flake8.readthedocs.io/en/latest/user/configuration.html 4 | # https://flake8.readthedocs.io/en/latest/user/error-codes.html 5 | 6 | # Note: there cannot be spaces after comma's here 7 | exclude = __init__.py 8 | ignore = 9 | # Extra space in brackets 10 | E20, 11 | # Multiple spaces around "," 12 | E231,E241, 13 | # Comments 14 | E26, 15 | # Import formatting 16 | E4, 17 | # Comparing types instead of isinstance 18 | E721, 19 | # Assigning lambda expression 20 | E731 21 | max-line-length = 120 22 | 23 | [bdist_wheel] 24 | universal=1 25 | 26 | [coverage:run] 27 | branch = False 28 | omit = 29 | bookstore/tests/* 30 | bookstore/_version.py 31 | versioneer.py 32 | 33 | [coverage:report] 34 | exclude_lines = 35 | if self\.debug: 36 | pragma: no cover 37 | raise AssertionError 38 | raise NotImplementedError 39 | if __name__ == .__main__.: 40 | ignore_errors = True 41 | omit = bookstore/tests/*,bookstore/_version.py 42 | 43 | [tool:pytest] 44 | filterwarnings = always 45 | -------------------------------------------------------------------------------- /bookstore/tests/test_s3_paths.py: -------------------------------------------------------------------------------- 1 | """Tests for s3 paths""" 2 | import pytest 3 | 4 | from bookstore.s3_paths import s3_display_path, s3_key 5 | 6 | 7 | def test_s3_paths(): 8 | bucket = 'mybucket' 9 | prefix = 'yo' 10 | path = 'pickles' 11 | assert s3_display_path(bucket, prefix, path) == 's3://mybucket/yo/pickles' 12 | 13 | 14 | def test_s3_paths_no_path(): 15 | bucket = 'mybucket' 16 | prefix = 'yo' 17 | assert s3_display_path(bucket, prefix) == 's3://mybucket/yo' 18 | 19 | 20 | def test_s3_paths_no_prefix(): 21 | bucket = 'mybucket' 22 | path = 'pickles' 23 | with pytest.raises(NameError): 24 | s3_display_path(bucket, prefix, path) 25 | 26 | 27 | def test_s3_key_no_prefix(): 28 | with pytest.raises(TypeError): 29 | s3_key(path='s3://mybucket') 30 | 31 | 32 | def test_s3_key_invalid_prefix(): 33 | prefix = 1234 34 | with pytest.raises(AttributeError): 35 | s3_key(prefix) 36 | 37 | 38 | def test_s3_key_valid_parameters(): 39 | prefix = 'workspace' 40 | path = 'project/manufacturing' 41 | assert s3_key(prefix, path) == 'workspace/project/manufacturing' 42 | -------------------------------------------------------------------------------- /docs/source/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | bookstore may be installed using Python 3.6 and above. 5 | 6 | After installation, bookstore can process Python 2 or Python 3 notebooks. 7 | 8 | Install from PyPI (recommended) 9 | ------------------------------- 10 | 11 | .. code-block:: bash 12 | 13 | python3 -m pip install bookstore 14 | 15 | Install from conda-forge 16 | ------------------------ 17 | 18 | .. code-block:: bash 19 | 20 | conda install -c conda-forge bookstore 21 | 22 | Install from Source 23 | ------------------- 24 | 25 | 1. Clone this repo: 26 | 27 | .. code-block:: bash 28 | 29 | git clone https://github.com/nteract/bookstore.git 30 | 31 | 2. Change directory to repo root: 32 | 33 | .. code-block:: bash 34 | 35 | cd bookstore 36 | 37 | 3. Install dependencies: 38 | 39 | .. code-block:: bash 40 | 41 | python3 -m pip install -r requirements.txt 42 | python3 -m pip install -r requirements-dev.txt 43 | 44 | 4. Install package from source: 45 | 46 | .. code-block:: bash 47 | 48 | python3 -m pip install . 49 | 50 | .. tip:: Don't forget the dot at the end of the command 51 | -------------------------------------------------------------------------------- /bookstore/utils.py: -------------------------------------------------------------------------------- 1 | """Utility and helper functions.""" 2 | import os 3 | 4 | from tempfile import TemporaryDirectory 5 | 6 | 7 | def url_path_join(*pieces): 8 | """Join components into a relative url. 9 | 10 | Use to prevent double slash when joining subpath. This will leave the 11 | initial and final / in place. 12 | 13 | Code based on Jupyter notebook `url_path_join`. 14 | """ 15 | initial = pieces[0].startswith('/') 16 | final = pieces[-1].endswith('/') 17 | stripped = [s.strip('/') for s in pieces] 18 | result = '/'.join(s for s in stripped if s) 19 | if initial: 20 | result = '/' + result 21 | if final: 22 | result = result + '/' 23 | if result == '//': 24 | result = '/' 25 | return result 26 | 27 | 28 | class TemporaryWorkingDirectory(TemporaryDirectory): 29 | """Utility for creating a temporary working directory. 30 | """ 31 | 32 | def __enter__(self): 33 | self.cwd = os.getcwd() 34 | os.chdir(self.name) 35 | return super().__enter__() 36 | 37 | def __exit__(self, exc, value, tb): 38 | os.chdir(self.cwd) 39 | return super().__exit__(exc, value, tb) 40 | -------------------------------------------------------------------------------- /bookstore/_version.py: -------------------------------------------------------------------------------- 1 | """Bookstore version info 2 | 3 | Use pep 440 version rules. 4 | 5 | No dot before alpha/beta/rc. Use dot before `.dev`. Examples:: 6 | 7 | - 0.1.0rc1 8 | - 0.1.0a1 9 | - 0.1.0b1.dev 10 | - 0.1.0.dev 11 | 12 | `version_info` tuple:: 13 | 14 | - major 15 | - minor 16 | - micro 17 | - type of release (b1, rc1, or "" for final or dev) 18 | - suffix (dev or "" to designate a final version) 19 | """ 20 | 21 | version_info = (2, 5, 2, "", "dev") 22 | 23 | __version__ = ".".join(map(str, version_info[:3])) + ".".join(version_info[3:]) 24 | 25 | 26 | def _check_version(bookstore_version, log): 27 | """Check version and log status""" 28 | if not bookstore_version: 29 | log.warning( 30 | f"Bookstore has no version header, which means it is likely < 2.0. Expected {__version__}" 31 | ) 32 | elif bookstore_version[:1].isdigit() is False: 33 | log.warning(f"Invalid version format. Expected {__version__}") 34 | else: 35 | if bookstore_version[:1] < '2': 36 | log.warning( 37 | f"{bookstore_version} is the deprecated bookstore project for Openstack. Expected {__version__}" 38 | ) 39 | else: 40 | log.debug(f"Bookstore version is {bookstore_version}.") 41 | -------------------------------------------------------------------------------- /bookstore/s3_paths.py: -------------------------------------------------------------------------------- 1 | """S3 path utilities""" 2 | 3 | 4 | # Our S3 path delimiter will remain fixed as '/' in all uses 5 | delimiter = "/" 6 | 7 | 8 | def _join(*args): 9 | """Join S3 bucket args together. 10 | 11 | Remove empty entries and strip left-leading ``/`` 12 | """ 13 | return delimiter.join(filter(lambda s: s != '', map(lambda s: s.lstrip(delimiter), args))) 14 | 15 | 16 | def s3_path(bucket, prefix, path=''): 17 | """Compute the s3 path. 18 | 19 | Parameters 20 | ---------- 21 | bucket : str 22 | S3 bucket name 23 | prefix : str 24 | prefix for workspace or publish 25 | path : str 26 | The storage location 27 | """ 28 | return _join(bucket, prefix, path) 29 | 30 | 31 | def s3_key(prefix, path=''): 32 | """Compute the s3 key 33 | 34 | Parameters 35 | ---------- 36 | prefix : str 37 | prefix for workspace or publish 38 | path : str 39 | The storage location 40 | """ 41 | return _join(prefix, path) 42 | 43 | 44 | def s3_display_path(bucket, prefix, path=''): 45 | """Create a display name for use in logs 46 | 47 | Parameters 48 | ---------- 49 | bucket : str 50 | S3 bucket name 51 | prefix : str 52 | prefix for workspace or publish 53 | path : str 54 | The storage location 55 | """ 56 | return 's3://' + s3_path(bucket, prefix, path) 57 | -------------------------------------------------------------------------------- /RELEASING.md: -------------------------------------------------------------------------------- 1 | # Releasing 2 | 3 | ## Pre-release 4 | 5 | - [ ] First check that the CHANGELOG is up to date for the next release version. 6 | 7 | - [ ] Update docs 8 | 9 | ## Installing twine package 10 | 11 | Install and upgrade, if needed,`twine` with `python3 -m pip install -U twine`. 12 | The long description of the package will not render on PyPI unless an up-to-date 13 | version is used. 14 | 15 | ## Create the release 16 | 17 | - [ ] Update version numbers in 18 | - [ ] `bookstore/_version.py` (version_info) 19 | - [ ] `docs/source/conf.py` (version and release) 20 | - [ ] `docs/source/bookstore_api.yaml` (info.version) 21 | - [ ] Commit the updated version 22 | - [ ] Clean the repo of all non-tracked files: `git clean -xdfi` 23 | - [ ] Commit and tag the release 24 | 25 | ``` 26 | git commit -am"release $VERSION" 27 | git tag $VERSION 28 | ``` 29 | - [ ] Push the tags and remove any existing `dist` directory files 30 | 31 | ``` 32 | git push && git push --tags 33 | rm -rf dist/* 34 | ``` 35 | 36 | - [ ] Build `sdist` and `wheel` 37 | 38 | ``` 39 | python setup.py sdist 40 | python setup.py bdist_wheel 41 | ``` 42 | 43 | ## Test and upload release to PyPI 44 | 45 | - [ ] Test the wheel and sdist locally 46 | - [ ] Upload to PyPI using `twine` over SSL 47 | 48 | ``` 49 | twine upload dist/* 50 | ``` 51 | 52 | - [ ] If all went well: 53 | - Change `bookstore/_version.py` back to `.dev` 54 | - Push directly to `master` and push `--tags` too. 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, nteract 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /docs/source/configuration.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | Commonly used configuration settings can be stored in ``BookstoreSettings`` in the 5 | ``jupyter_notebook_config.py`` file. These settings include: 6 | 7 | - workspace location 8 | - published storage location 9 | - S3 bucket information 10 | - AWS credentials for S3 11 | 12 | Example configuration 13 | --------------------- 14 | 15 | Here's an example of ``BookstoreSettings`` in the ``~/.jupyter/jupyter_notebook_config.py`` file: 16 | 17 | .. code-block:: python 18 | 19 | """jupyter notebook configuration 20 | The location for user installs on MacOS is ``~/.jupyter/jupyter_notebook_config.py``. 21 | See https://jupyter.readthedocs.io/en/latest/projects/jupyter-directories.html for additional locations. 22 | """ 23 | from bookstore import BookstoreContentsArchiver 24 | 25 | 26 | c.NotebookApp.contents_manager_class = BookstoreContentsArchiver 27 | 28 | c.BookstoreSettings.workspace_prefix = "/workspace/kylek/notebooks" 29 | c.BookstoreSettings.published_prefix = "/published/kylek/notebooks" 30 | 31 | c.BookstoreSettings.s3_bucket = "" 32 | 33 | # If bookstore uses an EC2 instance with a valid IAM role, there is no need to specify here 34 | c.BookstoreSettings.s3_access_key_id = 35 | c.BookstoreSettings.s3_secret_access_key = 36 | 37 | 38 | The root directory of bookstore's GitHub repo contains an example config 39 | called ``jupyter_config.py.example`` that shows how to configure 40 | ``BookstoreSettings``. 41 | -------------------------------------------------------------------------------- /docs/source/reference/clone.rst: -------------------------------------------------------------------------------- 1 | Cloning 2 | ======= 3 | 4 | The ``clone`` module 5 | -------------------- 6 | 7 | .. module:: bookstore.clone 8 | 9 | .. autofunction:: build_notebook_model 10 | 11 | .. autofunction:: build_file_model 12 | 13 | .. autofunction:: validate_relpath 14 | 15 | ``BookstoreCloneHandler`` 16 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 17 | 18 | .. autoclass:: BookstoreCloneHandler 19 | 20 | Methods 21 | ^^^^^^^ 22 | 23 | .. automethod:: BookstoreCloneHandler.initialize 24 | 25 | .. automethod:: BookstoreCloneHandler.get 26 | 27 | .. automethod:: BookstoreCloneHandler.construct_template_params 28 | 29 | .. automethod:: BookstoreCloneHandler.get_template 30 | 31 | ``BookstoreCloneAPIHandler`` 32 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 33 | 34 | .. autoclass:: BookstoreCloneAPIHandler 35 | 36 | Methods 37 | ^^^^^^^ 38 | 39 | .. automethod:: BookstoreCloneAPIHandler.initialize 40 | 41 | .. automethod:: BookstoreCloneAPIHandler.post 42 | 43 | .. automethod:: BookstoreCloneAPIHandler.build_content_model 44 | 45 | .. automethod:: BookstoreCloneAPIHandler.build_post_response_model 46 | 47 | ``BookstoreFSCloneHandler`` 48 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 49 | 50 | Methods 51 | ^^^^^^^ 52 | 53 | .. automethod:: BookstoreFSCloneHandler.initialize 54 | 55 | .. automethod:: BookstoreFSCloneHandler.get 56 | 57 | .. automethod:: BookstoreFSCloneHandler.construct_template_params 58 | 59 | .. automethod:: BookstoreFSCloneHandler.get_template 60 | 61 | ``BookstoreFSCloneAPIHandler`` 62 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 63 | 64 | .. autoclass:: BookstoreFSCloneAPIHandler 65 | 66 | Methods 67 | ^^^^^^^ 68 | 69 | .. automethod:: BookstoreFSCloneAPIHandler.initialize 70 | 71 | .. automethod:: BookstoreFSCloneAPIHandler.post 72 | 73 | .. automethod:: BookstoreFSCloneAPIHandler.build_content_model 74 | -------------------------------------------------------------------------------- /ci/s3.js: -------------------------------------------------------------------------------- 1 | // Optional according to minio docs, included for AWS compat 2 | const regionName = "us-east-1"; 3 | 4 | const Minio = require("minio"); 5 | 6 | function makeBucket(minioClient, bucketName) { 7 | return new Promise((resolve, reject) => { 8 | minioClient.makeBucket(bucketName, regionName, err => { 9 | if (err) { 10 | // When using the ci script locally, the bucket typically already exists 11 | if (err.code === "BucketAlreadyOwnedByYou") { 12 | console.warn("Bucket already created"); 13 | resolve(); 14 | return; 15 | } 16 | 17 | reject(err); 18 | return; 19 | } 20 | resolve(); 21 | }); 22 | }); 23 | } 24 | 25 | function getObject(minioClient, bucketName, objectName) { 26 | return new Promise((resolve, reject) => 27 | minioClient.getObject(bucketName, objectName, (err, dataStream) => { 28 | if (err) { 29 | reject(err); 30 | return; 31 | } 32 | 33 | const chunks = []; 34 | dataStream.on("data", chunk => chunks.push(chunk)); 35 | dataStream.on("error", reject); 36 | dataStream.on("end", () => { 37 | resolve(Buffer.concat(chunks).toString("utf8")); 38 | }); 39 | }) 40 | ); 41 | } 42 | 43 | class Client { 44 | constructor(s3Config) { 45 | this.minioClient = new Minio.Client(s3Config); 46 | } 47 | 48 | async makeBucket(bucketName) { 49 | const bucketExists = await this.minioClient.bucketExists(bucketName); 50 | 51 | if (bucketExists) { 52 | return; 53 | } 54 | 55 | return await makeBucket(this.minioClient, bucketName); 56 | } 57 | 58 | async getObject(bucketName, objectName) { 59 | return getObject(this.minioClient, bucketName, objectName); 60 | } 61 | } 62 | 63 | module.exports = { 64 | Client 65 | }; 66 | -------------------------------------------------------------------------------- /bookstore/tests/client/client_fixtures.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from bookstore.client.nb_client import NotebookSession, KernelInfo, LiveNotebookRecord 4 | 5 | 6 | @pytest.fixture 7 | def notebook_server_dict(): 8 | notebook_server_dict = { 9 | "base_url": "/", 10 | "hostname": "localhost", 11 | "notebook_dir": "/Users/username", 12 | "password": False, 13 | "pid": 20981, 14 | "port": 8888, 15 | "secure": False, 16 | "token": "e5814788aeef225172364fcdf1240b90729169a2ced375c7", 17 | "url": "http://localhost:8888/", 18 | } 19 | return notebook_server_dict 20 | 21 | 22 | @pytest.fixture 23 | def notebook_server_record(notebook_server_dict): 24 | notebook_server_record = LiveNotebookRecord(**notebook_server_dict) 25 | return notebook_server_record 26 | 27 | 28 | @pytest.fixture(scope="module") 29 | def kernel_info_dict(): 30 | info_dict = { 31 | "id": 'f92b7c8b-0858-4d10-903c-b0631540fb36', 32 | "name": 'dev', 33 | "last_activity": '2019-03-14T23:38:08.137987Z', 34 | "execution_state": 'idle', 35 | "connections": 0, 36 | } 37 | return info_dict 38 | 39 | 40 | @pytest.fixture(scope="module") 41 | def kernel_info(kernel_info_dict): 42 | kernel_info = KernelInfo(**kernel_info_dict) 43 | return kernel_info 44 | 45 | 46 | @pytest.fixture 47 | def session_dict(kernel_info_dict): 48 | session_dict = { 49 | "id": '68d9c58f-c57d-4133-8b41-5ec2731b268d', 50 | "path": 'Untitled38.ipynb', 51 | "name": '', 52 | "type": 'notebook', 53 | "kernel": kernel_info_dict, 54 | "notebook": {'path': 'Untitled38.ipynb', 'name': ''}, # deprecated API 55 | } 56 | return session_dict 57 | 58 | 59 | @pytest.fixture 60 | def notebook_session(session_dict): 61 | notebook_session = NotebookSession(**session_dict) 62 | return notebook_session 63 | -------------------------------------------------------------------------------- /bookstore/tests/test_archive.py: -------------------------------------------------------------------------------- 1 | """Tests for archive""" 2 | import asyncio 3 | import pytest 4 | import json 5 | import logging 6 | 7 | from bookstore.archive import ArchiveRecord, BookstoreContentsArchiver 8 | from nbformat.v4 import new_notebook 9 | 10 | 11 | def test_create_contentsarchiver(): 12 | assert BookstoreContentsArchiver() 13 | 14 | 15 | def test_create_contentsarchiver_invalid_args_count(): 16 | with pytest.raises(TypeError): 17 | BookstoreContentsArchiver(42, True, 'hello') 18 | 19 | 20 | @pytest.mark.asyncio 21 | async def test_archive_failure_on_no_lock(): 22 | archiver = BookstoreContentsArchiver() 23 | assert archiver 24 | 25 | record = ArchiveRecord('my_notebook_path.ipynb', json.dumps(new_notebook()), 100.2) 26 | assert record 27 | 28 | await archiver.archive(record) 29 | 30 | 31 | @pytest.mark.asyncio 32 | async def test_archive_abort_with_lock(caplog): 33 | """Acquire a lock in advance so that when the archiver attempts to archive, it will abort.""" 34 | 35 | archiver = BookstoreContentsArchiver() 36 | record = ArchiveRecord('my_notebook_path.ipynb', json.dumps(new_notebook()), 100.2) 37 | 38 | lock = asyncio.Lock() 39 | archiver.path_locks['my_notebook_path.ipynb'] = lock 40 | async with lock: 41 | with caplog.at_level(logging.INFO): 42 | await archiver.archive(record) 43 | assert 'Skipping archive of my_notebook_path.ipynb' in caplog.text 44 | 45 | 46 | def test_pre_save_hook(): 47 | archiver = BookstoreContentsArchiver() 48 | model = {"type": "notebook", "content": new_notebook()} 49 | target_path = "my_notebook_path.ipynb" 50 | 51 | archiver.run_pre_save_hook(model, target_path) 52 | 53 | 54 | def test_pre_save_hook_bad_model(): 55 | archiver = BookstoreContentsArchiver() 56 | model = {"type": "file", "content": new_notebook()} 57 | target_path = "my_notebook_path.ipynb" 58 | 59 | archiver.run_pre_save_hook(model, target_path) 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | _build/ 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | 58 | # Flask stuff: 59 | instance/ 60 | .webassets-cache 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # Jupyter Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # SageMath parsed files 81 | *.sage.py 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | .venv 88 | venv/ 89 | ENV/ 90 | 91 | # Spyder project settings 92 | .spyderproject 93 | .spyproject 94 | 95 | # Rope project settings 96 | .ropeproject 97 | 98 | # mkdocs documentation 99 | /site 100 | 101 | # mypy 102 | .mypy_cache/ 103 | 104 | # pyCharm 105 | .idea 106 | 107 | # pytest 108 | .pytest_cache 109 | 110 | # Binder example 111 | binder/*run*.ipynb 112 | 113 | # Local jupyter notebook config for working on bookstore 114 | jupyter_notebook_config.py 115 | jupyter_config.py 116 | 117 | # node modules to ignore 118 | node_modules/ 119 | -------------------------------------------------------------------------------- /running_ci_locally.md: -------------------------------------------------------------------------------- 1 | # Local Continuous Integration 2 | 3 | It helps when developing to be able to run integration tests locally. Since 4 | bookstore relies on accessing S3, this requires that we create a local server 5 | that can model how S3 works. 6 | 7 | We will be using [minio](https://docs.minio.io/) to mock S3 behavior. 8 | 9 | ## Setup Local CI environment 10 | 11 | To run the ci tests locally, you will need to have a few things set up: 12 | 13 | - a functioning `docker` service 14 | - define `/mnt/data/` and `/mnt/config/` and give full permissions 15 | (e.g., `chmod 777 /mnt/data`). 16 | = add `/mnt/data` and `/mnt/config` to be accessible from `docker`. You can do 17 | so by modifying Docker's preferences by going to `Docker → Preferences → File Sharing` 18 | and adding `/mnt/data` and `/mnt/config` to the list there. 19 | - an up-to-date version of `node`. 20 | 21 | ## Run Local tests 22 | 23 | 1. Open two terminals with the current working directory as the root `bookstore` 24 | directory. 25 | 26 | 2. In one terminal run `yarn test:server`. This will start up minio. 27 | 28 | 3. In the other terminal run `yarn test`. This will run the integration tests. 29 | 30 | ## Interactive python tests 31 | 32 | The CI scripts are designed to be self-contained and run in an automated setup. This makes it 33 | makes it harder to iterate rapidly when you don't want to test the _entire_ system but when 34 | you do need to integrate with a Jupyter server. 35 | 36 | In addition the CI scripts, we have included `./ci/clone_request.py` for testing the clone 37 | endpoint. This is particularly useful for the `/api/bookstore/cloned` endpoint because while it 38 | is an API to be used by other applications, it also acts as a user facing endpoint since it 39 | provides a landing page for confirming whether or not a clone is to be approved. 40 | 41 | It's often difficult to judge whether what is being served makes sense from a UI perspective 42 | without being able to investigate it directly. At the same time we'll need to access it as an 43 | API to ensure that the responses are well-behaved from an API standpoint. By using python to 44 | query a live server and a browser to visit the landing page, we can rapidly iterate between 45 | the API and UI contexts from the same live server's endpoint. 46 | 47 | We provide examples of `jupyter notebook` commands needed in that file as well for both 48 | accessing the `nteract-notebooks` S3 bucket as well as the Minio provided `bookstore` bucket 49 | (as used by the CI scripts). 50 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Python CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-python/ for more details 4 | # 5 | version: 2 6 | jobs: 7 | build: 8 | docker: 9 | - image: circleci/python:3.6.8-node-browsers 10 | 11 | - image: minio/minio:RELEASE.2018-11-06T01-01-02Z 12 | command: server /data 13 | environment: 14 | MINIO_ACCESS_KEY: ONLY_ON_CIRCLE 15 | MINIO_SECRET_KEY: CAN_WE_DO_THIS 16 | ports: 9000:9000 17 | 18 | working_directory: ~/repo 19 | 20 | steps: 21 | - checkout 22 | 23 | # Download and cache dependencies 24 | - restore_cache: 25 | keys: 26 | - v2-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt"}} 27 | # fallback to using the latest cache if no exact match is found 28 | - v2-dependencies- 29 | 30 | - run: 31 | name: install dependencies and version 32 | command: | 33 | python3 -m venv venv 34 | . venv/bin/activate 35 | pip install --upgrade pip setuptools wheel 36 | pip install -r requirements.txt 37 | pip install -r requirements-dev.txt 38 | pip install . 39 | 40 | - run: 41 | name: check types 42 | command: | 43 | . venv/bin/activate 44 | mypy bookstore --ignore-missing-imports 45 | - run: 46 | name: python tests 47 | command: | 48 | . venv/bin/activate 49 | pytest -v --maxfail=2 bookstore/tests 50 | 51 | - save_cache: 52 | paths: 53 | - ./venv 54 | key: v2-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} 55 | 56 | - run: 57 | name: package up bookstore 58 | command: | 59 | . venv/bin/activate 60 | # Package up the package 61 | python setup.py sdist bdist_wheel 62 | 63 | - run: 64 | name: create virtual environment for packaged release 65 | command: | 66 | python3 -m venv venv_packaged_integration 67 | . venv_packaged_integration/bin/activate 68 | pip install --upgrade pip setuptools wheel 69 | pip install -U --force-reinstall dist/bookstore*.whl 70 | 71 | - run: 72 | name: integration tests 73 | command: | 74 | . venv_packaged_integration/bin/activate 75 | # Install the dependencies for our integration tester 76 | npm i 77 | node ci/integration.js 78 | 79 | -------------------------------------------------------------------------------- /bookstore/tests/client/test_nb_client.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from bookstore.client.nb_client import ( 4 | NotebookSession, 5 | KernelInfo, 6 | extract_kernel_id, 7 | LiveNotebookRecord, 8 | ) 9 | 10 | from bookstore.tests.client.client_fixtures import * 11 | 12 | 13 | def test_notebook_server_record(notebook_server_record, notebook_server_dict): 14 | assert notebook_server_record.base_url == notebook_server_dict['base_url'] 15 | assert notebook_server_record.hostname == notebook_server_dict["hostname"] 16 | assert notebook_server_record.notebook_dir == notebook_server_dict["notebook_dir"] 17 | assert notebook_server_record.password == notebook_server_dict['password'] 18 | assert notebook_server_record.pid == notebook_server_dict["pid"] 19 | assert notebook_server_record.port == notebook_server_dict["port"] 20 | assert notebook_server_record.secure == notebook_server_dict["secure"] 21 | assert notebook_server_record.token == notebook_server_dict["token"] 22 | assert notebook_server_record.url == notebook_server_dict["url"] 23 | assert notebook_server_record == LiveNotebookRecord(**notebook_server_dict) 24 | 25 | 26 | def test_kernel_info_class(kernel_info_dict, kernel_info): 27 | assert kernel_info.id == kernel_info_dict['id'] 28 | assert kernel_info.name == kernel_info_dict["name"] 29 | assert kernel_info.last_activity == kernel_info_dict["last_activity"] 30 | assert kernel_info.execution_state == kernel_info_dict['execution_state'] 31 | assert kernel_info.connections == kernel_info_dict["connections"] 32 | assert kernel_info == KernelInfo(**kernel_info_dict) 33 | 34 | 35 | def test_notebook_session_class(notebook_session, session_dict): 36 | assert notebook_session.path == session_dict["path"] 37 | assert notebook_session.name == session_dict["name"] 38 | assert notebook_session.type == session_dict['type'] 39 | assert notebook_session.kernel == KernelInfo(**session_dict["kernel"]) 40 | assert notebook_session.notebook == session_dict["notebook"] 41 | assert notebook_session == NotebookSession(**session_dict) 42 | 43 | 44 | @pytest.mark.parametrize( 45 | "connection_file, expected_kernel_id", 46 | [ 47 | ( 48 | "kernel-f92b7c8b-0858-4d10-903c-b0631540fb36.json", 49 | "f92b7c8b-0858-4d10-903c-b0631540fb36", 50 | ), 51 | ( 52 | "kernel-ee2b7c8b-0858-4d10-903c-b0631540fb36.json", 53 | "ee2b7c8b-0858-4d10-903c-b0631540fb36", 54 | ), 55 | ], 56 | ) 57 | def test_extract_kernel_id(connection_file, expected_kernel_id): 58 | assert extract_kernel_id(connection_file) == expected_kernel_id 59 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = true 3 | envlist = clean, py{36,37}, flake8, black, mypy, manifest, docs 4 | 5 | # Linters 6 | [testenv:flake8] 7 | skip_install = true 8 | deps = flake8 9 | commands = flake8 bookstore --count --ignore=E,C,W,F401,F403,F811,F821,F841,E402,I100,I101,D400 --max-complexity=23 --max-line-length=104 --show-source --statistics 10 | 11 | [testenv:black] 12 | skip_install = true 13 | deps = black 14 | commands = black --check --verbose . 15 | 16 | # Typing 17 | [testenv:mypy] 18 | skip_install = true 19 | deps = mypy==0.660 20 | commands = mypy bookstore --ignore-missing-imports 21 | 22 | # Manifest 23 | [testenv:manifest] 24 | skip_install = true 25 | deps = check-manifest 26 | commands = check-manifest 27 | 28 | # Docs 29 | [testenv:docs] 30 | description = invoke sphinx-build to build the HTML docs 31 | deps = .[docs] 32 | commands = 33 | sphinx-build -d "{toxworkdir}/docs_doctree" docs/source "{toxworkdir}/docs_out" --color -bhtml {posargs} 34 | python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' 35 | 36 | # Distro 37 | [testenv:dist] 38 | skip_install = true 39 | setenv = 40 | SKIP_PIP_CHECK = 1 41 | # Have to use /bin/bash or the `*` will cause that argument to get quoted by the tox command line... 42 | commands = 43 | python setup.py sdist --dist-dir={distdir} bdist_wheel --dist-dir={distdir} 44 | /bin/bash -c 'python -m pip install -U --force-reinstall {distdir}/bookstore*.whl' 45 | /bin/bash -c 'python -m pip install -U --force-reinstall --no-deps {distdir}/bookstore*.tar.gz' 46 | 47 | [testenv] 48 | # disable Python's hash randomization for tests that stringify dicts, etc 49 | setenv = 50 | PYTHONHASHSEED = 0 51 | AWS_ACCESS_KEY_ID=foobar_key 52 | AWS_SECRET_ACCESS_KEY=foobar_secret 53 | passenv = 54 | * 55 | TOXENV 56 | CI 57 | TRAVIS 58 | TRAVIS_* 59 | CODECOV_* 60 | basepython = 61 | py36: python3.6 62 | py37: python3.7 63 | flake8: python3.6 64 | black: python3.6 65 | mypy: python3.6 66 | manifest: python3.6 67 | dist: python3.6 68 | docs: python3.6 69 | clean: python3.6 70 | deps = 71 | .[test] 72 | pytest 73 | depends = 74 | py36: clean 75 | commands = 76 | pytest -v --maxfail=2 --cov-config=.coveragerc --cov=bookstore -W always bookstore/tests/ 77 | py36: coverage report 78 | py36: coverage html 79 | commands_post = 80 | py36: codecov -e TOXENV 81 | 82 | [testenv:clean] 83 | deps = coverage 84 | skip_install = true 85 | commands = coverage erase 86 | 87 | [pytest] 88 | python_files = test_*.py 89 | env = 90 | AWS_SECRET_ACCESS_KEY=foobar_secret 91 | AWS_ACCESS_KEY_ID=foobar_key 92 | markers = 93 | asyncio -------------------------------------------------------------------------------- /ci/clone_request.py: -------------------------------------------------------------------------------- 1 | """ Helper for interactive debug of cloning endpoint. 2 | 3 | This module is intended to be used to interactively when debugging the 4 | bookstore cloning endpoint. 5 | 6 | To use and test this, you'll need to separately run a Jupyter server. 7 | 8 | Example Usage 9 | ------------- 10 | 11 | To test the 'nteract-notebooks' bucket on S3 you'll need to run a command like 12 | the following:: 13 | 14 | jupyter notebook --NotebookApp.allow_origin="*" --NotebookApp.disable_check_xsrf=True \ 15 | --no-browser --NotebookApp.token="" \ 16 | --BookstoreSettings.s3_bucket=nteract-notebooks 17 | 18 | To test the 'bookstore' bucket created with minio you'll need to run a command 19 | like the following:: 20 | 21 | jupyter notebook --NotebookApp.allow_origin="*" --NotebookApp.disable_check_xsrf=True \ 22 | --no-browser --NotebookApp.token="" \ 23 | --BookstoreSettings.s3_endpoint_url="http://localhost:9000" \ 24 | --BookstoreSettings.s3_access_key_id="ONLY_ON_CIRCLE" \ 25 | --BookstoreSettings.s3_secret_access_key="CAN_WE_DO_THIS" 26 | 27 | Additional examples 28 | ------------------- 29 | 30 | See the docstring under the ``if __name__`` statement below. 31 | """ 32 | import nbformat 33 | import pprint 34 | import requests 35 | 36 | 37 | def get(queries): 38 | return requests.get(f"http://localhost:8888/api/bookstore/cloned{queries}") 39 | 40 | 41 | def post(**kwargs): 42 | return requests.post("http://localhost:8888/api/bookstore/cloned", json={**kwargs}) 43 | 44 | 45 | if __name__ == "__main__": 46 | """ 47 | Examples 48 | -------- 49 | 50 | # tests s3 hosted 'nteract-notebooks' bucket 51 | response = get("?s3_bucket=nteract-notebooks&s3_key=published/whateverwewant.json") 52 | print(response.content) 53 | response = get("/?s3_bucket=nteract-notebooks&s3_key=published/whateverwewant.json") 54 | print(response.content) 55 | response = get("?s3_bucket=nteract-notebooks&s3_key=Introduction_to_Chainer.ipynb") 56 | print(response.content) 57 | 58 | 59 | # tests minio created 'bookstore' bucket 60 | response = get("?s3_bucket=bookstore&s3_key=ci-published/ci-published.ipynb") 61 | print(response.content) 62 | response = post(s3_bucket="nteract-notebooks", s3_key="Introduction_to_Chainer.ipynb") 63 | print(response.content) 64 | 65 | Changing the requests will enable you to test different situations and 66 | analyze the responses. 67 | """ 68 | response = post(s3_bucket="bookstore", s3_key="ci-published/ci-published.ipynb") 69 | pprint.pprint(response.json()) 70 | response = post( 71 | s3_bucket="bookstore", 72 | s3_key="ci-published/ci-published.ipynb", 73 | target_path="published/this_is_my_target_path.ipynb", 74 | ) 75 | pprint.pprint(response.json()) 76 | -------------------------------------------------------------------------------- /bookstore/client/store_client.py: -------------------------------------------------------------------------------- 1 | """Client to interact with a notebook's bookstore functionality.""" 2 | import requests 3 | 4 | from .nb_client import CurrentNotebookClient 5 | 6 | 7 | class BookstoreClient(CurrentNotebookClient): 8 | """EXPERIMENTAL SUPPORT: A client that allows access to a Bookstore from within a notebook. 9 | 10 | Parameters 11 | ---------- 12 | s3_bucket: str 13 | (optional) Provide a default bucket for this bookstore client to clone from. 14 | 15 | 16 | Attributes 17 | ---------- 18 | default_bucket : str 19 | The default bucket to be used for cloning. 20 | """ 21 | 22 | def __init__(self, s3_bucket=None): 23 | if s3_bucket: 24 | self.default_bucket = s3_bucket 25 | super().__init__() 26 | 27 | @property 28 | def publish_endpoint(self): 29 | """Helper to refer to construct the publish endpoint for this notebook server.""" 30 | api_endpoint = "/api/bookstore/publish/" 31 | return f"{self.url}{api_endpoint}" 32 | 33 | def publish(self, path=None): 34 | """Publish notebook to bookstore 35 | 36 | Parameters 37 | ---------- 38 | path : str 39 | (optional) Path that you wish to publish; defaults to current notebook. 40 | s3_object_key: str 41 | The the path we wish to clone to. 42 | """ 43 | if path is None: 44 | path = self.session.path 45 | nb_json = self.get_contents(path)['content'] 46 | json_body = {"type": "notebook", "content": nb_json} 47 | 48 | target_url = f"{self.publish_endpoint}{path}" 49 | 50 | response = self.req_session.put(target_url, json=json_body) 51 | return response 52 | 53 | @property 54 | def clone_endpoint(self): 55 | """Helper to refer to construct the clone endpoint for this notebook server.""" 56 | api_endpoint = "/api/bookstore/clone/" 57 | return f"{self.url}{api_endpoint}" 58 | 59 | def clone(self, s3_bucket="", s3_key="", target_path=""): 60 | """Clone files via bookstore. 61 | 62 | Parameters 63 | ---------- 64 | s3_bucket : str 65 | (optional) S3 bucket you wish to clone from; defaults to client's bucket. 66 | s3_object_key: str 67 | The object key describing the object you wish to clone from S3. 68 | target_path: str 69 | (optional) The location you wish to clone the object to; defaults to s3_object_key. 70 | """ 71 | s3_bucket = s3_bucket or self.default_bucket 72 | json_body = {"s3_bucket": s3_bucket, "s3_key": s3_key, "target_path": target_path} 73 | # TODO: Add a check for success 74 | response = self.req_session.post(f"{self.clone_endpoint}", json=json_body) 75 | return response 76 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """"setup.py 4 | 5 | Note: Do a Python check 6 | 7 | See: 8 | https://packaging.python.org/en/latest/distributing.html 9 | https://github.com/pypa/sampleproject 10 | """ 11 | from __future__ import print_function 12 | 13 | import os 14 | import sys 15 | from io import open 16 | from os import path 17 | 18 | from setuptools import setup 19 | 20 | v = sys.version_info 21 | 22 | if v[:2] < (3, 6): 23 | print('ERROR: Bookstore requires Python 3.6 or higher', file=sys.stderr) 24 | sys.exit(1) 25 | 26 | # We have the correct Python version, proceed. 27 | 28 | pjoin = os.path.join 29 | 30 | here = path.abspath(path.dirname(__file__)) 31 | 32 | # Get the long description from the README file 33 | with open(pjoin(here, 'README.md'), encoding='utf-8') as f: 34 | long_description = f.read() 35 | 36 | # Get the bookstore version 37 | ns = {} 38 | with open(pjoin(here, 'bookstore', '_version.py')) as f: 39 | exec(f.read(), {}, ns) 40 | 41 | 42 | target_dir = pjoin("etc", "jupyter", "jupyter_notebook_config.d") 43 | config_files = [pjoin("jupyter_config", "jupyter_notebook_config.d", "bookstore.json")] 44 | data_files = [(target_dir, config_files)] 45 | 46 | setup( 47 | name='bookstore', 48 | version=ns['__version__'], 49 | description='Storage Workflows for Notebooks', 50 | long_description=long_description, 51 | long_description_content_type='text/markdown', 52 | url='https://github.com/nteract/bookstore', 53 | author='nteract contributors', 54 | author_email='nteract@googlegroups.com', 55 | license='BSD', 56 | classifiers=[ 57 | 'Development Status :: 4 - Beta', 58 | 'Intended Audience :: Developers', 59 | 'Intended Audience :: System Administrators', 60 | 'Intended Audience :: Science/Research', 61 | 'License :: OSI Approved :: BSD License', 62 | 'Programming Language :: Python', 63 | 'Programming Language :: Python :: 3', 64 | ], 65 | # Note that this is a string of words separated by whitespace, not a list. 66 | keywords='jupyter storage nteract notebook', 67 | packages=['bookstore'], 68 | include_package_data=True, 69 | install_requires=[ 70 | 'future', 71 | 'futures ; python_version < "3.0"', 72 | 'ipython >= 5.0', 73 | 'notebook', 74 | 'tornado >= 5.1.1', 75 | 'aiobotocore', 76 | 'aioboto3', 77 | ], 78 | extras_require={ 79 | 'docs': ['sphinx', 'm2r', 'sphinxcontrib-napoleon', 'sphinxcontrib-openapi'], 80 | 'test': [ 81 | 'codecov', 82 | 'coverage', 83 | 'mock', 84 | 'mypy==0.660', 85 | 'pytest>=3.3', 86 | 'pytest-asyncio', 87 | 'pytest-cov', 88 | 'pytest-mock', 89 | 'black', 90 | 'tox', 91 | ], 92 | }, 93 | data_files=data_files, 94 | entry_points={}, 95 | project_urls={ 96 | 'Documentation': 'https://bookstore.readthedocs.io', 97 | 'Funding': 'https://nteract.io', 98 | 'Source': 'https://github.com/nteract/bookstore/', 99 | 'Tracker': 'https://github.com/nteract/bookstore/issues', 100 | }, 101 | ) 102 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities. 4 | 5 | We are committed to making participation in this project a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | Examples of behavior that contributes to creating a positive environment include: 8 | 9 | - Using welcoming and inclusive language 10 | - Being respectful of differing viewpoints and experiences 11 | - Gracefully accepting constructive criticism 12 | - Focusing on what is best for the community 13 | - Showing empathy towards other community members 14 | 15 | Examples of unacceptable behavior by participants include: 16 | 17 | - The use of sexualized language or imagery and unwelcome sexual attention or advances 18 | - Trolling, insulting/derogatory comments, and personal or political attacks 19 | - Public or private harassment 20 | - Publishing others’ private information, such as a physical or electronic address, without explicit permission 21 | - Other conduct which could reasonably be considered inappropriate in a professional setting 22 | 23 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 24 | 25 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 26 | 27 | By adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently applying these principles to every aspect of managing this project. Project maintainers who do not follow or enforce the Code of Conduct may be permanently removed from the project team. 28 | 29 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 30 | 31 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project maintainer at [rgbkrk@gmail.com]. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. Maintainers are obligated to maintain confidentiality with regard to the reporter of an incident. 32 | 33 | This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available from http://contributor-covenant.org/version/1/4/ 34 | -------------------------------------------------------------------------------- /docs/source/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | Data scientists and notebook users may develop locally on their system or save 5 | their notebooks to off-site or cloud storage. Additionally, they will often 6 | create a notebook and then over time make changes and update it. As they work, 7 | it's helpful to be able to **store versions** of a notebook. When making changes 8 | to the content and calculations over time, a data scientist using Bookstore can 9 | now request different versions from the remote storage, such as S3, and 10 | **clone** the notebook to their local system. 11 | 12 | .. note:: **store and clone** 13 | 14 | *store* 15 | 16 | User saves to Local System ------------------> Remote Data Store (i.e. S3) 17 | 18 | 19 | *clone* 20 | 21 | User requests a notebook to use locally <-------------- Remote Data Store (i.e. S3) 22 | 23 | 24 | After some time working with a notebook, the data scientist may want to save or 25 | share a polished notebook version with others. By **publishing a notebook**, the 26 | data scientist can display and share work that others can use at a later time. 27 | 28 | How to store and clone versions 29 | ------------------------------- 30 | 31 | Bookstore uses automatic notebook version management and specific storage paths 32 | when storing a notebook. 33 | 34 | Automatic notebook version management 35 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 36 | 37 | Every *save* of a notebook creates an *immutable copy* of the notebook on object 38 | storage. Initially, Bookstore supports S3 for object storage. 39 | 40 | To simplify implementation and management of versions, we currently rely on S3 41 | as the object store using `versioned buckets 42 | `_. When a 43 | notebook is saved, it overwrites the existing file in place using the versioned 44 | s3 buckets to handle the versioning. 45 | 46 | Storage paths 47 | ~~~~~~~~~~~~~ 48 | 49 | All notebooks are archived to a single versioned S3 bucket using specific 50 | **prefixes** to denote a user's workspace and an organization's publication of a 51 | user's notebook. This captures the lifecycle of the notebook on storage. To do 52 | this, bookstore allows users to set workspace and published storage paths. For 53 | example: 54 | 55 | - ``/workspace`` - where users edit and store notebooks 56 | - ``/published`` - notebooks to be shared to an organization 57 | 58 | Bookstore archives notebook versions by keeping the path intact (until a user 59 | changes them). For example, the prefixes that could be associated with storage 60 | types: 61 | 62 | - Notebook in "draft" form: ``/workspace/kylek/notebooks/mine.ipynb`` 63 | - Most recent published copy of a notebook: ``/published/kylek/notebooks/mine.ipynb`` 64 | 65 | .. note:: *Scheduling (Planned for a future release)* 66 | 67 | When scheduling execution of notebooks, each notebook path is a namespace 68 | that an external service can access. This helps when working with 69 | parameterized notebooks, such as with Papermill. Scheduled notebooks may 70 | also be referred to by the notebook ``key``. In addition, Bookstore can 71 | find version IDs as well. 72 | 73 | Easing the transition to Bookstore's storage plan 74 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 75 | 76 | Since many people use a regular filesystem, we'll start with writing to the 77 | ``/workspace`` prefix as Archival Storage (more specifically, writing on save 78 | using a ``post_save_hook`` for the Jupyter contents manager). 79 | 80 | How to publish a notebook 81 | ------------------------- 82 | 83 | To publish a notebook, Bookstore uses a publishing endpoint which is a 84 | ``serverextension`` to the classic Jupyter server. If you wish to publish 85 | notebooks, explicitly enable bookstore as a server extension to use the 86 | endpoint. By default, publishing is not enabled. 87 | 88 | To enable the extension globally, run:: 89 | 90 | jupyter serverextension enable --py bookstore 91 | 92 | If you wish to enable it only for your current environment, run:: 93 | 94 | jupyter serverextension enable --py bookstore --sys-prefix 95 | -------------------------------------------------------------------------------- /bookstore/tests/test_bookstore_config.py: -------------------------------------------------------------------------------- 1 | """Tests for bookstore config""" 2 | import logging 3 | 4 | import pytest 5 | 6 | from bookstore.bookstore_config import BookstoreSettings, validate_bookstore 7 | 8 | 9 | def test_validate_bookstore_defaults(): 10 | """Tests that all bookstore validates with default values.""" 11 | expected = { 12 | "bookstore_valid": False, 13 | "publish_valid": False, 14 | "archive_valid": False, 15 | "s3_clone_valid": True, 16 | "fs_clone_valid": False, 17 | } 18 | settings = BookstoreSettings() 19 | assert validate_bookstore(settings) == expected 20 | 21 | 22 | def test_validate_bookstore_published(): 23 | """Tests that bookstore does not validate with an empty published_prefix.""" 24 | expected = { 25 | "bookstore_valid": True, 26 | "publish_valid": False, 27 | "archive_valid": True, 28 | "s3_clone_valid": True, 29 | "fs_clone_valid": False, 30 | } 31 | settings = BookstoreSettings(s3_bucket="A_bucket", published_prefix="") 32 | assert validate_bookstore(settings) == expected 33 | 34 | 35 | def test_validate_bookstore_workspace(): 36 | """Tests that bookstore does not validate with an empty workspace_prefix.""" 37 | expected = { 38 | "bookstore_valid": True, 39 | "publish_valid": True, 40 | "archive_valid": False, 41 | "s3_clone_valid": True, 42 | "fs_clone_valid": False, 43 | } 44 | settings = BookstoreSettings(s3_bucket="A_bucket", workspace_prefix="") 45 | assert validate_bookstore(settings) == expected 46 | 47 | 48 | def test_validate_bookstore_endpoint(): 49 | """Tests that bookstore does not validate with an empty s3_endpoint_url.""" 50 | expected = { 51 | "bookstore_valid": False, 52 | "publish_valid": False, 53 | "archive_valid": False, 54 | "s3_clone_valid": True, 55 | "fs_clone_valid": False, 56 | } 57 | settings = BookstoreSettings(s3_endpoint_url="") 58 | assert validate_bookstore(settings) == expected 59 | 60 | 61 | def test_validate_bookstore_bucket(): 62 | """Tests that bookstore features validate with an s3_bucket.""" 63 | expected = { 64 | "bookstore_valid": True, 65 | "publish_valid": True, 66 | "archive_valid": True, 67 | "s3_clone_valid": True, 68 | "fs_clone_valid": False, 69 | } 70 | settings = BookstoreSettings(s3_bucket="A_bucket") 71 | assert validate_bookstore(settings) == expected 72 | 73 | 74 | def test_disable_cloning(): 75 | """Tests that cloning from s3_bucket can be disabled.""" 76 | expected = { 77 | "bookstore_valid": True, 78 | "publish_valid": True, 79 | "archive_valid": True, 80 | "s3_clone_valid": False, 81 | "fs_clone_valid": False, 82 | } 83 | settings = BookstoreSettings(s3_bucket="A_bucket", enable_s3_cloning=False) 84 | assert validate_bookstore(settings) == expected 85 | 86 | 87 | def test_enable_fs_cloning(): 88 | """Tests that file system cloning works even if s3 cloning is disabled.""" 89 | expected = { 90 | "bookstore_valid": False, 91 | "publish_valid": False, 92 | "archive_valid": False, 93 | "s3_clone_valid": False, 94 | "fs_clone_valid": True, 95 | } 96 | settings = BookstoreSettings(enable_s3_cloning=False, fs_cloning_basedir="/Users/bookstore") 97 | assert validate_bookstore(settings) == expected 98 | 99 | 100 | def test_relative_basepath(caplog): 101 | """Tests that file system cloning works even if s3 cloning is disabled.""" 102 | expected = { 103 | "bookstore_valid": False, 104 | "publish_valid": False, 105 | "archive_valid": False, 106 | "s3_clone_valid": False, 107 | "fs_clone_valid": False, 108 | } 109 | fs_cloning_basedir = "Users/jupyter" 110 | settings = BookstoreSettings(enable_s3_cloning=False, fs_cloning_basedir=fs_cloning_basedir) 111 | with caplog.at_level(logging.INFO): 112 | actual = validate_bookstore(settings) 113 | assert actual == expected 114 | assert f"{fs_cloning_basedir} is not an absolute path," in caplog.text 115 | -------------------------------------------------------------------------------- /bookstore/clone.html: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | Clone Landing page 13 | 61 | 62 | 104 | 105 | 106 | 107 |
108 |

Clone Confirmation

109 |

Preparing to copy the following file to your notebook's working directory:

110 | 
{{ source_description }}
111 |

Only copy code from locations that you recognize and trust.

112 | 113 | 116 |
117 | 118 | 119 | -------------------------------------------------------------------------------- /bookstore/tests/test_publish.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import json 3 | 4 | from unittest.mock import Mock 5 | 6 | import pytest 7 | 8 | from bookstore.publish import BookstorePublishAPIHandler 9 | from nbformat.v4 import new_notebook 10 | from tornado.testing import AsyncTestCase, gen_test 11 | from tornado.web import Application, HTTPError 12 | from tornado.httpserver import HTTPRequest 13 | from traitlets.config import Config 14 | 15 | 16 | def test_create_publish_handler_no_params(): 17 | with pytest.raises(TypeError): 18 | assert BookstorePublishAPIHandler() 19 | 20 | 21 | class TestPublishAPIHandler(AsyncTestCase): 22 | def setUp(self): 23 | super().setUp() 24 | mock_settings = { 25 | "BookstoreSettings": { 26 | "s3_access_key_id": "mock_id", 27 | "s3_secret_access_key": "mock_access", 28 | "s3_bucket": "my_bucket", 29 | "published_prefix": "custom_prefix", 30 | } 31 | } 32 | config = Config(mock_settings) 33 | 34 | self.mock_application = Mock( 35 | spec=Application, ui_methods={}, ui_modules={}, settings={"config": config} 36 | ) 37 | 38 | def put_handler(self, uri, body_dict=None, app=None): 39 | if body_dict is None: 40 | body_dict = {} 41 | if app is None: 42 | app = self.mock_application 43 | connection = Mock(context=Mock(protocol="https")) 44 | body = json.dumps(body_dict).encode('utf-8') 45 | payload_request = HTTPRequest( 46 | method='PUT', 47 | uri=uri, 48 | headers={"Host": "localhost:8888"}, 49 | body=body, 50 | connection=connection, 51 | ) 52 | return BookstorePublishAPIHandler(app, payload_request) 53 | 54 | @gen_test 55 | async def test_put_no_path(self): 56 | no_path_handler = self.put_handler('/bookstore/publish/') 57 | with pytest.raises(HTTPError): 58 | await no_path_handler.put('') 59 | 60 | @gen_test 61 | async def test_put_no_body(self): 62 | no_body_handler = self.put_handler('/bookstore/publish/hi') 63 | with pytest.raises(HTTPError): 64 | await no_body_handler.put('hi') 65 | 66 | @gen_test 67 | async def test_put_s3_error(self): 68 | """this test includes a valid body so that we get to the s3 part of our system""" 69 | body_dict = {'content': new_notebook(), 'type': "notebook"} 70 | ok_body_handler = self.put_handler('/bookstore/publish/hi', body_dict=body_dict) 71 | with pytest.raises(HTTPError): 72 | await ok_body_handler.put('hi') 73 | 74 | def test_prepare_response(self): 75 | expected = {"s3_path": "s3://my_bucket/custom_prefix/mylocal/path", "versionID": "eeeeAB"} 76 | empty_handler = self.put_handler('/bookstore/publish/hi') 77 | actual = empty_handler.prepare_response( 78 | {"VersionId": "eeeeAB"}, "s3://my_bucket/custom_prefix/mylocal/path" 79 | ) 80 | assert actual == expected 81 | 82 | def test_validate_model_no_type(self): 83 | body_dict = {'content': {}} 84 | empty_handler = self.put_handler('/bookstore/publish/hi') 85 | with pytest.raises(HTTPError): 86 | empty_handler.validate_model(body_dict) 87 | 88 | def test_validate_model_wrong_type(self): 89 | body_dict = {'content': {}, 'type': "file"} 90 | empty_handler = self.put_handler('/bookstore/publish/hi') 91 | with pytest.raises(HTTPError): 92 | empty_handler.validate_model(body_dict) 93 | 94 | def test_validate_model_empty_content(self): 95 | body_dict = {'content': {}, 'type': "notebook"} 96 | empty_handler = self.put_handler('/bookstore/publish/hi') 97 | with pytest.raises(HTTPError): 98 | empty_handler.validate_model(body_dict) 99 | 100 | def test_validate_model_bad_notebook(self): 101 | bad_notebook = new_notebook() 102 | bad_notebook['other_field'] = "hello" 103 | body_dict = {'content': bad_notebook, 'type': "notebook"} 104 | empty_handler = self.put_handler('/bookstore/publish/hi') 105 | with pytest.raises(HTTPError): 106 | empty_handler.validate_model(body_dict) 107 | 108 | def test_validate_model_good_notebook(self): 109 | body_dict = {'content': new_notebook(), 'type': "notebook"} 110 | empty_handler = self.put_handler('/bookstore/publish/hi') 111 | empty_handler.validate_model(body_dict) 112 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bookstore :books: 2 | 3 | [![Documentation Status](https://readthedocs.org/projects/bookstore/badge/?version=latest)](https://bookstore.readthedocs.io/en/latest/?badge=latest) 4 | [![Build Status](https://travis-ci.org/nteract/bookstore.svg?branch=master)](https://travis-ci.org/nteract/bookstore) 5 | [![CircleCI](https://circleci.com/gh/nteract/bookstore.svg?style=shield)](https://circleci.com/gh/nteract/bookstore) 6 | [![Codecov](https://codecov.io/gh/nteract/bookstore/branch/master/graph/badge.svg)](https://codecov.io/gh/nteract/bookstore) 7 | 8 | **bookstore** :books: provides tooling and workflow recommendations for storing :cd:, scheduling :calendar:, and publishing :book: notebooks. 9 | 10 | The full documentation is hosted on [ReadTheDocs](https://bookstore.readthedocs.io). 11 | 12 | ## How does bookstore work 13 | 14 | ### Automatic Notebook Versioning 15 | 16 | Every *save* of a notebook creates an *immutable copy* of the notebook on object storage. 17 | 18 | To simplify implementation, we currently rely on S3 as the object store, using [versioned buckets](https://docs.aws.amazon.com/AmazonS3/latest/dev/Versioning.html). 19 | 20 | 25 | 26 | ### Storage Paths 27 | 28 | All notebooks are archived to a single versioned S3 bucket with specific prefixes denoting the lifecycle of the notebook: 29 | 30 | - `/workspace` - where users edit 31 | - `/published` - public notebooks (to an organization) 32 | 33 | Each notebook path is a namespace that an external service ties into the schedule. We archive off versions, keeping the path intact (until a user changes them). 34 | 35 | | Prefix | Intent | 36 | |-----------------------------------------|------------------------| 37 | | `/workspace/kylek/notebooks/mine.ipynb` | Notebook in “draft” | 38 | | `/published/kylek/notebooks/mine.ipynb` | Current published copy | 39 | 40 | Scheduled notebooks will also be referred to by the notebook key. In addition, we'll need to be able to surface version IDs as well. 41 | 42 | ### Transitioning to this Storage Plan 43 | 44 | Since most people are on a regular filesystem, we'll start with writing to the 45 | `/workspace` prefix as Archival Storage (writing on save using a `post_save_hook` 46 | for a Jupyter contents manager). 47 | 48 | ### Publishing 49 | 50 | The bookstore publishing endpoint is a `serverextension` to the classic Jupyter 51 | server. This means you will need to explicitly enable the `serverextension` 52 | to use the endpoint. 53 | 54 | To do so, run: 55 | 56 | jupyter serverextension enable --py bookstore 57 | 58 | To enable it only for the current environment, run: 59 | 60 | jupyter serverextension enable --py bookstore --sys-prefix 61 | 62 | ## Installation 63 | 64 | **bookstore** requires Python 3.6 or higher. 65 | 66 | Note: Supports installation on Jupyter servers running Python 3.6 and above. 67 | Your notebooks can still be run in Python 2 or Python 3. 68 | 69 | 1. Clone this repo. 70 | 2. At the repo's root, enter in the Terminal: `python3 -m pip install .` (Tip: don't forget the dot at the end of the command) 71 | 72 | ## Configuration 73 | 74 | ```python 75 | # jupyter config 76 | # At ~/.jupyter/jupyter_notebook_config.py for user installs on macOS 77 | # See https://jupyter.readthedocs.io/en/latest/projects/jupyter-directories.html for other places to plop this 78 | 79 | from bookstore import BookstoreContentsArchiver 80 | 81 | c.NotebookApp.contents_manager_class = BookstoreContentsArchiver 82 | 83 | # All Bookstore settings are centralized on one config object so you don't have to configure it for each class 84 | c.BookstoreSettings.workspace_prefix = "/workspace/kylek/notebooks" 85 | c.BookstoreSettings.published_prefix = "/published/kylek/notebooks" 86 | 87 | c.BookstoreSettings.s3_bucket = "" 88 | 89 | # Note: if bookstore is used from an EC2 instance with the right IAM role, you don't 90 | # have to specify these 91 | c.BookstoreSettings.s3_access_key_id = 92 | c.BookstoreSettings.s3_secret_access_key = 93 | ``` 94 | 95 | ## Developing 96 | 97 | If you are developing on bookstore you will want to run the ci tests locally and to make releases. 98 | 99 | Use [CONTRIBUTING.md](./CONTRIBUTING.md) to learn more about contributing. 100 | Use [running_ci_locally.md](./running_ci_locally.md) to learn more about running ci tests locally. 101 | Use [running_python_tests.md](./running_python_tests.md) to learn about running tests locally. 102 | Use [RELEASING.md](./RELEASING.md) to learn more about releasing bookstore. 103 | -------------------------------------------------------------------------------- /bookstore/bookstore_config.py: -------------------------------------------------------------------------------- 1 | """Configuration settings for bookstore.""" 2 | import logging 3 | from pathlib import Path 4 | 5 | from traitlets import Integer, Unicode, Bool 6 | from traitlets.config import LoggingConfigurable 7 | 8 | log = logging.getLogger('bookstore_config') 9 | 10 | 11 | class BookstoreSettings(LoggingConfigurable): 12 | """Configuration for archival and publishing. 13 | 14 | Settings include storage directory locations, S3 authentication, 15 | additional S3 settings, and Bookstore resources. 16 | 17 | S3 authentication settings can be set, or they can be left unset when 18 | IAM is used. 19 | 20 | Like the Jupyter notebook, bookstore uses traitlets to handle 21 | configuration, loading from files or CLI. 22 | 23 | Attributes 24 | ---------- 25 | workspace_prefix : str(``workspace``) 26 | Directory to use for user workspace storage 27 | published_prefix : str(``published``) 28 | Directory to use for published notebook storage 29 | s3_access_key_id : str, optional 30 | Environment variable ``JPYNB_S3_ACCESS_KEY_ID`` 31 | s3_secret_access_key : str, optional 32 | Environment variable ``JPYNB_S3_SECRET_ACCESS_KEY`` 33 | s3_endpoint_url : str(``"https://s3.amazonaws.com"``) 34 | Environment variable ``JPYNB_S3_ENDPOINT_URL`` 35 | s3_region_name : str(``"us-east-1"``) 36 | Environment variable ``JPYNB_S3_REGION_NAME`` 37 | s3_bucket : str(``""``) 38 | Bucket name, environment variable ``JPYNB_S3_BUCKET`` 39 | max_threads : int(``16``) 40 | Maximum threads from the threadpool available for S3 read/writes 41 | enable_s3_cloning : bool(``True``) 42 | Enable cloning from s3. 43 | fs_cloning_basedir : str(``"/Users/jupyter"``) 44 | Absolute path to base directory used to clone from the local file system 45 | 46 | """ 47 | 48 | workspace_prefix = Unicode("workspace", help="Prefix for the live workspace notebooks").tag( 49 | config=True 50 | ) 51 | published_prefix = Unicode("published", help="Prefix for published notebooks").tag(config=True) 52 | enable_s3_cloning = Bool(True, help="Enable cloning from s3.").tag(config=True) 53 | 54 | s3_access_key_id = Unicode( 55 | help="S3/AWS access key ID", allow_none=True, default_value=None 56 | ).tag(config=True, env="JPYNB_S3_ACCESS_KEY_ID") 57 | s3_secret_access_key = Unicode( 58 | help="S3/AWS secret access key", allow_none=True, default_value=None 59 | ).tag(config=True, env="JPYNB_S3_SECRET_ACCESS_KEY") 60 | 61 | s3_endpoint_url = Unicode("https://s3.amazonaws.com", help="S3 endpoint URL").tag( 62 | config=True, env="JPYNB_S3_ENDPOINT_URL" 63 | ) 64 | s3_region_name = Unicode("us-east-1", help="Region name").tag( 65 | config=True, env="JPYNB_S3_REGION_NAME" 66 | ) 67 | s3_bucket = Unicode("", help="Bucket name to store notebooks").tag( 68 | config=True, env="JPYNB_S3_BUCKET" 69 | ) 70 | 71 | max_threads = Integer( 72 | 16, help="Maximum number of threads for the threadpool allocated for S3 read/writes" 73 | ).tag(config=True) 74 | 75 | fs_cloning_basedir = Unicode( 76 | "", help=("Absolute path to base directory used to clone from the local file system") 77 | ).tag(config=True) 78 | 79 | 80 | def validate_bookstore(settings: BookstoreSettings): 81 | """Check that settings exist. 82 | 83 | Parameters 84 | ---------- 85 | settings : bookstore.bookstore_config.BookstoreSettings 86 | Instantiated settings object to be validated. 87 | 88 | Returns 89 | ------- 90 | validation_checks : dict 91 | Statements about whether features are validly configured and available 92 | """ 93 | general_settings = [settings.s3_bucket != "", settings.s3_endpoint_url != ""] 94 | archive_settings = [*general_settings, settings.workspace_prefix != ""] 95 | published_settings = [*general_settings, settings.published_prefix != ""] 96 | s3_cloning_settings = [settings.enable_s3_cloning] 97 | fs_cloning_settings = [Path(settings.fs_cloning_basedir).is_absolute()] 98 | 99 | validation_checks = { 100 | "bookstore_valid": all(general_settings), 101 | "archive_valid": all(archive_settings), 102 | "publish_valid": all(published_settings), 103 | "s3_clone_valid": all(s3_cloning_settings), 104 | "fs_clone_valid": all(fs_cloning_settings), 105 | } 106 | if not validation_checks["fs_clone_valid"] and settings.fs_cloning_basedir != "": 107 | log.info( 108 | f"{settings.fs_cloning_basedir} is not an absolute path, file system cloning will be disabled." 109 | ) 110 | 111 | return validation_checks 112 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Oh, hello there! You're probably reading this because you are interested in 4 | contributing to nteract. That's great to hear! This document will help you 5 | through your journey of open source. Love it, cherish it, take it out to 6 | dinner, but most importantly: read it thoroughly! 7 | 8 | ## What do I need to know to help? 9 | 10 | Read the README.md file. This will help you set up the project. If you have 11 | questions, please ask on the nteract Slack channel. We're a welcoming project and 12 | are happy to answer your questions. 13 | 14 | ## How do I make a contribution? 15 | 16 | Never made an open source contribution before? Wondering how contributions work 17 | in the nteract world? Here's a quick rundown! 18 | 19 | 1. Find an issue that you are interested in addressing or a feature that you 20 | would like to address. 21 | 2. Fork the repository associated with the issue to your local GitHub 22 | organization. 23 | 3. Clone the repository to your local machine using: 24 | 25 | git clone https://github.com/github-username/repository-name.git 26 | 27 | 4. Create a new branch for your fix using: 28 | 29 | git checkout -b branch-name-here 30 | 31 | 5. Make the appropriate changes for the issue you are trying to address or the 32 | feature that you want to add. 33 | 6. You can run python unit tests using `pytest`. Running integration tests 34 | locally requires a more complicated setup. This setup is described in 35 | [running_ci_locally.md](./running_ci_locally.md) 36 | 7. Add and commit the changed files using `git add` and `git commit`. 37 | 8. Push the changes to the remote repository using: 38 | 39 | git push origin branch-name-here 40 | 41 | 9. Submit a pull request to the upstream repository. 42 | 10. Title the pull request per the requirements outlined in the section below. 43 | 11. Set the description of the pull request with a brief description of what you 44 | did and any questions you might have about what you did. 45 | 12. Wait for the pull request to be reviewed by a maintainer. 46 | 13. Make changes to the pull request if the reviewing maintainer recommends 47 | them. 48 | 14. Celebrate your success after your pull request is merged! :tada: 49 | 50 | ## How should I write my commit messages and PR titles? 51 | 52 | Good commit messages serve at least three important purposes: 53 | 54 | * To speed up the reviewing process. 55 | 56 | * To help us write a good release note. 57 | 58 | * To help the future maintainers of nteract/nteract (it could be you!), say 59 | five years into the future, to find out why a particular change was made to 60 | the code or why a specific feature was added. 61 | 62 | Structure your commit message like this: 63 | 64 | ```text 65 | > Short (50 chars or less) summary of changes 66 | > 67 | > More detailed explanatory text, if necessary. Wrap it to about 72 68 | > characters or so. In some contexts, the first line is treated as the 69 | > subject of an email and the rest of the text as the body. The blank 70 | > line separating the summary from the body is critical (unless you omit 71 | > the body entirely); tools like rebase can get confused if you run the 72 | > two together. 73 | > 74 | > Further paragraphs come after blank lines. 75 | > 76 | > - Bullet points are okay, too 77 | > 78 | > - Typically a hyphen or asterisk is used for the bullet, preceded by a 79 | > single space, with blank lines in between, but conventions vary here 80 | > 81 | ``` 82 | 83 | *Source:* https://git-scm.com/book/ch5-2.html 84 | 85 | ### DO 86 | 87 | * Write the summary line and description of what you have done in the 88 | imperative mode, that is as if you were commanding. Start the line 89 | with "Fix", "Add", "Change" instead of "Fixed", "Added", "Changed". 90 | * Always leave the second line blank. 91 | * Line break the commit message (to make the commit message readable 92 | without having to scroll horizontally in gitk). 93 | 94 | ### DON'T 95 | 96 | * Don't end the summary line with a period - it's a title and titles don't end 97 | with a period. 98 | 99 | ### Tips 100 | 101 | * If it seems difficult to summarize what your commit does, it may be because it 102 | includes several logical changes or bug fixes, and are better split up into 103 | several commits using `git add -p`. 104 | 105 | ### References 106 | 107 | The following blog post has a nice discussion of commit messages: 108 | 109 | * "On commit messages" http://who-t.blogspot.com/2009/12/on-commit-messages.html 110 | 111 | ## How fast will my PR be merged? 112 | 113 | Your pull request will be merged as soon as there are maintainers to review it 114 | and after tests have passed. You might have to make some changes before your 115 | PR is merged but as long as you adhere to the steps above and try your best, 116 | you should have no problem getting your PR merged. 117 | 118 | That's it! You're good to go! 119 | -------------------------------------------------------------------------------- /bookstore/handlers.py: -------------------------------------------------------------------------------- 1 | """Handlers for Bookstore API""" 2 | import json 3 | 4 | from notebook.base.handlers import APIHandler 5 | from notebook.base.handlers import path_regex 6 | from .utils import url_path_join 7 | from tornado import web 8 | 9 | from ._version import __version__ 10 | from .bookstore_config import BookstoreSettings 11 | from .bookstore_config import validate_bookstore 12 | from .publish import BookstorePublishAPIHandler 13 | from .clone import ( 14 | BookstoreCloneHandler, 15 | BookstoreCloneAPIHandler, 16 | BookstoreFSCloneHandler, 17 | BookstoreFSCloneAPIHandler, 18 | ) 19 | 20 | 21 | version = __version__ 22 | 23 | 24 | class BookstoreVersionHandler(APIHandler): 25 | """Handler responsible for Bookstore version information 26 | 27 | Used to lay foundations for the bookstore package. Though, frontends can use this endpoint for feature detection. 28 | 29 | Methods 30 | ------- 31 | get(self) 32 | Provides version info and feature availability based on serverside settings. 33 | build_response_dict(self) 34 | Helper to populate response. 35 | """ 36 | 37 | @web.authenticated 38 | def get(self): 39 | """GET /api/bookstore/ 40 | 41 | Returns version info and validation info for various bookstore features. 42 | """ 43 | self.finish(json.dumps(self.build_response_dict())) 44 | 45 | def build_response_dict(self): 46 | """Helper for building the version handler's response before serialization.""" 47 | return { 48 | "release": self.settings['bookstore']["release"], 49 | "features": self.settings['bookstore']["features"], 50 | } 51 | 52 | 53 | def build_settings_dict(validation): 54 | """Helper for building the settings info that will be assigned to the web_app.""" 55 | return {"release": version, "features": validation} 56 | 57 | 58 | def load_jupyter_server_extension(nb_app): 59 | web_app = nb_app.web_app 60 | host_pattern = '.*$' 61 | 62 | base_url = web_app.settings['base_url'] 63 | 64 | bookstore_settings = BookstoreSettings(parent=nb_app) 65 | validation = validate_bookstore(bookstore_settings) 66 | web_app.settings['bookstore'] = build_settings_dict(validation) 67 | handlers = collect_handlers(nb_app.log, base_url, validation) 68 | web_app.add_handlers(host_pattern, handlers) 69 | 70 | 71 | def collect_handlers(log, base_url, validation): 72 | """Utility that collects bookstore endpoints & handlers to be added to the webapp. 73 | 74 | This uses bookstore feature validation to determine which endpoints should be enabled. 75 | It returns all valid pairs of endpoint patterns and handler classes. 76 | 77 | Parameters 78 | ---------- 79 | log : logging.Logger 80 | Log (usually from the NotebookApp) for logging endpoint changes. 81 | base_url: str 82 | The base_url to which we append routes. 83 | validation: dict 84 | Validation dictionary for determining which endpoints to enable. 85 | 86 | Returns 87 | -------- 88 | 89 | List[Tuple[str, tornado.web.RequestHandler]] 90 | List of pairs of endpoint patterns and the handler used to handle requests at that endpoint. 91 | """ 92 | base_bookstore_pattern = url_path_join(base_url, '/bookstore') 93 | base_bookstore_api_pattern = url_path_join(base_url, '/api/bookstore') 94 | 95 | handlers = [] 96 | # Always enable the version handler for the API 97 | handlers.append((base_bookstore_api_pattern, BookstoreVersionHandler)) 98 | 99 | if validation['publish_valid']: 100 | log.info(f"[bookstore] Enabling bookstore publishing, version: {version}") 101 | handlers.append( 102 | ( 103 | url_path_join(base_bookstore_api_pattern, r"/publish%s" % path_regex), 104 | BookstorePublishAPIHandler, 105 | ) 106 | ) 107 | else: 108 | log.info("[bookstore] Publishing disabled. s3_bucket or endpoint are not configured.") 109 | 110 | if validation['s3_clone_valid']: 111 | log.info(f"[bookstore] Enabling bookstore cloning, version: {version}") 112 | handlers.append( 113 | (url_path_join(base_bookstore_api_pattern, r"/clone(?:/?)*"), BookstoreCloneAPIHandler) 114 | ), 115 | handlers.append( 116 | (url_path_join(base_bookstore_pattern, r"/clone(?:/?)*"), BookstoreCloneHandler) 117 | ) 118 | else: 119 | log.info(f"[bookstore] bookstore cloning disabled, version: {version}") 120 | 121 | if validation['fs_clone_valid']: 122 | log.info(f"[bookstore] Enabling filesystem cloning, version: {version}") 123 | handlers.append( 124 | (url_path_join(base_bookstore_pattern, r"/fs-clone(?:/?)*"), BookstoreFSCloneHandler) 125 | ) 126 | handlers.append( 127 | ( 128 | url_path_join(base_bookstore_api_pattern, r"/fs-clone(?:/?)*"), 129 | BookstoreFSCloneAPIHandler, 130 | ) 131 | ), 132 | else: 133 | log.info(f"[bookstore] bookstore cloning disabled, version: {version}") 134 | return handlers 135 | -------------------------------------------------------------------------------- /bookstore/publish.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import aiobotocore 4 | 5 | from botocore.exceptions import ClientError 6 | from nbformat import ValidationError 7 | from nbformat import validate as validate_nb 8 | from notebook.base.handlers import APIHandler, path_regex 9 | from notebook.services.contents.handlers import validate_model 10 | from tornado import web 11 | 12 | from .bookstore_config import BookstoreSettings 13 | from .s3_paths import s3_path 14 | from .s3_paths import s3_key 15 | from .s3_paths import s3_display_path 16 | from .utils import url_path_join 17 | 18 | 19 | class BookstorePublishAPIHandler(APIHandler): 20 | """Publish a notebook to the publish path""" 21 | 22 | def initialize(self): 23 | """Initialize a helper to get bookstore settings and session information quickly""" 24 | self.bookstore_settings = BookstoreSettings(config=self.config) 25 | self.session = aiobotocore.get_session() 26 | 27 | @web.authenticated 28 | async def put(self, path): 29 | """Publish a notebook on a given path. 30 | 31 | PUT /api/bookstore/publish 32 | 33 | The payload directly matches the contents API for PUT. 34 | 35 | Parameters 36 | ---------- 37 | path: str 38 | Path describing where contents should be published to, postfixed to the published_prefix . 39 | """ 40 | if path == '' or path == '/': 41 | raise web.HTTPError(400, "Must provide a path for publishing") 42 | path = path.lstrip('/') 43 | 44 | s3_object_key = s3_key(self.bookstore_settings.published_prefix, path) 45 | 46 | model = self.get_json_body() 47 | self.validate_model(model) 48 | 49 | full_s3_path = s3_display_path( 50 | self.bookstore_settings.s3_bucket, self.bookstore_settings.published_prefix, path 51 | ) 52 | self.log.info(f"Publishing to {full_s3_path}") 53 | 54 | obj = await self._publish(model['content'], s3_object_key) 55 | resp_content = self.prepare_response(obj, full_s3_path) 56 | 57 | self.set_status(obj['ResponseMetadata']['HTTPStatusCode']) 58 | self.finish(json.dumps(resp_content)) 59 | 60 | def validate_model(self, model): 61 | """Checks that the model given to the API handler meets bookstore's expected structure for a notebook. 62 | 63 | Pattern for surfacing nbformat validation errors originally written in 64 | https://github.com/jupyter/notebook/blob/a44a367c219b60a19bee003877d32c3ff1ce2412/notebook/services/contents/manager.py#L353-L355 65 | 66 | Parameters 67 | ---------- 68 | model: dict 69 | Request model for publishing describing the type and content of the object. 70 | 71 | Raises 72 | ------ 73 | tornado.web.HTTPError 74 | Your model does not validate correctly 75 | """ 76 | if not model: 77 | raise web.HTTPError(400, "Bookstore cannot publish an empty model") 78 | if model.get('type', "") != 'notebook': 79 | raise web.HTTPError(415, "Bookstore only publishes notebooks") 80 | 81 | content = model.get('content', {}) 82 | if content == {}: 83 | raise web.HTTPError(422, "Bookstore cannot publish empty contents") 84 | try: 85 | validate_nb(content) 86 | except ValidationError as e: 87 | raise web.HTTPError( 88 | 422, 89 | "Bookstore cannot publish invalid notebook. " 90 | "Validation errors are as follows: " 91 | f"{e.message} {json.dumps(e.instance, indent=1, default=lambda obj: '')}", 92 | ) 93 | 94 | async def _publish(self, content, s3_object_key): 95 | """Publish notebook model to the path 96 | 97 | Returns 98 | -------- 99 | dict 100 | S3 PutObject response object 101 | """ 102 | 103 | async with self.session.create_client( 104 | 's3', 105 | aws_secret_access_key=self.bookstore_settings.s3_secret_access_key, 106 | aws_access_key_id=self.bookstore_settings.s3_access_key_id, 107 | endpoint_url=self.bookstore_settings.s3_endpoint_url, 108 | region_name=self.bookstore_settings.s3_region_name, 109 | ) as client: 110 | self.log.info(f"Processing published write to {s3_object_key}") 111 | try: 112 | obj = await client.put_object( 113 | Bucket=self.bookstore_settings.s3_bucket, 114 | Key=s3_object_key, 115 | Body=json.dumps(content), 116 | ) 117 | except ClientError as e: 118 | status_code = e.response['ResponseMetadata'].get('HTTPStatusCode') 119 | raise web.HTTPError(status_code, e.args[0]) 120 | self.log.info(f"Done with published write to {s3_object_key}") 121 | 122 | return obj 123 | 124 | def prepare_response(self, obj, full_s3_path): 125 | """Prepares repsonse to publish PUT request. 126 | 127 | Parameters 128 | ---------- 129 | obj: dict 130 | Validation dictionary for determining which endpoints to enable. 131 | path: 132 | path to place after the published prefix in the designated bucket 133 | 134 | Returns 135 | -------- 136 | dict 137 | Model for responding to put request. 138 | """ 139 | 140 | resp_content = {"s3_path": full_s3_path} 141 | 142 | if 'VersionId' in obj: 143 | resp_content["versionID"] = obj['VersionId'] 144 | 145 | return resp_content 146 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## [Unreleased](https://github.com/nteract/bookstore/compare/2.5.1...HEAD) 4 | 5 | ## [2.5.1](https://github.com/nteract/bookstore/compare/2.5.1) 6 | 7 | This enables adds a new feature to bookstore cloning from s3, cloning specific versions of notebooks from versioned s3 buckets. 8 | 9 | Specifically, it introduces the `s3_version_id` query parameter to the `/bookstore/clone/` GET handler. 10 | 11 | So if you wanted to clone a specific version `myVersion` of `/workspace/my_notebook.ipynb` from the `my_bucket` S3 bucket, 12 | you would change the route from something like 13 | 14 | http://localhost:8888/bookstore/clone?s3_bucket=my_bucket&s3_key=workspace/my_notebook.ipynb 15 | 16 | to 17 | 18 | http://localhost:8888/bookstore/clone?s3_bucket=my_bucket&s3_key=workspace/my_notebook.ipynb&s3_version_id=myVersion 19 | 20 | ## [2.5.0](https://github.com/nteract/bookstore/compare/2.5.0) 21 | 22 | This switches the bookstore serverextension and landing page from using absolute urls to relative paths. 23 | 24 | ## [2.4.1](https://github.com/nteract/bookstore/releases/tag/2.4.1) 2019-08-6 25 | 26 | This improves the landing page experience with a cleaner and clearer landing page design. 27 | 28 | ## [2.4.0](https://github.com/nteract/bookstore/releases/tag/2.4.0) 2019-08-5 29 | 30 | Thank you to the following contributors: 31 | 32 | * Carol Willing 33 | * M Pacer 34 | * Peter Volpe 35 | 36 | The full list of changes they made can be seen [on GitHub](https://github.com/nteract/bookstore/issues?q=milestone%3A2.4.0) 37 | 38 | ### Significant changes 39 | 40 | #### Cloning 41 | 42 | As of 2.4.0 cloning from a local or network attached file system is now possible, but disabled by default. 43 | 44 | To enable this filesystem (`fs`) cloning, set `BookstoreSettings.fs_cloning_basedir` to the root directory from which you want people to be able to clone. 45 | 46 | Adding fs cloning offers users more flexibility to clone notebooks from attached filesystems, like EFS. For more information about the motivation and design of this endpoint, please see [this issue](https://github.com/nteract/bookstore/issues/154). 47 | 48 | ## [2.3.1](https://github.com/nteract/bookstore/releases/tag/2.3.1) 2019-07-16 49 | 50 | ### Fixing problems 51 | 52 | This fixes an issue that arose where in certain cases cloning would hang indefinitely when trying to read content [#145](https://github.com/nteract/bookstore/issues/145). 53 | 54 | ## [2.3.0](https://github.com/nteract/bookstore/releases/tag/2.3.0) 2019-07-02 55 | 56 | Thank you to the following contributors: 57 | 58 | * Carol Willing 59 | * Kyle Kelley 60 | * M Pacer 61 | * Matthew Seal 62 | * Safia Abdalla 63 | * Shelby Sturgis 64 | 65 | The full list of changes they made can be seen [on GitHub](https://github.com/nteract/bookstore/issues?q=milestone%3A2.3.0) 66 | 67 | ### Significant changes 68 | 69 | #### New Publishing endpoint 70 | 71 | Previously our publishing endpoint was `/api/bookstore/published`, it is now `/api/bookstore/publish`. 72 | 73 | #### Cloning 74 | 75 | As of 2.3.0 cloning from S3 is now enabled by default. 76 | 77 | Cloning allows access to multiple S3 buckets. To use them, you will need to set up your configuration for any such bucket. 78 | 79 | #### Massive Testing improvements 80 | 81 | We have built out a framework for unit-testing Tornado handlers. In addition, we have added a collection of unit tests that bring us to a coverage level in non-experimental code of well over 80%. 82 | 83 | #### `/api/bookstore/`: Features and Versions 84 | 85 | You can identify which features have been enabled and which version of bookstore is available by using the `/api/bookstore` endpoint. 86 | 87 | #### REST API Documentation 88 | 89 | All APIs are now documented at our [REST API docs](https://bookstore.readthedocs.io/en/latest/openapi.html) using the OpenAPI spec. 90 | 91 | ### Experimental 92 | #### Clients (subject to change in future releases) 93 | 94 | To enable access to bookstore publishing and cloning from within a notebook, we have created a Notebook and Bookstore clients. *This is still experimental* functionality at the moment and needs additional testing, so we discourage its use in production. 95 | The design relies on an assumption that a single kernel is attached to a single notebook, and will break if you use multiple notebooks attached to the same kernel. 96 | 97 | However, for those who wish to experiment, it offers some fun ways of exploring bookstore. 98 | 99 | Example: if you run a notebook from within the top-level [`bookstore/ci`](https://github.com/nteract/bookstore/tree/master/ci) directory while running the integration test server with `yarn test:server` (see more about [local integration testing](https://bookstore.readthedocs.io/en/latest/project/local_ci.html)), 100 | you should be able to publish from inside a notebook using the following code snippet:``` 101 | 102 | ```python 103 | from bookstore.client import BookstoreClient 104 | book_store = BookstoreClient() 105 | book_store.publish() 106 | ``` 107 | 108 | And if you have published your notebook to the local ci (e.g., publishing `my_notebook.ipynb` to the minio `bookstore` bucket with the `ci-published` published prefix), you can clone it from S3 using: 109 | 110 | ```python 111 | from bookstore.client import BookstoreClient 112 | book_store = BookstoreClient() 113 | book_store.clone("bookstore", "ci-published/my_notebook.ipynb") 114 | ``` 115 | 116 | ## Releases prior to 2.3.0 117 | 118 | [2.2.1 (2019-02-03)](https://github.com/nteract/bookstore/releases/tag/2.2.1) 119 | 120 | [2.2.0 (2019-01-29)](https://github.com/nteract/bookstore/releases/tag/2.2.0) 121 | 122 | [2.1.0 (2018-11-20)](https://github.com/nteract/bookstore/releases/tag/2.1.0) 123 | 124 | [2.0.0 (2018-11-13)](https://github.com/nteract/bookstore/releases/tag/2.0.0) 125 | 126 | [0.1 (2018=10-16)](https://github.com/nteract/bookstore/releases/tag/0.1) 127 | -------------------------------------------------------------------------------- /bookstore/archive.py: -------------------------------------------------------------------------------- 1 | """Archival of notebooks""" 2 | 3 | import json 4 | from asyncio import Lock 5 | from typing import Dict 6 | from typing import NamedTuple 7 | 8 | import aiobotocore 9 | import nbformat 10 | from notebook.services.contents.filemanager import FileContentsManager 11 | from tornado import ioloop 12 | 13 | from .bookstore_config import BookstoreSettings 14 | from .s3_paths import s3_key, s3_display_path 15 | 16 | 17 | class ArchiveRecord(NamedTuple): 18 | """Represents an archival record. 19 | 20 | An `ArchiveRecord` uses a Typed version of `collections.namedtuple()`. The 21 | record is immutable. 22 | 23 | Example 24 | ------- 25 | 26 | An archive record (`filepath`, `content`, `queued_time`) contains: 27 | 28 | - a `filepath` to the record 29 | - the `content` for archival 30 | - the `queued time` length of time waiting in the queue for archiving 31 | """ 32 | 33 | filepath: str 34 | content: str 35 | queued_time: float # TODO: refactor to a datetime time 36 | 37 | 38 | class BookstoreContentsArchiver(FileContentsManager): 39 | """Manages archival of notebooks to storage (S3) when notebook save occurs. 40 | 41 | This class is a custom Jupyter 42 | `FileContentsManager `_ 43 | which holds information on storage location, path to it, and file to be 44 | written. 45 | 46 | Example 47 | ------- 48 | 49 | - Bookstore settings combine with the parent Jupyter application settings. 50 | - A session is created for the current event loop. 51 | - To write to a particular path on S3, acquire a lock. 52 | - After acquiring the lock, `archive` method authenticates using the storage 53 | service's credentials. 54 | - If allowed, the notebook is queued to be written to storage (i.e. S3). 55 | 56 | Attributes 57 | ---------- 58 | 59 | path_locks : dict 60 | Dictionary of paths to storage and the lock associated with a path. 61 | path_lock_ready: asyncio mutex lock 62 | A mutex lock associated with a path. 63 | """ 64 | 65 | def __init__(self, *args, **kwargs): 66 | super(FileContentsManager, self).__init__(*args, **kwargs) 67 | 68 | # opt ourselves into being part of the Jupyter App that should have Bookstore Settings applied 69 | self.settings = BookstoreSettings(parent=self) 70 | 71 | self.log.info( 72 | "Archiving notebooks to {}".format( 73 | s3_display_path(self.settings.s3_bucket, self.settings.workspace_prefix) 74 | ) 75 | ) 76 | 77 | try: 78 | # create a session object from the current event loop 79 | self.session = aiobotocore.get_session() 80 | except Exception: 81 | self.log.warn("Unable to create a session") 82 | raise 83 | 84 | # a collection of locks per path to suppress writing while the path may be in use 85 | self.path_locks: Dict[str, Lock] = {} 86 | self.path_lock_ready = Lock() 87 | 88 | async def archive(self, record: ArchiveRecord): 89 | """Process a record to write to storage. 90 | 91 | Acquire a path lock before archive. Writing to storage will only be 92 | allowed to a path if a valid `path_lock` is held and the path is not 93 | locked by another process. 94 | 95 | Parameters 96 | ---------- 97 | 98 | record : ArchiveRecord 99 | A notebook and where it should be written to storage 100 | """ 101 | async with self.path_lock_ready: 102 | lock = self.path_locks.get(record.filepath) 103 | 104 | if lock is None: 105 | lock = Lock() 106 | self.path_locks[record.filepath] = lock 107 | 108 | # Skip writes when a given path is already locked 109 | if lock.locked(): 110 | self.log.info("Skipping archive of %s", record.filepath) 111 | return 112 | 113 | async with lock: 114 | try: 115 | async with self.session.create_client( 116 | 's3', 117 | aws_secret_access_key=self.settings.s3_secret_access_key, 118 | aws_access_key_id=self.settings.s3_access_key_id, 119 | endpoint_url=self.settings.s3_endpoint_url, 120 | region_name=self.settings.s3_region_name, 121 | ) as client: 122 | self.log.info("Processing storage write of %s", record.filepath) 123 | file_key = s3_key(self.settings.workspace_prefix, record.filepath) 124 | await client.put_object( 125 | Bucket=self.settings.s3_bucket, Key=file_key, Body=record.content 126 | ) 127 | self.log.info("Done with storage write of %s", record.filepath) 128 | except Exception as e: 129 | self.log.error( 130 | 'Error while archiving file: %s %s', record.filepath, e, exc_info=True 131 | ) 132 | 133 | def run_pre_save_hook(self, model, path, **kwargs): 134 | """Send request to store notebook to S3. 135 | 136 | This hook offloads the storage request to the event loop. 137 | When the event loop is available for execution of the request, the 138 | storage of the notebook will be done and the write to storage occurs. 139 | 140 | Parameters 141 | ---------- 142 | 143 | model : dict 144 | The type of file and its contents 145 | path : str 146 | The storage location 147 | """ 148 | if model["type"] != "notebook": 149 | self.log.debug( 150 | "Bookstore only archives notebooks, " 151 | f"request does not state that {path} is a notebook." 152 | ) 153 | return 154 | 155 | content = nbformat.writes(nbformat.from_dict(model["content"])) 156 | 157 | loop = ioloop.IOLoop.current() 158 | 159 | # Offload archival and schedule write to storage with the current event loop 160 | loop.spawn_callback( 161 | self.archive, 162 | ArchiveRecord( 163 | content=content, filepath=path, queued_time=ioloop.IOLoop.current().time() 164 | ), 165 | ) 166 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | import os 16 | import sys 17 | 18 | import m2r 19 | 20 | sys.path.insert(0, os.path.abspath('..')) 21 | 22 | # -- Project information ----------------------------------------------------- 23 | 24 | project = 'bookstore' 25 | copyright = '2018, nteract project' 26 | author = 'nteract project' 27 | 28 | # The short X.Y version 29 | version = '2.5' 30 | # The full version, including alpha/beta/rc tags 31 | release = '2.5.2dev0' 32 | 33 | 34 | # -- General configuration --------------------------------------------------- 35 | 36 | # If your documentation needs a minimal Sphinx version, state it here. 37 | # 38 | # needs_sphinx = '1.0' 39 | 40 | # Add any Sphinx extension module names here, as strings. They can be 41 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 42 | # ones. 43 | extensions = [ 44 | 'sphinx.ext.autodoc', 45 | 'sphinx.ext.intersphinx', 46 | 'sphinx.ext.mathjax', 47 | 'sphinx.ext.napoleon', 48 | 'sphinxcontrib.openapi', 49 | 'm2r', 50 | ] 51 | 52 | # Add any paths that contain templates here, relative to this directory. 53 | templates_path = ['_templates'] 54 | 55 | # The suffix(es) of source filenames. 56 | # You can specify multiple suffix as a list of string: 57 | source_suffix = ['.rst', '.md'] 58 | 59 | # The master toctree document. 60 | master_doc = 'index' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | # This pattern also affects html_static_path and html_extra_path. 72 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 73 | 74 | # The name of the Pygments (syntax highlighting) style to use. 75 | pygments_style = None 76 | 77 | 78 | # -- Options for HTML output ------------------------------------------------- 79 | 80 | # The theme to use for HTML and HTML Help pages. See the documentation for 81 | # a list of builtin themes. 82 | # 83 | html_theme = 'alabaster' 84 | 85 | html_logo = '_static/images/nteract_logo_compact_purple.png' 86 | 87 | # Theme options are theme-specific and customize the look and feel of a theme 88 | # further. For a list of options available for each theme, see the 89 | # documentation. 90 | # 91 | html_theme_options = { 92 | 'show_related': True, 93 | 'description': 'Notebook storage workflows for the masses.', 94 | 'github_user': 'nteract', 95 | 'github_repo': 'bookstore', 96 | 'github_banner': False, 97 | 'github_button': False, 98 | 'show_powered_by': True, 99 | 'extra_nav_links': { 100 | 'GitHub Repo': 'http://github.com/nteract/bookstore', 101 | 'Issue Tracker': 'http://github.com/nteract/bookstore/issues', 102 | }, 103 | } 104 | 105 | # Add any paths that contain custom static files (such as style sheets) here, 106 | # relative to this directory. They are copied after the builtin static files, 107 | # so a file named "default.css" will overwrite the builtin "default.css". 108 | html_static_path = ['_static'] 109 | 110 | # Custom sidebar templates, must be a dictionary that maps document names 111 | # to template names. 112 | # 113 | # The default sidebars (for documents that don't match any pattern) are 114 | # defined by theme itself. Builtin themes are using these templates by 115 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 116 | # 'searchbox.html']``. 117 | # 118 | html_sidebars = {'**': ['about.html', 'searchbox.html', 'navigation.html', 'sourcelink.html']} 119 | 120 | 121 | # -- Options for HTMLHelp output --------------------------------------------- 122 | 123 | # Output file base name for HTML help builder. 124 | htmlhelp_basename = 'bookstoredoc' 125 | 126 | 127 | # -- Options for LaTeX output ------------------------------------------------ 128 | 129 | latex_elements = { 130 | # The paper size ('letterpaper' or 'a4paper'). 131 | # 132 | # 'papersize': 'letterpaper', 133 | # The font size ('10pt', '11pt' or '12pt'). 134 | # 135 | # 'pointsize': '10pt', 136 | # Additional stuff for the LaTeX preamble. 137 | # 138 | # 'preamble': '', 139 | # Latex figure (float) alignment 140 | # 141 | # 'figure_align': 'htbp', 142 | } 143 | 144 | # Grouping the document tree into LaTeX files. List of tuples 145 | # (source start file, target name, title, 146 | # author, documentclass [howto, manual, or own class]). 147 | latex_documents = [ 148 | (master_doc, 'bookstore.tex', 'bookstore Documentation', 'nteract project', 'manual') 149 | ] 150 | 151 | 152 | # -- Options for manual page output ------------------------------------------ 153 | 154 | # One entry per manual page. List of tuples 155 | # (source start file, name, description, authors, manual section). 156 | man_pages = [(master_doc, 'bookstore', 'bookstore Documentation', [author], 1)] 157 | 158 | 159 | # -- Options for Texinfo output ---------------------------------------------- 160 | 161 | # Grouping the document tree into Texinfo files. List of tuples 162 | # (source start file, target name, title, author, 163 | # dir menu entry, description, category) 164 | texinfo_documents = [ 165 | ( 166 | master_doc, 167 | 'bookstore', 168 | 'bookstore Documentation', 169 | author, 170 | 'bookstore', 171 | 'One line description of project.', 172 | 'Miscellaneous', 173 | ) 174 | ] 175 | 176 | 177 | # -- Options for Epub output ------------------------------------------------- 178 | 179 | # Bibliographic Dublin Core info. 180 | epub_title = project 181 | 182 | # The unique identifier of the text. This can be a ISBN number 183 | # or the project homepage. 184 | # 185 | # epub_identifier = '' 186 | 187 | # A unique identification for the text. 188 | # 189 | # epub_uid = '' 190 | 191 | # A list of files that should not be packed into the epub file. 192 | epub_exclude_files = ['search.html'] 193 | 194 | 195 | # -- Extension configuration ------------------------------------------------- 196 | -------------------------------------------------------------------------------- /ci/jupyter.js: -------------------------------------------------------------------------------- 1 | const child_process = require("child_process"); 2 | const { genToken } = require("./token"); 3 | const { sleep } = require("./sleep"); 4 | const { url_path_join } = require("./utils"); 5 | 6 | // "Polyfill" XMLHttpRequest for rxjs' ajax to use 7 | global.XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest; 8 | const { ajax } = require("rxjs/ajax"); 9 | 10 | class JupyterServer { 11 | constructor(config = {}) { 12 | this.port = config.port || 9988; 13 | this.ip = config.ip || "127.0.0.1"; 14 | this.scheme = config.scheme || "http"; 15 | this.token = null; 16 | this.baseUrl = config.baseUrl || "mybaseUrl/ipynb/"; 17 | 18 | // Launch the server from the directory of this script by default 19 | this.cwd = config.cwd || __dirname; 20 | 21 | this.process = null; 22 | this.up = false; 23 | } 24 | 25 | async start() { 26 | if (!this.token) { 27 | this.token = await genToken(); 28 | } 29 | 30 | // Kill off any prexisting process before creating a new one 31 | if (this.process) { 32 | this.process.kill(); 33 | } 34 | 35 | this.process = child_process.spawn( 36 | "jupyter", 37 | [ 38 | "notebook", 39 | "--no-browser", 40 | `--NotebookApp.token=${this.token}`, 41 | `--NotebookApp.disable_check_xsrf=True`, 42 | `--NotebookApp.base_url=${this.baseUrl}`, 43 | `--port=${this.port}`, 44 | `--ip=${this.ip}`, 45 | `--log-level=10` 46 | ], 47 | { cwd: this.cwd } 48 | ); 49 | 50 | ////// Refactor me later, streams are a bit messy with async await 51 | ////// Let's use spawn-rx in the future and make some clean rxjs with timeouts 52 | this.process.stdout.on("data", data => { 53 | const s = data.toString(); 54 | process.stdout.write(s); 55 | }); 56 | this.process.stderr.on("data", data => { 57 | const s = data.toString(); 58 | process.stderr.write(s); 59 | 60 | if (s.includes("Jupyter Notebook is running at")) { 61 | this.up = true; 62 | } 63 | }); 64 | this.process.stdout.on("end", data => 65 | console.log("jupyter server terminated") 66 | ); 67 | 68 | await sleep(3000); 69 | 70 | if (!this.up) { 71 | console.log("jupyter has not come up after 3 seconds, waiting 3 more"); 72 | await sleep(3000); 73 | 74 | if (!this.up) { 75 | throw new Error("jupyter has not come up after 6 seconds, bailing"); 76 | } 77 | } 78 | } 79 | 80 | async writeNotebook(path, notebook) { 81 | // Once https://github.com/nteract/nteract/pull/3651 is merged, we can use 82 | // rx-jupyter for writing a notebook to the contents API 83 | const apiPath = "/api/contents/"; 84 | const xhr = await ajax({ 85 | url: url_path_join(this.endpoint, apiPath, path), 86 | responseType: "json", 87 | createXHR: () => new XMLHttpRequest(), 88 | method: "PUT", 89 | body: { 90 | type: "notebook", 91 | content: notebook 92 | }, 93 | headers: { 94 | "Content-Type": "application/json", 95 | Authorization: `token ${this.token}` 96 | } 97 | }).toPromise(); 98 | 99 | return xhr; 100 | } 101 | 102 | async publishNotebook(path, notebook) { 103 | // Once https://github.com/nteract/nteract/pull/3651 is merged, we can use 104 | // rx-jupyter for writing a notebook to the contents API 105 | const apiPath = "/api/bookstore/publish/"; 106 | const xhr = await ajax({ 107 | url: url_path_join(this.endpoint, apiPath, path), 108 | responseType: "json", 109 | createXHR: () => new XMLHttpRequest(), 110 | method: "PUT", 111 | body: { 112 | type: "notebook", 113 | content: notebook 114 | }, 115 | headers: { 116 | "Content-Type": "application/json", 117 | Authorization: `token ${this.token}` 118 | } 119 | }).toPromise(); 120 | 121 | return xhr; 122 | } 123 | 124 | populateS3CloneLandingQuery(s3Bucket, s3Key) { 125 | return url_path_join( 126 | this.endpoint, 127 | `/bookstore/clone?s3_bucket=${s3Bucket}&s3_key=${s3Key}` 128 | ); 129 | } 130 | populateS3CloneQuery() { 131 | return url_path_join(this.endpoint, `/api/bookstore/clone`); 132 | } 133 | async cloneS3NotebookLanding(s3Bucket, s3Key) { 134 | // Once https://github.com/nteract/nteract/pull/3651 is merged, we can use 135 | // rx-jupyter for writing a notebook to the contents API 136 | const xhr = await ajax({ 137 | url: this.populateS3CloneLandingQuery(s3Bucket, s3Key), 138 | responseType: "text", 139 | createXHR: () => new XMLHttpRequest(), 140 | method: "GET", 141 | headers: { 142 | Authorization: `token ${this.token}` 143 | } 144 | }).toPromise(); 145 | 146 | return xhr; 147 | } 148 | async cloneS3Notebook(s3Bucket, s3Key) { 149 | const query = { 150 | url: this.populateS3CloneQuery(), 151 | responseType: "json", 152 | createXHR: () => new XMLHttpRequest(), 153 | body: { s3_bucket: s3Bucket, s3_key: s3Key }, 154 | method: "POST", 155 | headers: { 156 | "Content-Type": "application/json", 157 | Authorization: `token ${this.token}` 158 | } 159 | }; 160 | const xhr = await ajax(query).toPromise(); 161 | 162 | return xhr; 163 | } 164 | populateFSCloneLandingQuery(relpath) { 165 | return url_path_join( 166 | this.endpoint, 167 | `/bookstore/fs-clone?relpath=${relpath}` 168 | ); 169 | } 170 | populateFSCloneQuery() { 171 | return url_path_join(this.endpoint, `/api/bookstore/fs-clone`); 172 | } 173 | async cloneFSNotebookLanding(relpath) { 174 | const xhr = await ajax({ 175 | url: this.populateFSCloneLandingQuery(relpath), 176 | responseType: "text", 177 | createXHR: () => new XMLHttpRequest(), 178 | method: "GET", 179 | headers: { 180 | Authorization: `token ${this.token}` 181 | } 182 | }).toPromise(); 183 | 184 | return xhr; 185 | } 186 | async cloneFSNotebook(relpath) { 187 | const query = { 188 | url: this.populateFSCloneQuery(), 189 | responseType: "json", 190 | createXHR: () => new XMLHttpRequest(), 191 | body: { relpath }, 192 | method: "POST", 193 | headers: { 194 | "Content-Type": "application/json", 195 | Authorization: `token ${this.token}` 196 | } 197 | }; 198 | const xhr = await ajax(query).toPromise(); 199 | 200 | return xhr; 201 | } 202 | 203 | async deleteNotebook(path) { 204 | const apiPath = "/api/contents/"; 205 | const xhr = await ajax({ 206 | url: url_path_join(this.endpoint, apiPath, path), 207 | responseType: "json", 208 | createXHR: () => new XMLHttpRequest(), 209 | method: "DELETE", 210 | headers: { 211 | Authorization: `token ${this.token}` 212 | } 213 | }).toPromise(); 214 | 215 | return xhr; 216 | } 217 | shutdown() { 218 | this.process.kill(); 219 | } 220 | 221 | get endpoint() { 222 | return url_path_join( 223 | `${this.scheme}://${this.ip}:${this.port}`, 224 | this.baseUrl 225 | ); 226 | } 227 | } 228 | 229 | module.exports = { 230 | JupyterServer 231 | }; 232 | -------------------------------------------------------------------------------- /bookstore/tests/test_handlers.py: -------------------------------------------------------------------------------- 1 | """Tests for handlers""" 2 | from unittest.mock import Mock 3 | 4 | import pytest 5 | import logging 6 | from unittest.mock import Mock 7 | 8 | from bookstore._version import __version__ 9 | from bookstore.handlers import collect_handlers, build_settings_dict, BookstoreVersionHandler 10 | from bookstore.bookstore_config import BookstoreSettings, validate_bookstore 11 | from bookstore.clone import ( 12 | BookstoreCloneHandler, 13 | BookstoreCloneAPIHandler, 14 | BookstoreFSCloneHandler, 15 | BookstoreFSCloneAPIHandler, 16 | ) 17 | from bookstore.publish import BookstorePublishAPIHandler 18 | from notebook.base.handlers import path_regex 19 | from tornado.testing import AsyncTestCase 20 | from tornado.web import Application, HTTPError 21 | from tornado.httpserver import HTTPRequest 22 | from traitlets.config import Config 23 | 24 | log = logging.getLogger('test_handlers') 25 | version = __version__ 26 | 27 | from traitlets.config import Config 28 | 29 | 30 | def test_collect_handlers_all(): 31 | expected = [ 32 | ('/api/bookstore', BookstoreVersionHandler), 33 | ('/api/bookstore/publish%s' % path_regex, BookstorePublishAPIHandler), 34 | ('/api/bookstore/clone(?:/?)*', BookstoreCloneAPIHandler), 35 | ('/bookstore/clone(?:/?)*', BookstoreCloneHandler), 36 | ('/bookstore/fs-clone(?:/?)*', BookstoreFSCloneHandler), 37 | ('/api/bookstore/fs-clone(?:/?)*', BookstoreFSCloneAPIHandler), 38 | ] 39 | web_app = Application() 40 | mock_settings = { 41 | "BookstoreSettings": {"s3_bucket": "mock_bucket", "fs_cloning_basedir": "/Users/jupyter"} 42 | } 43 | bookstore_settings = BookstoreSettings(config=Config(mock_settings)) 44 | validation = validate_bookstore(bookstore_settings) 45 | handlers = collect_handlers(log, '/', validation) 46 | assert expected == handlers 47 | 48 | 49 | def test_collect_handlers_no_clone(): 50 | expected = [ 51 | ('/api/bookstore', BookstoreVersionHandler), 52 | ('/api/bookstore/publish%s' % path_regex, BookstorePublishAPIHandler), 53 | ] 54 | web_app = Application() 55 | mock_settings = {"BookstoreSettings": {"s3_bucket": "mock_bucket", "enable_s3_cloning": False}} 56 | bookstore_settings = BookstoreSettings(config=Config(mock_settings)) 57 | validation = validate_bookstore(bookstore_settings) 58 | handlers = collect_handlers(log, '/', validation) 59 | assert expected == handlers 60 | 61 | 62 | def test_collect_handlers_no_publish(): 63 | expected = [ 64 | ('/api/bookstore', BookstoreVersionHandler), 65 | ('/api/bookstore/clone(?:/?)*', BookstoreCloneAPIHandler), 66 | ('/bookstore/clone(?:/?)*', BookstoreCloneHandler), 67 | ('/bookstore/fs-clone(?:/?)*', BookstoreFSCloneHandler), 68 | ('/api/bookstore/fs-clone(?:/?)*', BookstoreFSCloneAPIHandler), 69 | ] 70 | web_app = Application() 71 | mock_settings = { 72 | "BookstoreSettings": { 73 | "s3_bucket": "mock_bucket", 74 | "published_prefix": "", 75 | "fs_cloning_basedir": "/Users/jupyter", 76 | } 77 | } 78 | bookstore_settings = BookstoreSettings(config=Config(mock_settings)) 79 | validation = validate_bookstore(bookstore_settings) 80 | handlers = collect_handlers(log, '/', validation) 81 | assert expected == handlers 82 | 83 | 84 | def test_collect_only_fs_clone(): 85 | expected = [ 86 | ('/api/bookstore', BookstoreVersionHandler), 87 | ('/bookstore/fs-clone(?:/?)*', BookstoreFSCloneHandler), 88 | ('/api/bookstore/fs-clone(?:/?)*', BookstoreFSCloneAPIHandler), 89 | ] 90 | web_app = Application() 91 | mock_settings = { 92 | "BookstoreSettings": { 93 | "published_prefix": "", 94 | "fs_cloning_basedir": "/Users/jupyter", 95 | "enable_s3_cloning": False, 96 | } 97 | } 98 | bookstore_settings = BookstoreSettings(config=Config(mock_settings)) 99 | validation = validate_bookstore(bookstore_settings) 100 | handlers = collect_handlers(log, '/', validation) 101 | assert expected == handlers 102 | 103 | 104 | def test_collect_handlers_only_version(): 105 | expected = [('/api/bookstore', BookstoreVersionHandler)] 106 | web_app = Application() 107 | mock_settings = {"BookstoreSettings": {"enable_s3_cloning": False}} 108 | bookstore_settings = BookstoreSettings(config=Config(mock_settings)) 109 | validation = validate_bookstore(bookstore_settings) 110 | handlers = collect_handlers(log, '/', validation) 111 | assert expected == handlers 112 | 113 | 114 | @pytest.fixture(scope="class") 115 | def bookstore_settings(request): 116 | mock_settings = { 117 | "BookstoreSettings": { 118 | "s3_access_key_id": "mock_id", 119 | "s3_secret_access_key": "mock_access", 120 | "s3_bucket": "my_bucket", 121 | "fs_cloning_basedir": "/Users/jupyter", 122 | } 123 | } 124 | config = Config(mock_settings) 125 | bookstore_settings = BookstoreSettings(config=config) 126 | if request.cls is not None: 127 | request.cls.bookstore_settings = bookstore_settings 128 | return bookstore_settings 129 | 130 | 131 | def test_build_settings_dict(bookstore_settings): 132 | expected = { 133 | 'features': { 134 | 'archive_valid': True, 135 | 'bookstore_valid': True, 136 | 'publish_valid': True, 137 | 's3_clone_valid': True, 138 | 'fs_clone_valid': True, 139 | }, 140 | 'release': version, 141 | } 142 | validation = validate_bookstore(bookstore_settings) 143 | assert expected == build_settings_dict(validation) 144 | 145 | 146 | @pytest.mark.usefixtures("bookstore_settings") 147 | class TestCloneAPIHandler(AsyncTestCase): 148 | def setUp(self): 149 | super().setUp() 150 | 151 | validation = validate_bookstore(self.bookstore_settings) 152 | self.mock_application = Mock( 153 | spec=Application, 154 | ui_methods={}, 155 | ui_modules={}, 156 | settings={"bookstore": build_settings_dict(validation)}, 157 | transforms=[], 158 | ) 159 | 160 | def get_handler(self, uri, app=None): 161 | if app is None: 162 | app = self.mock_application 163 | connection = Mock(context=Mock(protocol="https")) 164 | payload_request = HTTPRequest( 165 | method='GET', 166 | uri=uri, 167 | headers={"Host": "localhost:8888"}, 168 | body=None, 169 | connection=connection, 170 | ) 171 | return BookstoreVersionHandler(app, payload_request) 172 | 173 | def test_get(self): 174 | """This is a simple test of the get API at /api/bookstore 175 | 176 | The most notable feature is the need to set _transforms on the handler. 177 | 178 | The default value of handler()._transforms is `None`. 179 | This is iterated over when handler().flush() is called, raising a TypeError. 180 | 181 | In normal usage, the application assigns this when it creates a handler delegate. 182 | 183 | Because our mock application does not do this 184 | As a result this raises an error when self.finish() (and therefore self.flush()) is called. 185 | 186 | At runtime on a live Jupyter server, application.transforms == []. 187 | """ 188 | get_handler = self.get_handler('/api/bookstore/') 189 | setattr(get_handler, '_transforms', []) 190 | return_val = get_handler.get() 191 | assert return_val is None 192 | 193 | def test_build_response(self): 194 | empty_handler = self.get_handler('/api/bookstore/') 195 | expected = { 196 | 'features': { 197 | 'archive_valid': True, 198 | 'bookstore_valid': True, 199 | 'publish_valid': True, 200 | 's3_clone_valid': True, 201 | 'fs_clone_valid': True, 202 | }, 203 | 'release': version, 204 | } 205 | assert empty_handler.build_response_dict() == expected 206 | -------------------------------------------------------------------------------- /docs/source/bookstore_api.yaml: -------------------------------------------------------------------------------- 1 | openapi: 3.0.1 2 | servers: 3 | - url: http://localhost:8888 4 | description: Local Server 5 | info: 6 | title: Bookstore public API 7 | description: Bookstore API docs 8 | termsOfService: http://swagger.io/terms/ 9 | license: 10 | name: BSD 3-clause 11 | url: https://github.com/nteract/bookstore/blob/master/LICENSE 12 | version: 2.5.2 13 | externalDocs: 14 | description: Find out more about Bookstore 15 | url: https://bookstore.readthedocs.io/en/latest/ 16 | paths: 17 | /api/bookstore: 18 | get: 19 | tags: 20 | - info 21 | summary: Info about bookstore 22 | responses: 23 | 200: 24 | description: Successfully requested 25 | content: 26 | application/json: 27 | schema: 28 | $ref: '#/components/schemas/VersionInfo' 29 | /bookstore/clone: 30 | get: 31 | tags: 32 | - clone 33 | summary: Landing page for initiating cloning. 34 | description: This serves a simple html page that allows avoiding xsrf issues on a jupyter server. 35 | parameters: 36 | - name: s3_bucket 37 | in: query 38 | description: S3_bucket being targeted 39 | required: true 40 | style: form 41 | schema: 42 | type: string 43 | - name: s3_key 44 | in: query 45 | description: S3 object key being requested 46 | required: true 47 | style: form 48 | schema: 49 | type: string 50 | - name: s3_version_id 51 | in: query 52 | description: S3 object key being requested 53 | required: false 54 | style: form 55 | schema: 56 | type: string 57 | responses: 58 | 200: 59 | description: successful operation 60 | content: 61 | text/html: 62 | schema: 63 | type: string 64 | 400: 65 | description: Must have a key to clone from 66 | content: {} 67 | /api/bookstore/clone: 68 | post: 69 | tags: 70 | - clone 71 | summary: Trigger clone from s3 72 | requestBody: 73 | description: Information about which notebook to clone from s3 74 | content: 75 | application/json: 76 | schema: 77 | $ref: '#/components/schemas/S3CloneFileRequest' 78 | required: true 79 | responses: 80 | 200: 81 | description: Successfully cloned 82 | content: 83 | application/json: 84 | schema: 85 | $ref: '#/components/schemas/Contents' 86 | 400: 87 | description: Must have a key to clone from 88 | content: {} 89 | /bookstore/fs-clone: 90 | get: 91 | tags: 92 | - clone 93 | summary: Landing page for initiating file-system cloning. 94 | description: This serves a simple html page that allows avoiding xsrf issues on a jupyter server. 95 | parameters: 96 | - name: relpath 97 | in: query 98 | description: relative path being targeted 99 | required: true 100 | style: form 101 | schema: 102 | type: string 103 | responses: 104 | 200: 105 | description: successful operation 106 | content: 107 | text/html: 108 | schema: 109 | type: string 110 | 400: 111 | description: Request malformed, must provide a relative path. 112 | content: {} 113 | 404: 114 | description: Request to clone from a path outside of base directory 115 | content: {} 116 | /api/bookstore/fs-clone: 117 | post: 118 | tags: 119 | - clone 120 | summary: Trigger clone from file system 121 | requestBody: 122 | description: Information about what to clone from the accessible file system 123 | content: 124 | application/json: 125 | schema: 126 | $ref: '#/components/schemas/FSCloneFileRequest' 127 | required: true 128 | responses: 129 | 200: 130 | description: Successfully cloned 131 | content: 132 | application/json: 133 | schema: 134 | $ref: '#/components/schemas/Contents' 135 | 400: 136 | description: Malformed request. Provide a valid relative path. 137 | content: {} 138 | 404: 139 | description: Invalid request. Cloning from a path outside of the base directory is not allowed. 140 | content: {} 141 | /api/bookstore/publish/{path}: 142 | put: 143 | tags: 144 | - publish 145 | parameters: 146 | - in: path 147 | name: path 148 | required: true 149 | schema: 150 | type: string 151 | description: Path to publish to, it will be prefixed by the preconfigured published bucket. 152 | summary: Publish a notebook to s3 153 | requestBody: 154 | description: Information about the notebook contents to publish to s3 155 | content: 156 | application/json: 157 | schema: 158 | $ref: '#/components/schemas/PublishableContents' 159 | required: true 160 | 161 | responses: 162 | 200: 163 | description: Successfully published. 164 | content: 165 | application/json: 166 | schema: 167 | $ref: '#/components/schemas/S3PublishFileResponse' 168 | 169 | components: 170 | schemas: 171 | S3CloneFileRequest: 172 | type: object 173 | required: 174 | - s3_bucket 175 | - s3_key 176 | properties: 177 | s3_bucket: 178 | type: string 179 | s3_key: 180 | type: string 181 | s3_version_id: 182 | type: string 183 | target_path: 184 | type: string 185 | FSCloneFileRequest: 186 | type: object 187 | required: 188 | - relpath 189 | properties: 190 | relpath: 191 | type: string 192 | target_path: 193 | type: string 194 | S3PublishFileResponse: 195 | type: object 196 | required: 197 | - s3_path 198 | properties: 199 | s3_path: 200 | type: string 201 | versionID: 202 | type: string 203 | PublishableContents: 204 | description: "A object representing contents that can be published. This is currently a subset of the fields required for the Contents API." 205 | type: object 206 | required: 207 | - type 208 | - content 209 | properties: 210 | type: 211 | type: string 212 | description: Type of content 213 | enum: 214 | - notebook 215 | content: 216 | $ref: https://raw.githubusercontent.com/jupyter/nbformat/master/nbformat/v4/nbformat.v4.schema.json 217 | Contents: 218 | description: "A contents object. The content and format keys may be null if content is not contained. If type is 'file', then the mimetype will be null." 219 | type: object 220 | required: 221 | - type 222 | - name 223 | - path 224 | - writable 225 | - created 226 | - last_modified 227 | - mimetype 228 | - format 229 | - content 230 | properties: 231 | name: 232 | type: string 233 | description: "Name of file or directory, equivalent to the last part of the path" 234 | path: 235 | type: string 236 | description: Full path for file or directory 237 | type: 238 | type: string 239 | description: Type of content 240 | enum: 241 | - directory 242 | - file 243 | - notebook 244 | writable: 245 | type: boolean 246 | description: indicates whether the requester has permission to edit the file 247 | created: 248 | type: string 249 | description: Creation timestamp 250 | format: date-time 251 | last_modified: 252 | type: string 253 | description: Last modified timestamp 254 | format: date-time 255 | size: 256 | type: integer 257 | description: "The size of the file or notebook in bytes. If no size is provided, defaults to null." 258 | mimetype: 259 | type: string 260 | description: "The mimetype of a file. If content is not null, and type is 'file', this will contain the mimetype of the file, otherwise this will be null." 261 | content: 262 | type: string 263 | description: "The content, if requested (otherwise null). Will be an array if type is 'directory'" 264 | format: 265 | type: string 266 | description: Format of content (one of null, 'text', 'base64', 'json') 267 | FeatureValidationInfo: 268 | type: object 269 | required: 270 | - bookstore_valid 271 | - archive_valid 272 | - publish_valid 273 | - s3_clone_valid 274 | - fs_clone_valid 275 | properties: 276 | bookstore_valid: 277 | type: boolean 278 | archive_valid: 279 | type: boolean 280 | publish_valid: 281 | type: boolean 282 | s3_clone_valid: 283 | type: boolean 284 | fs_clone_valid: 285 | type: boolean 286 | VersionInfo: 287 | type: object 288 | properties: 289 | release: 290 | type: string 291 | features: 292 | $ref: '#/components/schemas/FeatureValidationInfo' 293 | -------------------------------------------------------------------------------- /ci/integration.js: -------------------------------------------------------------------------------- 1 | const child_process = require("child_process"); 2 | const path = require("path"); 3 | const fs = require("fs"); 4 | 5 | const _ = require("lodash"); 6 | 7 | const s3 = require("./s3"); 8 | const { JupyterServer } = require("./jupyter"); 9 | const { url_path_join } = require("./utils"); 10 | 11 | const { sleep } = require("./sleep"); 12 | 13 | // Keep the jupyter server around to make sure we can destroy on bad exit 14 | let jupyterServer = null; 15 | 16 | function cleanupJupyter() { 17 | if (jupyterServer && jupyterServer.process && !jupyterServer.process.killed) { 18 | console.log("cleaning up a rogue jupyter server"); 19 | jupyterServer.process.kill(); 20 | } 21 | } 22 | 23 | // Clean up on general close 24 | process.on("exit", cleanupJupyter); 25 | 26 | // Clean up on ctrl+c 27 | process.on("SIGINT", cleanupJupyter); 28 | 29 | // Clean up from `kill pid`, e.g. nodemon restart 30 | process.on("SIGUSR1", cleanupJupyter); 31 | process.on("SIGUSR2", cleanupJupyter); 32 | 33 | // Clean up from uncaught exceptions 34 | process.on("uncaughtException", cleanupJupyter); 35 | 36 | // Catch all rogue promise rejections to ensure we fail CI 37 | process.on("unhandledRejection", error => { 38 | cleanupJupyter(); 39 | console.log("unhandledRejection", error); 40 | console.error(error.stack); 41 | process.exit(2); 42 | }); 43 | 44 | console.log("running bookstore integration tests"); 45 | 46 | const main = async () => { 47 | const bucketName = "bookstore"; 48 | 49 | jupyterServer = new JupyterServer(); 50 | await jupyterServer.start(); 51 | 52 | const s3Config = { 53 | endPoint: "127.0.0.1", 54 | port: 9000, 55 | useSSL: false, 56 | accessKey: "ONLY_ON_CIRCLE", 57 | secretKey: "CAN_WE_DO_THIS" 58 | }; 59 | 60 | // Instantiate the minio client with the endpoint 61 | // and access keys as shown below. 62 | var s3Client = new s3.Client(s3Config); 63 | 64 | await s3Client.makeBucket(bucketName); 65 | console.log(`Created bucket ${bucketName}`); 66 | 67 | async function compareS3Notebooks(filepath, originalNotebook) { 68 | /***** Check notebook from S3 *****/ 69 | const rawNotebook = await s3Client.getObject( 70 | bucketName, 71 | `ci-workspace/${filepath}` 72 | ); 73 | 74 | console.log(filepath); 75 | console.log(rawNotebook); 76 | 77 | const notebook = JSON.parse(rawNotebook); 78 | 79 | if (!_.isEqual(notebook, originalNotebook)) { 80 | console.error("original"); 81 | console.error(originalNotebook); 82 | console.error("from s3"); 83 | console.error(notebook); 84 | throw new Error("Notebook on S3 does not match what we sent"); 85 | } 86 | 87 | console.log("Notebook on S3 matches what we sent"); 88 | 89 | /***** Check notebook from Disk *****/ 90 | const diskNotebook = await new Promise((resolve, reject) => 91 | fs.readFile(path.join(__dirname, filepath), (err, data) => { 92 | if (err) { 93 | reject(err); 94 | } else { 95 | resolve(JSON.parse(data)); 96 | } 97 | }) 98 | ); 99 | 100 | if (!_.isEqual(diskNotebook, originalNotebook)) { 101 | console.error("original"); 102 | console.error(originalNotebook); 103 | console.error("from disk"); 104 | console.error(diskNotebook); 105 | throw new Error("Notebook on Disk does not match what we sent"); 106 | } 107 | } 108 | 109 | async function comparePublishedNotebooks(filepath, originalNotebook) { 110 | /***** Check published notebook from S3 prefix *****/ 111 | const rawNotebook = await s3Client.getObject( 112 | bucketName, 113 | `ci-published/${filepath}` 114 | ); 115 | 116 | console.log(filepath); 117 | console.log(rawNotebook); 118 | 119 | const notebook = JSON.parse(rawNotebook); 120 | console.log("Checking whether Notebook on s3 matches what we sent."); 121 | 122 | if (!_.isEqual(notebook, originalNotebook)) { 123 | console.error("original"); 124 | console.error(originalNotebook); 125 | console.error("from s3"); 126 | console.error(notebook); 127 | throw new Error("Notebook on S3 does not match what we sent"); 128 | } 129 | 130 | console.log("Notebook on S3 matches what we sent"); 131 | } 132 | 133 | const originalNotebook = { 134 | cells: [ 135 | { 136 | cell_type: "code", 137 | execution_count: null, 138 | metadata: {}, 139 | outputs: [], 140 | source: ["import this"] 141 | } 142 | ], 143 | metadata: { 144 | kernelspec: { 145 | display_name: "Python 3", 146 | language: "python", 147 | name: "python3" 148 | }, 149 | language_info: { 150 | codemirror_mode: { 151 | name: "ipython", 152 | version: 3 153 | }, 154 | file_extension: ".py", 155 | mimetype: "text/x-python", 156 | name: "python", 157 | nbconvert_exporter: "python", 158 | pygments_lexer: "ipython3", 159 | version: "3.7.0" 160 | } 161 | }, 162 | nbformat: 4, 163 | nbformat_minor: 2 164 | }; 165 | 166 | const EmptyNotebook = { 167 | cells: [ 168 | { 169 | cell_type: "code", 170 | execution_count: null, 171 | metadata: {}, 172 | outputs: [], 173 | source: [] 174 | } 175 | ], 176 | metadata: { 177 | kernelspec: { 178 | display_name: "dev", 179 | language: "python", 180 | name: "dev" 181 | }, 182 | language_info: { 183 | codemirror_mode: { 184 | name: "ipython", 185 | version: 3 186 | }, 187 | file_extension: ".py", 188 | mimetype: "text/x-python", 189 | name: "python", 190 | nbconvert_exporter: "python", 191 | pygments_lexer: "ipython3", 192 | version: "3.6.8" 193 | } 194 | }, 195 | nbformat: 4, 196 | nbformat_minor: 2 197 | }; 198 | 199 | await jupyterServer.writeNotebook( 200 | "ci-local-writeout.ipynb", 201 | originalNotebook 202 | ); 203 | 204 | await jupyterServer.publishNotebook("ci-published.ipynb", originalNotebook); 205 | 206 | await comparePublishedNotebooks("ci-published.ipynb", originalNotebook); 207 | 208 | const basicNotebook = { 209 | cells: [], 210 | nbformat: 4, 211 | nbformat_minor: 2, 212 | metadata: { 213 | save: 1 214 | } 215 | }; 216 | 217 | for (var ii = 0; ii < 4; ii++) { 218 | await jupyterServer.writeNotebook("ci-local-writeout2.ipynb", { 219 | cells: [], 220 | nbformat: 4, 221 | nbformat_minor: 2, 222 | metadata: { 223 | save: ii 224 | } 225 | }); 226 | await jupyterServer.writeNotebook("ci-local-writeout3.ipynb", { 227 | cells: [{ cell_type: "markdown", source: "# Hello world", metadata: {} }], 228 | nbformat: 4, 229 | nbformat_minor: 2, 230 | metadata: {} 231 | }); 232 | await sleep(100); 233 | } 234 | 235 | function s3CloneLandingQueryCheck(s3Key, expectedQueryString) { 236 | const populatedQueryString = jupyterServer.populateS3CloneLandingQuery( 237 | bucketName, 238 | s3Key 239 | ); 240 | if (!_.isEqual(populatedQueryString, expectedQueryString)) { 241 | console.error("created"); 242 | console.error(populatedQueryString); 243 | console.error("expected"); 244 | console.error(expectedQueryString); 245 | throw new Error("Query was not formed properly"); 246 | } 247 | console.log(`Query matched ${expectedQueryString}`); 248 | } 249 | 250 | function checkS3CloneLandingResponse(res, expected) { 251 | const content = res.response; 252 | if (!content.includes(expected)) { 253 | console.error("Response is ill-formed:"); 254 | console.error(content); 255 | console.error("It is expected to contain:"); 256 | console.error(expected); 257 | throw new Error("Ill-formed response."); 258 | } 259 | console.log(`Clone endpoint for ${expected} reached successfully!`); 260 | } 261 | 262 | const publishedPath = "ci-published/ci-published.ipynb"; 263 | s3CloneLandingQueryCheck( 264 | publishedPath, 265 | url_path_join( 266 | jupyterServer.endpoint, 267 | `bookstore/clone?s3_bucket=${bucketName}&s3_key=${publishedPath}` 268 | ) 269 | ); 270 | const s3CloneLandingRes = await jupyterServer.cloneS3NotebookLanding( 271 | bucketName, 272 | publishedPath 273 | ); 274 | checkS3CloneLandingResponse(s3CloneLandingRes, publishedPath); 275 | 276 | await jupyterServer.cloneS3Notebook(bucketName, publishedPath); 277 | await jupyterServer.cloneFSNotebook("test_files/EmptyNotebook.ipynb"); 278 | // Wait for minio to have the notebook 279 | // Future iterations of this script should poll to get the notebook 280 | await sleep(2000); 281 | 282 | await compareS3Notebooks("ci-published.ipynb", originalNotebook); 283 | await compareS3Notebooks("ci-local-writeout.ipynb", originalNotebook); 284 | await compareS3Notebooks("ci-local-writeout2.ipynb", { 285 | cells: [], 286 | nbformat: 4, 287 | nbformat_minor: 2, 288 | metadata: { 289 | save: 3 290 | } 291 | }); 292 | await compareS3Notebooks("EmptyNotebook.ipynb", EmptyNotebook); 293 | 294 | await sleep(700); 295 | 296 | await jupyterServer.deleteNotebook("ci-published.ipynb"); 297 | await jupyterServer.deleteNotebook("ci-local-writeout.ipynb"); 298 | await jupyterServer.deleteNotebook("ci-local-writeout2.ipynb"); 299 | await jupyterServer.deleteNotebook("ci-local-writeout3.ipynb"); 300 | await jupyterServer.deleteNotebook("EmptyNotebook.ipynb"); 301 | 302 | jupyterServer.shutdown(); 303 | 304 | console.log("📚 Bookstore Integration Complete 📚"); 305 | }; 306 | 307 | main(); 308 | -------------------------------------------------------------------------------- /bookstore/client/nb_client.py: -------------------------------------------------------------------------------- 1 | """Client for accessing notebook server endpoints from within a notebook. 2 | """ 3 | import json 4 | import os 5 | import re 6 | from copy import deepcopy 7 | from typing import NamedTuple 8 | 9 | import requests 10 | from IPython import get_ipython 11 | from notebook.notebookapp import list_running_servers 12 | 13 | 14 | def extract_kernel_id(connection_file): 15 | """Get the kernel id string from a file""" 16 | # regex is used as a more robust approach than lstrip 17 | connection_filename = os.path.basename(connection_file) 18 | kernel_id = re.sub(r"kernel-(.*)\.json", r"\1", connection_filename) 19 | return kernel_id 20 | 21 | 22 | class LiveNotebookRecord(NamedTuple): 23 | """Representation of live notebook server. 24 | 25 | This is a record of an object returned by 26 | `notebook.notebookapp.list_running_servers()`. 27 | 28 | Example 29 | ------- 30 | :: 31 | 32 | [{'base_url': '/', 33 | 'hostname': 'localhost', 34 | 'notebook_dir': '/Users/mpacer/jupyter/eg_notebooks', 35 | 'password': False, 36 | 'pid': 96033, 37 | 'port': 8888, 38 | 'secure': False, 39 | 'token': '', 40 | 'url': 'http://localhost:8888/'}] 41 | 42 | """ 43 | 44 | base_url: str 45 | hostname: str 46 | notebook_dir: str 47 | password: bool 48 | pid: int 49 | port: int 50 | secure: bool 51 | token: str 52 | url: str 53 | 54 | 55 | class KernelInfo: 56 | """Representation of kernel info returned by the notebook's /api/kernel endpoint. 57 | 58 | Attributes 59 | ---------- 60 | id: str 61 | name: str 62 | last_activity: str 63 | execution_state: str 64 | connections: int 65 | 66 | Example 67 | ------- 68 | :: 69 | 70 | {id: 'f92b7c8b-0858-4d10-903c-b0631540fb36', 71 | name: 'dev', 72 | last_activity: '2019-03-14T23:38:08.137987Z', 73 | execution_state: 'idle', 74 | connections: 0} 75 | """ 76 | 77 | def __init__(self, *args, id, name, last_activity, execution_state, connections): 78 | self.model = { 79 | "id": id, 80 | "name": name, 81 | "last_activity": last_activity, 82 | "execution_state": execution_state, 83 | "connections": connections, 84 | } 85 | self.id = id 86 | self.name = name 87 | self.last_activity = last_activity 88 | self.execution_state = execution_state 89 | self.connections = connections 90 | 91 | def __repr__(self): 92 | return json.dumps(self.model, indent=2) 93 | 94 | def __eq__(self, other): 95 | if isinstance(other, KernelInfo): 96 | cmp_attrs = [ 97 | self.id == other.id, 98 | self.name == other.name, 99 | self.last_activity == other.last_activity, 100 | self.execution_state == other.execution_state, 101 | self.connections == other.connections, 102 | ] 103 | return all(cmp_attrs) 104 | else: 105 | return False 106 | 107 | 108 | class NotebookSession: 109 | """Representation of session info returned by the notebook's /api/sessions/ endpoint. 110 | 111 | Attributes 112 | ---------- 113 | id: str 114 | path: str 115 | name: str 116 | type: str 117 | kernel: KernelInfo 118 | notebook: dict 119 | model: dict 120 | Record of the raw response (without converting the KernelInfo). 121 | 122 | Example 123 | ------- 124 | :: 125 | 126 | {id: '68d9c58f-c57d-4133-8b41-5ec2731b268d', 127 | path: 'Untitled38.ipynb', 128 | name: '', 129 | type: 'notebook', 130 | kernel: KernelInfo(id='f92b7c8b-0858-4d10-903c-b0631540fb36', 131 | name='dev', 132 | last_activity='2019-03-14T23:38:08.137987Z', 133 | execution_state='idle', 134 | connections=0), 135 | notebook: {'path': 'Untitled38.ipynb', 'name': ''}} 136 | """ 137 | 138 | def __init__(self, *args, path, name, type, kernel, notebook={}, **kwargs): 139 | self.model = { 140 | "path": path, 141 | "name": name, 142 | "type": type, 143 | "kernel": kernel, 144 | "notebook": notebook, 145 | } 146 | self.path = path 147 | self.name = name 148 | self.type = type 149 | self.kernel = KernelInfo(**kernel) 150 | self.notebook = notebook 151 | 152 | def __repr__(self): 153 | return json.dumps(self.model, indent=2) 154 | 155 | def __eq__(self, other): 156 | if isinstance(other, NotebookSession): 157 | cmp_attrs = [ 158 | # self.id == other.id, 159 | self.path == other.path, 160 | self.name == other.name, 161 | self.type == other.type, 162 | self.kernel == other.kernel, 163 | self.notebook == other.notebook, 164 | ] 165 | return all(cmp_attrs) 166 | else: 167 | return False 168 | 169 | 170 | class NotebookClient: 171 | """EXPERIMENTAL SUPPORT: Client used to interact with a notebook server from within a notebook. 172 | 173 | Parameters 174 | ---------- 175 | nb_config: dict 176 | Dictionary of info compatible with creating a LiveNotebookRecord. 177 | 178 | Attributes 179 | ---------- 180 | nb_config: dict 181 | Dictionary of info compatible with creating a LiveNotebookRecord. 182 | nb_record: LiveNotebookRecord 183 | LiveNotebookRecord of info for this notebook 184 | url: str 185 | url from nb_record minus final / 186 | token: str 187 | token used for authenticating requests serverside 188 | xsrf_token: str 189 | xsrf_token used in cookie for authenticating requests 190 | req_session: requests.Session 191 | Session to be reused across methods 192 | """ 193 | 194 | def __init__(self, nb_config): 195 | self.nb_config = nb_config 196 | self.nb_record = LiveNotebookRecord(**self.nb_config) 197 | self.url = self.nb_record.url.rstrip( 198 | "/" 199 | ) # So that we can have full API endpoints without double // 200 | self.setup_auth() 201 | self.setup_request_sessions() 202 | 203 | def setup_auth(self): 204 | """ Sets up token access for authorizing requests to notebook server. 205 | 206 | This sets the notebook token as self.token and the xsrf_token as self.xsrf_token. 207 | """ 208 | self.token = self.nb_record.token 209 | first = requests.get(f"{self.url}/login") 210 | self.xsrf_token = first.cookies.get("_xsrf", "") 211 | 212 | def setup_request_sessions(self): 213 | """ Sets up a requests.Session object for sharing headers across API requests. """ 214 | self.req_session = requests.Session() 215 | self.req_session.headers.update(self.headers) 216 | 217 | @property 218 | def sessions(self): 219 | """Current notebook sessions. Reissues request on each call. """ 220 | return { 221 | session['kernel']['id']: NotebookSession(**session) for session in self.get_sessions() 222 | } 223 | 224 | @property 225 | def headers(self): 226 | """Default headers to be shared across requests. """ 227 | headers = { 228 | 'Authorization': f'token {self.token}', 229 | 'X-XSRFToken': self.xsrf_token, 230 | "Content-Type": "application/json", 231 | } 232 | return headers 233 | 234 | @property 235 | def sessions_endpoint(self): 236 | """Current server's kernels API endpoint.""" 237 | api_endpoint = "/api/sessions/" 238 | return f"{self.url}{api_endpoint}" 239 | 240 | def get_sessions(self): 241 | """Requests info about current sessions from notebook server.""" 242 | target_url = f"{self.sessions_endpoint}" 243 | resp = self.req_session.get(target_url) 244 | return resp.json() 245 | 246 | @property 247 | def kernels_endpoint(self): 248 | """Current server's kernels API endpoint.""" 249 | api_endpoint = "/api/kernels/" 250 | return f"{self.url}{api_endpoint}" 251 | 252 | def get_kernels(self): 253 | """Requests info about current kernels from notebook server.""" 254 | target_url = f"{self.kernels_endpoint}" 255 | resp = self.req_session.get(target_url) 256 | return resp.json() 257 | 258 | @property 259 | def kernels(self): 260 | """Current notebook kernels. Reissues request on each call.""" 261 | return self.get_kernels() 262 | 263 | @property 264 | def contents_endpoint(self): 265 | """Current server's contents API endpoint.""" 266 | api_endpoint = "/api/contents/" 267 | return f"{self.url}{api_endpoint}" 268 | 269 | def get_contents(self, path): 270 | """Requests info about current contents from notebook server.""" 271 | target_url = f"{self.contents_endpoint}{path}" 272 | resp = self.req_session.get(target_url) 273 | return resp.json() 274 | 275 | 276 | class NotebookClientCollection: 277 | """EXPERIMENTAL SUPPORT: Representation of a collection of notebook clients""" 278 | 279 | # TODO: refactor from lambda to a def 280 | nb_client_gen = lambda: (NotebookClient(x) for x in list_running_servers()) 281 | sessions = {x.url: x.sessions for x in nb_client_gen()} 282 | 283 | @classmethod 284 | def current_server(cls): 285 | """class method for current notebook server""" 286 | 287 | current_kernel_id = extract_kernel_id(get_ipython().parent.parent.connection_file) 288 | for server_url, session_dict in cls.sessions.items(): 289 | for session_id, session in session_dict.items(): 290 | if session.kernel.id == current_kernel_id: 291 | return next( 292 | client for client in cls.nb_client_gen() if client.url == server_url 293 | ) 294 | 295 | 296 | class CurrentNotebookClient(NotebookClient): 297 | """EXPERIMENTAL SUPPORT: Represents the currently active notebook client.""" 298 | 299 | def __init__(self): 300 | self.nb_client = NotebookClientCollection.current_server() 301 | super().__init__(self.nb_client.nb_config) 302 | self.session = self.sessions[self.kernel_id] 303 | 304 | @property 305 | def connection_file(self): 306 | """Connection file for connecting to current notebook's kernel.""" 307 | return get_ipython().parent.parent.connection_file 308 | 309 | @property 310 | def kernel_id(self): 311 | """Kernel id for identifying which notebook is currently being used by this session.""" 312 | return extract_kernel_id(self.connection_file) 313 | -------------------------------------------------------------------------------- /bookstore/tests/test_clone.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import uuid 4 | import os 5 | 6 | from unittest.mock import Mock 7 | from pathlib import Path 8 | 9 | import pytest 10 | import nbformat 11 | 12 | from jinja2 import Environment 13 | from notebook.services.contents.filemanager import FileContentsManager 14 | from tornado.testing import AsyncTestCase, gen_test 15 | from tornado.web import Application, HTTPError 16 | from tornado.httpserver import HTTPRequest 17 | from traitlets.config import Config 18 | 19 | from bookstore.bookstore_config import BookstoreSettings 20 | from bookstore.clone import ( 21 | build_notebook_model, 22 | build_file_model, 23 | BookstoreCloneHandler, 24 | BookstoreCloneAPIHandler, 25 | validate_relpath, 26 | BookstoreFSCloneHandler, 27 | BookstoreFSCloneAPIHandler, 28 | ) 29 | from bookstore.utils import TemporaryWorkingDirectory 30 | 31 | from . import test_dir 32 | 33 | 34 | log = logging.getLogger('test_clone') 35 | 36 | 37 | def test_build_notebook_model(): 38 | content = nbformat.v4.new_notebook() 39 | expected = { 40 | "type": "notebook", 41 | "format": "json", 42 | "content": content, 43 | "name": "my_notebook_name.ipynb", 44 | "path": "test_directory/my_notebook_name.ipynb", 45 | } 46 | path = "./test_directory/my_notebook_name.ipynb" 47 | nb_content = nbformat.writes(content) 48 | assert build_notebook_model(nb_content, path) == expected 49 | 50 | 51 | def test_build_file_model(): 52 | content = "my fancy file" 53 | expected = { 54 | "type": "file", 55 | "format": "text", 56 | "content": content, 57 | "name": "file_name.txt", 58 | "path": "test_directory/file_name.txt", 59 | } 60 | path = "./test_directory/file_name.txt" 61 | assert build_file_model(content, path) == expected 62 | 63 | 64 | class TestCloneHandler(AsyncTestCase): 65 | def setUp(self): 66 | super().setUp() 67 | self.mock_application = Mock( 68 | spec=Application, 69 | ui_methods={}, 70 | ui_modules={}, 71 | # base_url in Tornado settings must begin and end with '/' 72 | settings={'jinja2_env': Environment(), "base_url": "/"}, 73 | ) 74 | 75 | def get_handler(self, uri, app=None): 76 | if app is None: 77 | app = self.mock_application 78 | connection = Mock(context=Mock(protocol="https")) 79 | payload_request = HTTPRequest( 80 | method='GET', 81 | uri=uri, 82 | headers={"Host": "localhost:8888"}, 83 | body=None, 84 | connection=connection, 85 | ) 86 | return BookstoreCloneHandler(app, payload_request) 87 | 88 | @gen_test 89 | async def test_get_no_param(self): 90 | empty_handler = self.get_handler('/bookstore/clone') 91 | with pytest.raises(HTTPError): 92 | await empty_handler.get() 93 | 94 | @gen_test 95 | async def test_get_no_bucket(self): 96 | no_bucket_handler = self.get_handler('/bookstore/clone?s3_bucket=&s3_key=hi') 97 | with pytest.raises(HTTPError): 98 | await no_bucket_handler.get() 99 | 100 | @gen_test 101 | async def test_get_no_object_key(self): 102 | no_object_key_handler = self.get_handler('/bookstore/clone?s3_bucket=hello&s3_key=') 103 | with pytest.raises(HTTPError): 104 | await no_object_key_handler.get() 105 | 106 | @gen_test 107 | async def test_get_success(self): 108 | success_handler = self.get_handler('/bookstore/clone?s3_bucket=hello&s3_key=my_key') 109 | await success_handler.get() 110 | 111 | def test_gen_template_params(self): 112 | expected = { 113 | 'post_model': {'s3_bucket': 'hello', 's3_key': 'my_key', 's3_version_id': None}, 114 | 'clone_api_url': '/api/bookstore/clone', 115 | 'redirect_contents_url': '/', 116 | 'source_description': "'my_key' from the s3 bucket 'hello'", 117 | } 118 | success_handler = self.get_handler('/bookstore/clone?s3_bucket=hello&s3_key=my_key') 119 | output = success_handler.construct_template_params( 120 | s3_bucket="hello", s3_object_key="my_key" 121 | ) 122 | assert expected == output 123 | 124 | def test_gen_template_params_s3_version_id(self): 125 | expected = { 126 | 'post_model': {'s3_bucket': 'hello', 's3_key': 'my_key', 's3_version_id': "my_version"}, 127 | 'clone_api_url': '/api/bookstore/clone', 128 | 'redirect_contents_url': '/', 129 | 'source_description': "'my_key' version: my_version from the s3 bucket 'hello'", 130 | } 131 | success_handler = self.get_handler( 132 | '/bookstore/clone?s3_bucket=hello&s3_key=my_key&s3_version_id=my_version' 133 | ) 134 | output = success_handler.construct_template_params( 135 | s3_bucket="hello", s3_object_key="my_key", s3_version_id="my_version" 136 | ) 137 | assert expected == output 138 | 139 | def test_gen_template_params_base_url(self): 140 | expected = { 141 | 'post_model': {'s3_bucket': 'hello', 's3_key': 'my_key', 's3_version_id': None}, 142 | 'clone_api_url': '/my_base_url/api/bookstore/clone', 143 | 'redirect_contents_url': '/my_base_url', 144 | 'source_description': "'my_key' from the s3 bucket 'hello'", 145 | } 146 | mock_app = Mock( 147 | spec=Application, 148 | ui_methods={}, 149 | ui_modules={}, 150 | settings={'jinja2_env': Environment(), "base_url": "/my_base_url/"}, 151 | ) 152 | 153 | success_handler = self.get_handler( 154 | '/bookstore/clone?s3_bucket=hello&s3_key=my_key', app=mock_app 155 | ) 156 | output = success_handler.construct_template_params( 157 | s3_bucket="hello", s3_object_key="my_key" 158 | ) 159 | assert expected == output 160 | 161 | 162 | class TestCloneAPIHandler(AsyncTestCase): 163 | def setUp(self): 164 | super().setUp() 165 | mock_settings = { 166 | "BookstoreSettings": { 167 | "s3_access_key_id": "mock_id", 168 | "s3_secret_access_key": "mock_access", 169 | } 170 | } 171 | config = Config(mock_settings) 172 | 173 | self.mock_application = Mock( 174 | spec=Application, 175 | ui_methods={}, 176 | ui_modules={}, 177 | settings={ 178 | 'jinja2_env': Environment(), 179 | "config": config, 180 | "contents_manager": FileContentsManager(), 181 | }, 182 | ) 183 | 184 | def post_handler(self, body_dict, app=None): 185 | if app is None: 186 | app = self.mock_application 187 | body = json.dumps(body_dict).encode('utf-8') 188 | payload_request = HTTPRequest( 189 | method='POST', uri="/api/bookstore/clone", headers=None, body=body, connection=Mock() 190 | ) 191 | return BookstoreCloneAPIHandler(app, payload_request) 192 | 193 | @gen_test 194 | async def test_post_no_body(self): 195 | post_body_dict = {} 196 | empty_handler = self.post_handler(post_body_dict) 197 | with pytest.raises(HTTPError): 198 | await empty_handler.post() 199 | 200 | @gen_test 201 | async def test_post_empty_bucket(self): 202 | post_body_dict = {"s3_key": "my_key", "s3_bucket": ""} 203 | empty_bucket_handler = self.post_handler(post_body_dict) 204 | with pytest.raises(HTTPError): 205 | await empty_bucket_handler.post() 206 | 207 | @gen_test 208 | async def test_post_empty_key(self): 209 | post_body_dict = {"s3_key": "", "s3_bucket": "my_bucket"} 210 | empty_key_handler = self.post_handler(post_body_dict) 211 | with pytest.raises(HTTPError): 212 | await empty_key_handler.post() 213 | 214 | @gen_test 215 | async def test_post_nonsense_params(self): 216 | post_body_dict = {"s3_key": "my_key", "s3_bucket": "my_bucket"} 217 | success_handler = self.post_handler(post_body_dict) 218 | with pytest.raises(HTTPError): 219 | await success_handler.post() 220 | 221 | @gen_test 222 | async def test_private_clone_nonsense_params(self): 223 | s3_bucket = "my_key" 224 | s3_object_key = "my_bucket" 225 | post_body_dict = {"s3_key": s3_object_key, "s3_bucket": s3_bucket} 226 | success_handler = self.post_handler(post_body_dict) 227 | with pytest.raises(HTTPError): 228 | await success_handler._clone(s3_bucket, s3_object_key) 229 | 230 | def test_build_s3_request_object(self): 231 | expected = {"Bucket": "my_bucket", "Key": "my_key"} 232 | s3_bucket = "my_bucket" 233 | s3_object_key = "my_key" 234 | post_body_dict = {"s3_key": "my_key", "s3_bucket": "my_bucket"} 235 | success_handler = self.post_handler(post_body_dict) 236 | actual = success_handler._build_s3_request_object(s3_bucket, s3_object_key) 237 | assert actual == expected 238 | 239 | def test_build_s3_request_object_version_id(self): 240 | expected = {"Bucket": "my_bucket", "Key": "my_key", "VersionId": "my_version"} 241 | 242 | s3_bucket = "my_bucket" 243 | s3_object_key = "my_key" 244 | s3_version_id = "my_version" 245 | 246 | post_body_dict = { 247 | "s3_key": s3_object_key, 248 | "s3_bucket": s3_bucket, 249 | "s3_version_id": s3_version_id, 250 | } 251 | success_handler = self.post_handler(post_body_dict) 252 | actual = success_handler._build_s3_request_object( 253 | s3_bucket, s3_object_key, s3_version_id=s3_version_id 254 | ) 255 | assert actual == expected 256 | 257 | def test_build_post_response_model(self): 258 | content = "some arbitrary content" 259 | expected = { 260 | "type": "file", 261 | "format": "text", 262 | "content": content, 263 | "name": "file_name.txt", 264 | "path": "test_directory/file_name.txt", 265 | "s3_path": "s3://my_bucket/original_key/may_be_different_than_storage.txt", 266 | 'versionID': "eeee222eee", 267 | } 268 | 269 | s3_bucket = "my_bucket" 270 | s3_object_key = "original_key/may_be_different_than_storage.txt" 271 | obj = {'VersionId': "eeee222eee"} 272 | model = { 273 | "type": "file", 274 | "format": "text", 275 | "content": content, 276 | "name": "file_name.txt", 277 | "path": "test_directory/file_name.txt", 278 | } 279 | handler = self.post_handler({}) 280 | actual = handler.build_post_response_model(model, obj, s3_bucket, s3_object_key) 281 | assert actual == expected 282 | 283 | @gen_test 284 | async def test_build_text_content_model(self): 285 | content = "some content" 286 | expected = { 287 | "type": "file", 288 | "format": "text", 289 | "content": content, 290 | "name": "file_name.txt", 291 | "path": "test_directory/file_name.txt", 292 | } 293 | 294 | path = "test_directory/file_name.txt" 295 | success_handler = self.post_handler({}) 296 | model = success_handler.build_content_model(content, path) 297 | assert model == expected 298 | 299 | @gen_test 300 | async def test_build_notebook_content_model(self): 301 | content = nbformat.v4.new_notebook() 302 | expected = { 303 | "type": "notebook", 304 | "format": "json", 305 | "content": content, 306 | "name": "file_name.ipynb", 307 | "path": "test_directory/file_name.ipynb", 308 | } 309 | 310 | str_content = nbformat.writes(content) 311 | 312 | path = "test_directory/file_name.ipynb" 313 | success_handler = self.post_handler({}) 314 | model = success_handler.build_content_model(str_content, path) 315 | assert model == expected 316 | 317 | 318 | def test_validate_relpath(): 319 | relpath = 'hi' 320 | settings = BookstoreSettings(fs_cloning_basedir="/anything") 321 | fs_clonepath = validate_relpath(relpath, settings, log) 322 | assert fs_clonepath == Path("/anything/hi") 323 | 324 | 325 | def test_validate_relpath_empty_relpath(caplog): 326 | relpath = '' 327 | settings = BookstoreSettings(fs_cloning_basedir="/anything") 328 | with pytest.raises(HTTPError): 329 | with caplog.at_level(logging.INFO): 330 | fs_clonepath = validate_relpath(relpath, settings, log) 331 | assert "Request received with empty relpath." in caplog.text 332 | 333 | 334 | def test_validate_relpath_escape_basedir(caplog): 335 | relpath = '../hi' 336 | settings = BookstoreSettings(fs_cloning_basedir="/anything") 337 | with pytest.raises(HTTPError): 338 | with caplog.at_level(logging.INFO): 339 | fs_clonepath = validate_relpath(relpath, settings, log) 340 | assert f"Request to clone from a path outside of base directory" in caplog.text 341 | 342 | 343 | class TestFSCloneHandler(AsyncTestCase): 344 | def setUp(self): 345 | super().setUp() 346 | mock_settings = {"BookstoreSettings": {"fs_cloning_basedir": test_dir}} 347 | self.config = Config(mock_settings) 348 | self.mock_application = Mock( 349 | spec=Application, 350 | ui_methods={}, 351 | ui_modules={}, 352 | settings={'jinja2_env': Environment(), 'config': self.config, "base_url": "/"}, 353 | ) 354 | 355 | def get_handler(self, uri, app=None): 356 | if app is None: 357 | app = self.mock_application 358 | connection = Mock(context=Mock(protocol="https")) 359 | payload_request = HTTPRequest( 360 | method='GET', 361 | uri=uri, 362 | headers={"Host": "localhost:8888"}, 363 | body=None, 364 | connection=connection, 365 | ) 366 | return BookstoreFSCloneHandler(app, payload_request) 367 | 368 | @gen_test 369 | async def test_get_no_param(self): 370 | empty_handler = self.get_handler('/bookstore/fs-clone') 371 | with pytest.raises(HTTPError): 372 | await empty_handler.get() 373 | 374 | @gen_test 375 | async def test_get_empty_relpath(self): 376 | empty_relpath_handler = self.get_handler('/bookstore/fs-clone?relpath=') 377 | with pytest.raises(HTTPError): 378 | await empty_relpath_handler.get() 379 | 380 | @gen_test 381 | async def test_get_escape_basedir(self): 382 | escape_basedir_handler = self.get_handler('/bookstore/fs-clone?relpath=../hi') 383 | with pytest.raises(HTTPError): 384 | await escape_basedir_handler.get() 385 | 386 | @gen_test 387 | async def test_get_success(self): 388 | success_handler = self.get_handler('/bookstore/fs-clone?relpath=my/test/path.ipynb') 389 | await success_handler.get() 390 | 391 | def test_gen_template_params(self): 392 | expected = { 393 | 'post_model': {'relpath': 'my/test/path.ipynb'}, 394 | 'clone_api_url': '/api/bookstore/fs-clone', 395 | 'redirect_contents_url': '/', 396 | 'source_description': '/Users/jupyter/my/test/path.ipynb', 397 | } 398 | success_handler = self.get_handler('/bookstore/fs-clone?relpath=my/test/path.ipynb') 399 | output = success_handler.construct_template_params( 400 | relpath='my/test/path.ipynb', fs_clonepath='/Users/jupyter/my/test/path.ipynb' 401 | ) 402 | assert expected == output 403 | 404 | def test_gen_template_params_base_url(self): 405 | # We can only test `base_url`s that begin and end with `/` 406 | mock_app = Mock( 407 | spec=Application, 408 | ui_methods={}, 409 | ui_modules={}, 410 | settings={ 411 | 'jinja2_env': Environment(), 412 | "base_url": '/my_base_url/', 413 | "config": self.config, 414 | }, 415 | ) 416 | 417 | expected = { 418 | 'post_model': {'relpath': 'my/test/path.ipynb'}, 419 | 'clone_api_url': '/my_base_url/api/bookstore/fs-clone', 420 | 'redirect_contents_url': '/my_base_url', 421 | 'source_description': '/Users/jupyter/my/test/path.ipynb', 422 | } 423 | 424 | success_handler = self.get_handler( 425 | '/bookstore/fs-clone?relpath=my/test/path.ipynb', app=mock_app 426 | ) 427 | output = success_handler.construct_template_params( 428 | relpath='my/test/path.ipynb', fs_clonepath='/Users/jupyter/my/test/path.ipynb' 429 | ) 430 | assert expected == output 431 | 432 | 433 | class TestFSCloneAPIHandler(AsyncTestCase): 434 | def setUp(self): 435 | super().setUp() 436 | mock_settings = { 437 | "BookstoreSettings": {"fs_cloning_basedir": os.path.join(test_dir, 'test_files')} 438 | } 439 | config = Config(mock_settings) 440 | 441 | self.mock_application = Mock( 442 | spec=Application, 443 | ui_methods={}, 444 | ui_modules={}, 445 | settings={ 446 | 'jinja2_env': Environment(), 447 | "config": config, 448 | "contents_manager": FileContentsManager(), 449 | "base_url": "/", 450 | }, 451 | ) 452 | 453 | def post_handler(self, body_dict, app=None): 454 | if app is None: 455 | app = self.mock_application 456 | body = json.dumps(body_dict).encode('utf-8') 457 | payload_request = HTTPRequest( 458 | method='POST', uri="/api/bookstore/fs-clone", headers=None, body=body, connection=Mock() 459 | ) 460 | return BookstoreFSCloneAPIHandler(app, payload_request) 461 | 462 | @gen_test 463 | async def test_post_no_body(self): 464 | post_body_dict = {} 465 | empty_handler = self.post_handler(post_body_dict) 466 | with pytest.raises(HTTPError): 467 | await empty_handler.post() 468 | 469 | @gen_test 470 | async def test_post_empty_relpath(self): 471 | post_body_dict = {"relpath": ""} 472 | empty_relpath_handler = self.post_handler(post_body_dict) 473 | with pytest.raises(HTTPError): 474 | await empty_relpath_handler.post() 475 | 476 | @gen_test 477 | async def test_post_basedir_escape(self): 478 | post_body_dict = {"relpath": "../myfile.txt"} 479 | empty_relpath_handler = self.post_handler(post_body_dict) 480 | with pytest.raises(HTTPError): 481 | await empty_relpath_handler.post() 482 | 483 | @gen_test 484 | async def test_post_nonsense_params(self): 485 | post_body_dict = {"relpath": str(uuid.uuid4())} 486 | success_handler = self.post_handler(post_body_dict) 487 | with pytest.raises(HTTPError): 488 | await success_handler.post() 489 | 490 | @gen_test 491 | async def test_post_success_notebook(self): 492 | post_body_dict = {"relpath": 'EmptyNotebook.ipynb'} 493 | with open(os.path.join(test_dir, 'test_files/EmptyNotebook.ipynb'), 'r') as f: 494 | expected = json.load(f) 495 | success_handler = self.post_handler(post_body_dict) 496 | setattr(success_handler, '_transforms', []) 497 | 498 | with TemporaryWorkingDirectory() as tmp: 499 | await success_handler.post() 500 | with open('EmptyNotebook.ipynb') as f: 501 | actual = json.load(f) 502 | assert actual == expected 503 | 504 | @gen_test 505 | async def test_build_text_content_model(self): 506 | content = "some content" 507 | expected = { 508 | "type": "file", 509 | "format": "text", 510 | "content": content, 511 | "name": "file_name.txt", 512 | "path": "test_directory/file_name.txt", 513 | } 514 | 515 | path = "test_directory/file_name.txt" 516 | post_handler = self.post_handler({}) 517 | model = post_handler.build_content_model(content, path) 518 | assert model == expected 519 | 520 | @gen_test 521 | async def test_build_notebook_content_model(self): 522 | content = nbformat.v4.new_notebook() 523 | expected = { 524 | "type": "notebook", 525 | "format": "json", 526 | "content": content, 527 | "name": "file_name.ipynb", 528 | "path": "test_directory/file_name.ipynb", 529 | } 530 | 531 | str_content = nbformat.writes(content) 532 | 533 | path = "test_directory/file_name.ipynb" 534 | post_handler = self.post_handler({}) 535 | model = post_handler.build_content_model(str_content, path) 536 | assert model == expected 537 | -------------------------------------------------------------------------------- /bookstore/clone.py: -------------------------------------------------------------------------------- 1 | """Handler to clone notebook from storage.""" 2 | import json 3 | import os 4 | 5 | from copy import deepcopy 6 | from pathlib import Path 7 | 8 | import aiobotocore 9 | 10 | from botocore.exceptions import ClientError 11 | from jinja2 import FileSystemLoader 12 | from notebook.base.handlers import IPythonHandler, APIHandler 13 | from tornado import web 14 | 15 | from . import PACKAGE_DIR 16 | from .bookstore_config import BookstoreSettings 17 | from .s3_paths import s3_path, s3_display_path 18 | from .utils import url_path_join 19 | 20 | BOOKSTORE_FILE_LOADER = FileSystemLoader(PACKAGE_DIR) 21 | 22 | 23 | def build_notebook_model(content, path): 24 | """Helper that builds a Contents API compatible model for notebooks. 25 | 26 | Parameters 27 | ---------- 28 | content : str 29 | The content of the model. 30 | path : str 31 | The path to be targeted. 32 | 33 | Returns 34 | -------- 35 | dict 36 | Jupyter Contents API compatible model for notebooks 37 | """ 38 | model = { 39 | "type": "notebook", 40 | "format": "json", 41 | "content": json.loads(content), 42 | "name": os.path.basename(os.path.relpath(path)), 43 | "path": os.path.relpath(path), 44 | } 45 | return model 46 | 47 | 48 | def build_file_model(content, path): 49 | """Helper that builds a Contents API compatible model for files. 50 | 51 | Parameters 52 | ---------- 53 | content: str 54 | The content of the model 55 | path : str 56 | The path to be targeted. 57 | 58 | Returns 59 | -------- 60 | dict 61 | Jupyter Contents API compatible model for files 62 | """ 63 | model = { 64 | "type": "file", 65 | "format": "text", 66 | "content": content, 67 | "name": os.path.basename(os.path.relpath(path)), 68 | "path": os.path.relpath(path), 69 | } 70 | return model 71 | 72 | 73 | class BookstoreCloneHandler(IPythonHandler): 74 | """Prepares and provides clone options page, populating UI with clone option parameters. 75 | 76 | Provides handling for ``GET`` requests when cloning a notebook 77 | from storage (S3). Launches a user interface with cloning options. 78 | 79 | Methods 80 | ------- 81 | initialize(self) 82 | Helper to access bookstore settings. 83 | get(self) 84 | Checks for valid storage settings and render a UI for clone options. 85 | construct_template_params(self, s3_bucket, s3_object_key, s3_version_id=None) 86 | Helper to populate Jinja template for cloning option page. 87 | get_template(self, name) 88 | Loads a Jinja template and its related settings. 89 | 90 | See also 91 | -------- 92 | `Jupyter Notebook reference on Custom Handlers `_ 93 | """ 94 | 95 | def initialize(self): 96 | """Helper to retrieve bookstore setting for the session.""" 97 | self.bookstore_settings = BookstoreSettings(config=self.config) 98 | 99 | @web.authenticated 100 | async def get(self): 101 | """GET /bookstore/clone?s3_bucket=&s3_key= 102 | 103 | Renders an options page that will allow you to clone a notebook 104 | from a specific bucket via the Bookstore cloning API. 105 | 106 | s3_bucket is the bucket you wish to clone from. 107 | s3_key is the object key that you wish to clone. 108 | """ 109 | s3_bucket = self.get_argument("s3_bucket") 110 | if s3_bucket == '' or s3_bucket == "/": 111 | raise web.HTTPError(400, "Requires an S3 bucket in order to clone") 112 | 113 | # s3_paths module has an s3_key function; s3_object_key avoids confusion 114 | s3_object_key = self.get_argument("s3_key") 115 | if s3_object_key == '' or s3_object_key == '/': 116 | raise web.HTTPError(400, "Requires an S3 object key in order to clone") 117 | 118 | s3_version_id = self.get_argument("s3_version_id", default=None) 119 | 120 | self.log.info(f"Setting up cloning landing page for {s3_object_key}") 121 | 122 | template_params = self.construct_template_params(s3_bucket, s3_object_key, s3_version_id) 123 | self.set_header('Content-Type', 'text/html') 124 | self.write(self.render_template('clone.html', **template_params)) 125 | 126 | def construct_template_params(self, s3_bucket, s3_object_key, s3_version_id=None): 127 | """Helper that takes valid S3 parameters and populates UI template 128 | 129 | Returns 130 | -------- 131 | 132 | dict 133 | Template parameters in a dictionary 134 | """ 135 | # here we closely match the logic in how default_url is defined in 136 | # https://github.com/jupyter/notebook/blob/55e93b9ffef0df9158586e65a34164f7a903e1e0/notebook/notebookapp.py#L1415-L1417 137 | if self.default_url.startswith(self.base_url): 138 | redirect_contents_url = self.default_url 139 | else: 140 | redirect_contents_url = url_path_join(self.base_url, self.default_url) 141 | clone_api_url = url_path_join(self.base_url, "/api/bookstore/clone") 142 | model = {"s3_bucket": s3_bucket, "s3_key": s3_object_key, "s3_version_id": s3_version_id} 143 | version_text = ' version: ' + s3_version_id if s3_version_id else '' 144 | template_params = { 145 | "post_model": model, 146 | "clone_api_url": clone_api_url, 147 | "redirect_contents_url": redirect_contents_url, 148 | "source_description": f"'{s3_object_key}'{version_text} from the s3 bucket '{s3_bucket}'", 149 | } 150 | return template_params 151 | 152 | def get_template(self, name): 153 | """Loads a Jinja template by name.""" 154 | return BOOKSTORE_FILE_LOADER.load(self.settings['jinja2_env'], name) 155 | 156 | 157 | class BookstoreCloneAPIHandler(APIHandler): 158 | """Handle notebook clone from storage. 159 | 160 | Provides API handling for ``POST`` and clones a notebook 161 | from storage (S3). 162 | 163 | Methods 164 | ------- 165 | initialize(self) 166 | Helper to access bookstore settings. 167 | post(self) 168 | Clone a notebook from the location specified by the payload. 169 | build_content_model(self, obj, path) 170 | Helper that takes a response from S3 and creates a ContentsAPI compatible model. 171 | build_post_response_model(self, model, obj, s3_bucket, s3_object_key) 172 | Helper that takes a Jupyter Contents API compliant model and adds cloning specific information. 173 | 174 | See also 175 | -------- 176 | `Jupyter Notebook reference on Custom Handlers `_ 177 | """ 178 | 179 | def initialize(self): 180 | """Helper to retrieve bookstore setting for the session.""" 181 | self.bookstore_settings = BookstoreSettings(config=self.config) 182 | 183 | self.session = aiobotocore.get_session() 184 | 185 | def _build_s3_request_object(self, s3_bucket, s3_object_key, s3_version_id=None): 186 | """Helper to build object request with the appropriate keys for S3 APIs. 187 | 188 | Parameters 189 | ---------- 190 | s3_bucket: str 191 | Log (usually from the NotebookApp) for logging endpoint changes. 192 | s3_object_key: str 193 | The the path we wish to clone to. 194 | s3_version_id: str, optional 195 | The version id provided. Default is None, which gets the latest version 196 | """ 197 | s3_kwargs = {"Bucket": s3_bucket, "Key": s3_object_key} 198 | if s3_version_id is not None: 199 | s3_kwargs['VersionId'] = s3_version_id 200 | return s3_kwargs 201 | 202 | async def _clone(self, s3_bucket, s3_object_key, s3_version_id=None): 203 | """Main function that handles communicating with S3 to initiate the clone. 204 | 205 | Parameters 206 | ---------- 207 | s3_bucket: str 208 | Log (usually from the NotebookApp) for logging endpoint changes. 209 | s3_object_key: str 210 | The the path we wish to clone to. 211 | s3_version_id: str, optional 212 | The version id provided. Default is None, which gets the latest version 213 | """ 214 | 215 | self.log.info(f"bucket: {s3_bucket}") 216 | self.log.info(f"key: {s3_object_key}") 217 | 218 | async with self.session.create_client( 219 | 's3', 220 | aws_secret_access_key=self.bookstore_settings.s3_secret_access_key, 221 | aws_access_key_id=self.bookstore_settings.s3_access_key_id, 222 | endpoint_url=self.bookstore_settings.s3_endpoint_url, 223 | region_name=self.bookstore_settings.s3_region_name, 224 | ) as client: 225 | self.log.info(f"Processing clone of {s3_object_key}") 226 | try: 227 | s3_kwargs = self._build_s3_request_object(s3_bucket, s3_object_key, s3_version_id) 228 | obj = await client.get_object(**s3_kwargs) 229 | content = (await obj['Body'].read()).decode('utf-8') 230 | except ClientError as e: 231 | status_code = e.response['ResponseMetadata'].get('HTTPStatusCode') 232 | raise web.HTTPError(status_code, e.args[0]) 233 | 234 | self.log.info(f"Obtained contents for {s3_object_key}") 235 | 236 | return obj, content 237 | 238 | @web.authenticated 239 | async def post(self): 240 | """POST /api/bookstore/clone 241 | 242 | Clone a notebook to the path specified in the payload. 243 | 244 | The payload type for the request should be:: 245 | 246 | { 247 | "s3_bucket": string, 248 | "s3_key": string, 249 | "target_path"?: string 250 | "s3_version_id"?: string 251 | } 252 | 253 | The response payload should match the standard Jupyter contents 254 | API POST response. 255 | """ 256 | model = self.get_json_body() 257 | s3_bucket = model.get("s3_bucket", "") 258 | if s3_bucket == '' or s3_bucket == "/": 259 | raise web.HTTPError(400, "Must have a bucket to clone from") 260 | 261 | # s3_paths module has an s3_key function; s3_object_key avoids confusion 262 | s3_object_key = model.get("s3_key", "") 263 | if s3_object_key == '' or s3_object_key == '/': 264 | raise web.HTTPError(400, "Must have a key to clone from") 265 | 266 | target_path = model.get("target_path", "") or os.path.basename( 267 | os.path.relpath(s3_object_key) 268 | ) 269 | s3_version_id = model.get("s3_version_id", None) 270 | 271 | self.log.info(f"About to clone from {s3_object_key}") 272 | obj, content = await self._clone(s3_bucket, s3_object_key, s3_version_id=s3_version_id) 273 | 274 | content_model = self.build_content_model(content, target_path) 275 | 276 | self.log.info(f"Completing clone for {s3_object_key}") 277 | self.contents_manager.save(content_model, content_model['path']) 278 | 279 | resp_model = self.build_post_response_model(content_model, obj, s3_bucket, s3_object_key) 280 | 281 | self.set_status(obj['ResponseMetadata']['HTTPStatusCode']) 282 | self.set_header('Content-Type', 'application/json') 283 | self.finish(resp_model) 284 | 285 | def build_content_model(self, content, target_path): 286 | """Helper that takes a response from S3 and creates a ContentsAPI compatible model. 287 | 288 | If the file at target_path already exists, this increments the file name. 289 | 290 | Parameters 291 | ---------- 292 | content : str 293 | string encoded file content 294 | target_path : str 295 | The the path we wish to clone to, may be incremented if already present. 296 | 297 | Returns 298 | -------- 299 | dict 300 | `Jupyter Contents API compatible model `_ 301 | """ 302 | path = self.contents_manager.increment_filename(target_path, insert='-') 303 | if os.path.splitext(path)[1] in [".ipynb", ".jpynb"]: 304 | model = build_notebook_model(content, path) 305 | else: 306 | model = build_file_model(content, path) 307 | return model 308 | 309 | def build_post_response_model(self, model, obj, s3_bucket, s3_object_key): 310 | """Helper that takes a Jupyter Contents API compliant model and adds cloning specific information. 311 | 312 | Parameters 313 | ---------- 314 | model : dict 315 | Jupyter Contents API model 316 | obj : dict 317 | Log (usually from the NotebookApp) for logging endpoint changes. 318 | s3_bucket : str 319 | The S3 bucket we are cloning from 320 | s3_object_key: str 321 | The S3 key we are cloning 322 | 323 | Returns 324 | -------- 325 | dict 326 | Model with additional info about the S3 cloning 327 | """ 328 | model = deepcopy(model) 329 | model["s3_path"] = s3_display_path(s3_bucket, s3_object_key) 330 | if 'VersionId' in obj: 331 | model["versionID"] = obj['VersionId'] 332 | return model 333 | 334 | 335 | def validate_relpath(relpath, settings, log): 336 | """Validates that a relative path appropriately resolves given bookstore settings. 337 | 338 | Parameters 339 | ---------- 340 | relpath : string 341 | Relative path to a notebook to be cloned. 342 | settings : BookstoreSettings 343 | Bookstore configuration. 344 | log : logging.Logger 345 | Log (usually from the NotebookApp) for logging endpoint changes. 346 | 347 | Returns 348 | -------- 349 | Path 350 | Absolute path to file to be cloned. 351 | """ 352 | if relpath == '': 353 | log.info("Request received with empty relpath.") 354 | raise web.HTTPError(400, "Request malformed, must provide a non-empty relative path.") 355 | 356 | fs_basedir = Path(settings.fs_cloning_basedir) 357 | 358 | fs_clonepath = Path(os.path.realpath(os.path.join(fs_basedir, relpath))) 359 | 360 | if fs_basedir not in fs_clonepath.parents: 361 | log.info(f"Request to clone from a path outside of base directory: {fs_clonepath}.") 362 | raise web.HTTPError(404, f"{fs_clonepath} is outside root cloning directory.") 363 | 364 | return fs_clonepath 365 | 366 | 367 | class BookstoreFSCloneHandler(IPythonHandler): 368 | """Prepares and provides file system clone options page, populating UI with clone option parameters. 369 | 370 | Provides handling for ``GET`` requests when cloning a notebook 371 | from a local file system. Launches a user interface with cloning options. 372 | 373 | Methods 374 | ------- 375 | initialize(self) 376 | Helper to access bookstore settings. 377 | get(self) 378 | Checks for valid storage settings and render a UI for clone options. 379 | construct_template_params(self, relpath) 380 | Helper to populate Jinja template for cloning option page. 381 | get_template(self, name) 382 | Loads a Jinja template and its related settings. 383 | 384 | See also 385 | -------- 386 | `Jupyter Notebook reference on Custom Handlers `_ 387 | """ 388 | 389 | def initialize(self): 390 | """Helper to retrieve bookstore setting for the session.""" 391 | self.bookstore_settings = BookstoreSettings(config=self.config) 392 | 393 | @web.authenticated 394 | async def get(self): 395 | """GET /bookstore/fs-clone?relpath= 396 | 397 | Renders an options page that will allow you to clone a notebook 398 | from a via the Bookstore file-system cloning API. 399 | 400 | relpath is the relative path that you wish to clone from 401 | """ 402 | 403 | relpath = self.get_argument("relpath") 404 | 405 | fs_clonepath = validate_relpath(relpath, self.bookstore_settings, self.log) 406 | 407 | self.log.info(f"Setting up cloning landing page for {fs_clonepath}") 408 | 409 | template_params = self.construct_template_params(relpath, fs_clonepath) 410 | self.set_header('Content-Type', 'text/html') 411 | self.write(self.render_template('clone.html', **template_params)) 412 | 413 | def construct_template_params(self, relpath, fs_clonepath): 414 | """Helper that takes a valid relpath and populates UI template 415 | 416 | Returns 417 | -------- 418 | 419 | dict 420 | Template parameters in a dictionary 421 | """ 422 | # here we closely match the logic in how default_url is defined in 423 | # https://github.com/jupyter/notebook/blob/55e93b9ffef0df9158586e65a34164f7a903e1e0/notebook/notebookapp.py#L1415-L1417 424 | if self.default_url.startswith(self.base_url): 425 | redirect_contents_url = self.default_url 426 | else: 427 | redirect_contents_url = url_path_join(self.base_url, self.default_url) 428 | clone_api_url = url_path_join(self.base_url, "/api/bookstore/fs-clone") 429 | model = {"relpath": relpath} 430 | 431 | template_params = { 432 | "post_model": model, 433 | "clone_api_url": clone_api_url, 434 | "redirect_contents_url": redirect_contents_url, 435 | "source_description": fs_clonepath, 436 | } 437 | return template_params 438 | 439 | def get_template(self, name): 440 | """Loads a Jinja template by name.""" 441 | return BOOKSTORE_FILE_LOADER.load(self.settings['jinja2_env'], name) 442 | 443 | 444 | class BookstoreFSCloneAPIHandler(APIHandler): 445 | """Handle notebook clone from an accessible file system (local or cloud). 446 | 447 | Provides API handling for ``POST`` and clones a notebook 448 | from the specified file system (local or cloud). 449 | 450 | Methods 451 | ------- 452 | initialize(self) 453 | Helper to access bookstore settings. 454 | post(self) 455 | Clone a notebook from the filesystem location specified by the payload. 456 | build_content_model(self, content, path) 457 | Helper for creating a Jupyter ContentsAPI compatible model. 458 | 459 | See also 460 | -------- 461 | `Jupyter Notebook reference on Custom Handlers `_ 462 | """ 463 | 464 | def initialize(self): 465 | """Helper to retrieve bookstore setting for the session.""" 466 | self.bookstore_settings = BookstoreSettings(config=self.config) 467 | 468 | def _get_content(self, path): 469 | """Helper for getting content from a specified filepath. 470 | 471 | Parameters 472 | ---------- 473 | path : str 474 | File system path to a notebook 475 | """ 476 | self.log.info(f"Reading content from {path}") 477 | if os.path.splitext(path)[1] in [".ipynb", ".jpynb"]: 478 | content = self.contents_manager._read_notebook(path) 479 | else: 480 | content = self.contents_manager._read_file(path, format=None) 481 | return content 482 | 483 | @web.authenticated 484 | async def post(self): 485 | """POST /api/bookstore/fs-clone 486 | 487 | Clone a notebook to the path specified in the payload. 488 | 489 | The payload type for the request should be:: 490 | 491 | { 492 | "relpath": string, 493 | "target_path": string #optional 494 | } 495 | 496 | The response payload should match the standard Jupyter contents 497 | API POST response. 498 | """ 499 | model = self.get_json_body() 500 | 501 | relpath = model.get("relpath", "") 502 | fs_clonepath = validate_relpath(relpath, self.bookstore_settings, self.log) 503 | 504 | target_path = model.get("target_path", "") or os.path.basename(os.path.relpath(relpath)) 505 | 506 | nb = self._get_content(str(fs_clonepath)) 507 | content_model = self.build_content_model(json.dumps(nb), target_path) 508 | 509 | self.log.info(f"Completing clone from {fs_clonepath} to {target_path}") 510 | self.contents_manager.save(content_model, content_model['path']) 511 | 512 | self.set_status(200) 513 | self.set_header('Content-Type', 'application/json') 514 | self.finish(content_model) 515 | 516 | def build_content_model(self, content, target_path): 517 | """Helper that takes a content and creates a ContentsAPI compatible model. 518 | 519 | If the file at target_path already exists, this increments the file name. 520 | 521 | Parameters 522 | ---------- 523 | content : dict or string 524 | dict or string encoded file content 525 | target_path : str 526 | The the path we wish to clone to, may be incremented if already present. 527 | 528 | Returns 529 | -------- 530 | dict 531 | `Jupyter Contents API compatible model `_ 532 | """ 533 | path = self.contents_manager.increment_filename(target_path, insert='-') 534 | if os.path.splitext(path)[1] in [".ipynb", ".jpynb"]: 535 | model = build_notebook_model(content, path) 536 | self.contents_manager.validate_notebook_model(model) 537 | else: 538 | model = build_file_model(content, path) 539 | return model 540 | --------------------------------------------------------------------------------