├── nbflow ├── __init__.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── util.py │ └── test_nbflow.py ├── __main__.py ├── _version.py ├── example │ ├── README.md │ ├── SConstruct │ └── analyses │ │ ├── gen_data.ipynb │ │ └── analyze_data.ipynb ├── scons.py └── extractor.py ├── MANIFEST.in ├── requirements.txt ├── scripts └── nbflow ├── tox.ini ├── environment.yml ├── .travis.yml ├── .gitignore ├── LICENSE ├── setup.py └── README.md /nbflow/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nbflow/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt -------------------------------------------------------------------------------- /nbflow/__main__.py: -------------------------------------------------------------------------------- 1 | from .extractor import main 2 | main() 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | traitlets 2 | nbconvert>=4.1 3 | scons>=3.0.0 4 | ipykernel 5 | -------------------------------------------------------------------------------- /scripts/nbflow: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from nbflow.extractor import main 4 | main() 5 | -------------------------------------------------------------------------------- /nbflow/_version.py: -------------------------------------------------------------------------------- 1 | version_info = (0, 1, 0, 'dev') 2 | __version__ = '.'.join(map(str, version_info)) 3 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py35, py36, py37 3 | 4 | [testenv] 5 | commands = py.test -v -x 6 | deps = pytest -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: nbflow 2 | channels: 3 | dependencies: 4 | - pip: 5 | - "git+git://github.com/jhamrick/nbflow.git" -------------------------------------------------------------------------------- /nbflow/example/README.md: -------------------------------------------------------------------------------- 1 | # Example nbflow setup 2 | 3 | To execute the notebooks in this example, run from this directory: 4 | 5 | ``` 6 | scons 7 | ``` 8 | -------------------------------------------------------------------------------- /nbflow/example/SConstruct: -------------------------------------------------------------------------------- 1 | import os 2 | from nbflow.scons import setup 3 | 4 | env = Environment(ENV=os.environ) 5 | setup(env, ["analyses"], ARGUMENTS) 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | python: 4 | - 2.7 5 | - 3.5 6 | - 3.6 7 | - 3.7-dev 8 | install: 9 | - pip install tox-travis 10 | script: 11 | - tox -------------------------------------------------------------------------------- /nbflow/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | import tempfile 4 | import shutil 5 | from textwrap import dedent 6 | 7 | 8 | @pytest.fixture 9 | def temp_cwd(request): 10 | orig_dir = os.getcwd() 11 | path = tempfile.mkdtemp() 12 | os.chdir(path) 13 | 14 | def fin(): 15 | os.chdir(orig_dir) 16 | shutil.rmtree(path) 17 | request.addfinalizer(fin) 18 | 19 | return path 20 | 21 | 22 | @pytest.fixture 23 | def sconstruct(temp_cwd): 24 | with open("SConstruct", "w") as fh: 25 | fh.write(dedent( 26 | """ 27 | import os 28 | from nbflow.scons import setup 29 | 30 | env = Environment(ENV=os.environ) 31 | setup(env, ["."], ARGUMENTS) 32 | """ 33 | )) 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | # IPython Notebook 62 | .ipynb_checkpoints 63 | 64 | # Scons 65 | .sconsign.dblite 66 | 67 | \.pytest_cache/ 68 | 69 | MANIFEST 70 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Jessica B. Hamrick 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of nbflow nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /nbflow/tests/util.py: -------------------------------------------------------------------------------- 1 | import subprocess as sp 2 | import os 3 | 4 | from nbconvert.preprocessors import ClearOutputPreprocessor 5 | from nbformat import read, write 6 | from nbformat.v4 import new_notebook, new_code_cell 7 | from copy import deepcopy 8 | 9 | 10 | def run_command(cmd, retcode=0): 11 | p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.STDOUT) 12 | code = p.wait() 13 | stdout, _ = p.communicate() 14 | stdout = stdout.replace(b"\x1b[?1034h", b"") 15 | if code != retcode: 16 | print(stdout) 17 | raise RuntimeError("command returned unexpected code: {}".format(code)) 18 | 19 | return stdout 20 | 21 | 22 | def clear_notebooks(root): 23 | """Clear the outputs of documentation notebooks.""" 24 | 25 | preprocessor = ClearOutputPreprocessor() 26 | 27 | for dirpath, dirnames, filenames in os.walk(root): 28 | 29 | for filename in sorted(filenames): 30 | if os.path.splitext(filename)[1] == '.ipynb': 31 | # read in the notebook 32 | pth = os.path.join(dirpath, filename) 33 | with open(pth, 'r') as fh: 34 | orig_nb = read(fh, 4) 35 | 36 | # copy the original notebook 37 | new_nb = deepcopy(orig_nb) 38 | 39 | # check outputs of all the cells 40 | new_nb = preprocessor.preprocess(new_nb, {})[0] 41 | 42 | # clear metadata 43 | new_nb.metadata = {} 44 | 45 | # write the notebook back to disk 46 | with open(pth, 'w') as fh: 47 | write(new_nb, fh, 4) 48 | 49 | 50 | def create_notebook(name, cells): 51 | nb = new_notebook() 52 | for cell in cells: 53 | nb.cells.append(new_code_cell(source=cell)) 54 | 55 | with open(name, "w") as fh: 56 | write(nb, fh, 4) 57 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Distributed under the terms of the Modified BSD License. 5 | 6 | import os 7 | from distutils.core import setup 8 | 9 | 10 | # get paths to all the extension files 11 | extension_files = [] 12 | for (dirname, dirnames, filenames) in os.walk("nbflow/example"): 13 | root = os.path.relpath(dirname, "nbflow") 14 | if '.ipynb_checkpoints' in dirname: 15 | continue 16 | for filename in filenames: 17 | if filename.endswith(".pyc") or filename == '.sconsign.dblite': 18 | continue 19 | extension_files.append(os.path.join(root, filename)) 20 | 21 | 22 | name = 'nbflow' 23 | here = os.path.abspath(os.path.dirname(__file__)) 24 | version_ns = {} 25 | with open(os.path.join(here, name, '_version.py')) as f: 26 | exec(f.read(), {}, version_ns) 27 | 28 | 29 | setup_args = dict( 30 | name=name, 31 | version=version_ns['__version__'], 32 | description='A tool that supports one-button reproducible workflows with the Jupyter Notebook and Scons', 33 | author='Jessica B. Hamrick', 34 | author_email='jhamrick@berkeley.edu', 35 | license='BSD', 36 | url='https://github.com/jhamrick/nbflow', 37 | keywords=['Notebooks'], 38 | classifiers=[ 39 | 'License :: OSI Approved :: BSD License', 40 | 'Programming Language :: Python', 41 | 'Programming Language :: Python :: 2', 42 | 'Programming Language :: Python :: 2.7', 43 | 'Programming Language :: Python :: 3', 44 | 'Programming Language :: Python :: 3.6', 45 | ], 46 | packages=['nbflow'], 47 | scripts=['scripts/nbflow'], 48 | package_data={ 49 | 'nbflow': extension_files 50 | } 51 | ) 52 | 53 | 54 | setup_args['install_requires'] = install_requires = [] 55 | with open('requirements.txt') as f: 56 | for line in f.readlines(): 57 | req = line.strip() 58 | if not req or req.startswith(('-e', '#')): 59 | continue 60 | install_requires.append(req) 61 | 62 | setup(**setup_args) 63 | -------------------------------------------------------------------------------- /nbflow/example/analyses/gen_data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "__depends__ = []\n", 12 | "__dest__ = \"../results/data.json\"" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 2, 18 | "metadata": { 19 | "collapsed": true 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "import random\n", 24 | "import json\n", 25 | "import os" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 3, 31 | "metadata": {}, 32 | "outputs": [ 33 | { 34 | "data": { 35 | "text/plain": [ 36 | "[0.7947434550704049,\n", 37 | " 0.7689497615073635,\n", 38 | " 0.4055562370238408,\n", 39 | " 0.1437725456662664,\n", 40 | " 0.06290778106817208,\n", 41 | " 0.9690115597726553,\n", 42 | " 0.512838996781445,\n", 43 | " 0.09813919031940244,\n", 44 | " 0.31852186939224447,\n", 45 | " 0.41326319707820003]" 46 | ] 47 | }, 48 | "execution_count": 3, 49 | "metadata": {}, 50 | "output_type": "execute_result" 51 | } 52 | ], 53 | "source": [ 54 | "random.seed(\"92831\")\n", 55 | "data = [random.random() for x in range(100)]\n", 56 | "data[:10]" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 4, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "if not os.path.exists(\"../results\"):\n", 66 | " os.mkdir(\"../results\")\n", 67 | "\n", 68 | "with open(__dest__, \"w\") as fh:\n", 69 | " json.dump(data, fh)" 70 | ] 71 | } 72 | ], 73 | "metadata": { 74 | "kernelspec": { 75 | "display_name": "Python 3", 76 | "language": "python", 77 | "name": "python3" 78 | }, 79 | "language_info": { 80 | "codemirror_mode": { 81 | "name": "ipython", 82 | "version": 2 83 | }, 84 | "file_extension": ".py", 85 | "mimetype": "text/x-python", 86 | "name": "python", 87 | "nbconvert_exporter": "python", 88 | "pygments_lexer": "ipython2", 89 | "version": "2.7.14" 90 | } 91 | }, 92 | "nbformat": 4, 93 | "nbformat_minor": 1 94 | } 95 | -------------------------------------------------------------------------------- /nbflow/scons.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sys 3 | import subprocess as sp 4 | from functools import partial 5 | 6 | import nbconvert 7 | 8 | 9 | def build_cmd(notebook, timeout): 10 | cmd = [ 11 | "jupyter", "nbconvert", 12 | "--log-level=ERROR", 13 | "--ExecutePreprocessor.timeout=" + timeout, 14 | "--execute", 15 | "--inplace", 16 | "--to", "notebook" 17 | ] 18 | 19 | if nbconvert.__version__ < '4.2.0': 20 | cmd.extend(['--output', notebook]) 21 | cmd.append(notebook) 22 | 23 | return cmd 24 | 25 | def build_notebook(target, source, env, timeout="120"): 26 | notebook = str(source[0]) 27 | code = sp.call(build_cmd(notebook, timeout)) 28 | if code != 0: 29 | raise RuntimeError("Error executing notebook") 30 | 31 | # we need to touch each of the targets so that they have a later 32 | # modification time than the source -- otherwise scons will think that the 33 | # targets always are out of date and need to be rebuilt 34 | for t in target: 35 | sp.call(["touch", str(t)]) 36 | 37 | return None 38 | 39 | 40 | def print_cmd_line(s, targets, sources, env): 41 | """s is the original command line string 42 | targets is the list of target nodes 43 | sources is the list of source nodes 44 | env is the environment""" 45 | if len(targets) == 0: 46 | sys.stdout.write("%s --> None\n"% str(sources[0])) 47 | else: 48 | for target in targets: 49 | if str(target).startswith('.phony'): 50 | target = 'None' 51 | sys.stdout.write("%s --> %s\n"% (str(sources[0]), str(target))) 52 | 53 | 54 | def setup(env, directories, args): 55 | env['PRINT_CMD_LINE_FUNC'] = print_cmd_line 56 | env.Decider('timestamp-newer') 57 | DEPENDENCIES = json.loads(sp.check_output([sys.executable, "-m", "nbflow"] + directories).decode('UTF-8')) 58 | timeout = args.get('timeout', None) 59 | if timeout is not None: 60 | build_notebook_timeout = partial(build_notebook, timeout=str(timeout)) 61 | else: 62 | build_notebook_timeout = build_notebook 63 | for script in DEPENDENCIES: 64 | deps = DEPENDENCIES[script] 65 | if len(deps['targets']) == 0: 66 | targets = ['.phony_{}'.format(script)] 67 | else: 68 | targets = deps['targets'] 69 | env.Command(targets, [script] + deps['sources'], build_notebook_timeout) 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NBFlow 2 | 3 | [![Build Status](https://travis-ci.org/jhamrick/nbflow.svg?branch=master)](https://travis-ci.org/jhamrick/nbflow) [![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/jhamrick/nbflow/master) 4 | 5 | A tool that supports one-button reproducible workflows with the Jupyter Notebook and Scons. **Note: this currently only supports Python kernels.** 6 | 7 | > **UPDATE: Scons >= 3.0.0 now supports Python 3 so Python 2 isn't needed anymore!** Now Nbflow and Scons are both Python 2 and 3 compatible so you can choose whichever you want. 8 | 9 | > The actual version of Scons (3.0.1) currently supports Python >= 3.5, which is the default in Ubuntu 16.04 10 | 11 | ## Installation 12 | 13 | To install, run: 14 | 15 | Linux: 16 | ``` 17 | pip3 install git+git://github.com/jhamrick/nbflow.git 18 | ``` 19 | 20 | Windows: 21 | ``` 22 | pip install git+git://github.com/jhamrick/nbflow.git 23 | ``` 24 | 25 | ## Usage 26 | 27 | For a complete example of how to use nbflow, check out [the example](nbflow/example) 28 | in this repository. 29 | 30 | You can now you Binder to check the example online: 31 | 32 | 1. Entre in Binder [here](https://mybinder.org/v2/gh/jhamrick/nbflow/master) or through the badge above 33 | 2. Open a terminal 34 | 3. Run `cd nbflow/example` 35 | 4. Run `scons` 36 | 5. Check the results in the `results` directory 37 | 38 | Optionally you can modify the notebook in this online environment and check how the results change. 39 | 40 | ### Analysis notebooks 41 | 42 | For each notebook that you want executed, you MUST include two special variables 43 | in the first code cell: 44 | 45 | * `__depends__` -- a list of relative paths to files that the notebook depends 46 | on 47 | * `__dest__` -- either a relative path, or list of relative paths, to files that 48 | the notebook produces 49 | 50 | For example, the first cell in [one of the example notebooks](nbflow/example/analyses/analyze_data.ipynb) 51 | is: 52 | 53 | ```python 54 | __depends__ = ["../results/data.json"] 55 | __dest__ = "../results/stats.json" 56 | ``` 57 | 58 | ### SConstruct file 59 | 60 | You need a `SConstruct` file in the root of you analysis directory. In this 61 | `SConstruct` file you will need to import nbflow and use it to setup your scons 62 | environment, e.g.: 63 | 64 | ```python 65 | import os 66 | from nbflow.scons import setup 67 | 68 | env = Environment(ENV=os.environ) 69 | setup(env, ["analyses"]) 70 | ``` 71 | 72 | The second argument of the `setup` command takes a list of folder names that 73 | contain analysis notebooks. 74 | 75 | ### Running nbflow 76 | 77 | Once you have setup your analysis notebooks and your `SConstruct` file, you can 78 | run your notebooks by just running the `scons` command from the root of your 79 | analysis directory. 80 | -------------------------------------------------------------------------------- /nbflow/extractor.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import glob 4 | import json 5 | import sys 6 | 7 | from nbformat.v4 import reads 8 | from traitlets.config import Application, Unicode 9 | 10 | from ._version import __version__ 11 | 12 | 13 | class DependencyExtractor(Application): 14 | 15 | name = 'nbflow' 16 | description = 'Extract the hierarchy of dependencies from notebooks in the specified folder.' 17 | version = __version__ 18 | 19 | def extract_parameters(self, nb): 20 | # find the first code cell 21 | defs_cell = None 22 | for cell in nb.cells: 23 | if cell.cell_type == 'code': 24 | defs_cell = cell 25 | break 26 | 27 | if defs_cell is None: 28 | return {} 29 | 30 | defs_code = defs_cell.source 31 | globals_dict = {} 32 | locals_dict = {} 33 | exec(defs_code, globals_dict, locals_dict) 34 | return locals_dict 35 | 36 | def resolve_path(self, source, path): 37 | dirname = os.path.dirname(source) 38 | return os.path.abspath(os.path.join(dirname, path)) 39 | 40 | def get_dependencies(self, dirnames): 41 | dependencies = {} 42 | 43 | for dirname in dirnames: 44 | files = glob.glob("{}/*.ipynb".format(dirname)) 45 | 46 | for filename in files: 47 | modname = os.path.splitext(os.path.basename(filename))[0] 48 | with open(filename, "r") as fh: 49 | nb = reads(fh.read()) 50 | 51 | params = self.extract_parameters(nb) 52 | if '__depends__' not in params: 53 | continue 54 | if '__dest__' not in params: 55 | raise ValueError("__dest__ is not defined in {}".format(filename)) 56 | 57 | # get sources that are specified in the file 58 | sources = [self.resolve_path(filename, x) for x in params['__depends__']] 59 | 60 | targets = params['__dest__'] 61 | if not isinstance(targets, list): 62 | if targets is None: 63 | targets = [] 64 | else: 65 | targets = [targets] 66 | targets = [self.resolve_path(filename, x) for x in targets] 67 | 68 | dependencies[os.path.join(dirname, '{}.ipynb'.format(modname))] = { 69 | 'targets': targets, 70 | 'sources': sources 71 | } 72 | 73 | return json.dumps(dependencies, indent=2) 74 | 75 | def start(self): 76 | if len(self.extra_args) == 0: 77 | self.log.error("No directory names specified.") 78 | sys.exit(1) 79 | 80 | print(self.get_dependencies(self.extra_args)) 81 | 82 | 83 | def main(): 84 | DependencyExtractor.launch_instance() 85 | -------------------------------------------------------------------------------- /nbflow/example/analyses/analyze_data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "__depends__ = [\"../results/data.json\"]\n", 12 | "__dest__ = \"../results/stats.json\"" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 2, 18 | "metadata": { 19 | "collapsed": true 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "import json" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 3, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "import time\n", 33 | "time.sleep(5) # Simulate long running task" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 3, 39 | "metadata": { 40 | "collapsed": true 41 | }, 42 | "outputs": [], 43 | "source": [ 44 | "with open(\"../results/data.json\", \"r\") as fh:\n", 45 | " data = json.load(fh)" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 4, 51 | "metadata": {}, 52 | "outputs": [ 53 | { 54 | "data": { 55 | "text/plain": [ 56 | "[0.7947434550704049,\n", 57 | " 0.7689497615073635,\n", 58 | " 0.4055562370238408,\n", 59 | " 0.1437725456662664,\n", 60 | " 0.06290778106817208,\n", 61 | " 0.9690115597726553,\n", 62 | " 0.512838996781445,\n", 63 | " 0.09813919031940244,\n", 64 | " 0.31852186939224447,\n", 65 | " 0.41326319707820003]" 66 | ] 67 | }, 68 | "execution_count": 4, 69 | "metadata": {}, 70 | "output_type": "execute_result" 71 | } 72 | ], 73 | "source": [ 74 | "data[:10]" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 5, 80 | "metadata": {}, 81 | "outputs": [ 82 | { 83 | "data": { 84 | "text/plain": [ 85 | "{'mean': 0.49688692836011145, 'var': 0.08984597204318795}" 86 | ] 87 | }, 88 | "execution_count": 5, 89 | "metadata": {}, 90 | "output_type": "execute_result" 91 | } 92 | ], 93 | "source": [ 94 | "mean = sum(data) / len(data)\n", 95 | "var = sum(((x - mean) ** 2) / (len(data) - 1) for x in data)\n", 96 | "results = dict(mean=mean, var=var)\n", 97 | "results" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 6, 103 | "metadata": { 104 | "collapsed": true 105 | }, 106 | "outputs": [], 107 | "source": [ 108 | "with open(__dest__, \"w\") as fh:\n", 109 | " json.dump(results, fh)" 110 | ] 111 | } 112 | ], 113 | "metadata": { 114 | "kernelspec": { 115 | "display_name": "Python 3", 116 | "language": "python", 117 | "name": "python3" 118 | }, 119 | "language_info": { 120 | "codemirror_mode": { 121 | "name": "ipython", 122 | "version": 3 123 | }, 124 | "file_extension": ".py", 125 | "mimetype": "text/x-python", 126 | "name": "python", 127 | "nbconvert_exporter": "python", 128 | "pygments_lexer": "ipython3", 129 | "version": "3.6.4" 130 | } 131 | }, 132 | "nbformat": 4, 133 | "nbformat_minor": 1 134 | } 135 | -------------------------------------------------------------------------------- /nbflow/tests/test_nbflow.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import shutil 4 | 5 | from textwrap import dedent 6 | from .util import run_command, clear_notebooks, create_notebook 7 | import json 8 | 9 | 10 | def test_nbflow_no_args(temp_cwd): 11 | run_command([sys.executable, "-m", "nbflow"], retcode=1) 12 | 13 | 14 | def test_notebook_long_excecution(temp_cwd, sconstruct): 15 | # copy example files 16 | root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "example")) 17 | shutil.copytree(os.path.join(root, "analyses"), "analyses") 18 | shutil.copy(os.path.join(root, "SConstruct"), "SConstruct") 19 | clear_notebooks("analyses") 20 | 21 | run_command(["scons","timeout=1"], retcode=2) 22 | 23 | def test_example(temp_cwd): 24 | # copy example files 25 | root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "example")) 26 | shutil.copytree(os.path.join(root, "analyses"), "analyses") 27 | shutil.copy(os.path.join(root, "SConstruct"), "SConstruct") 28 | clear_notebooks("analyses") 29 | 30 | # check the explicit output of the nbflow command 31 | output = run_command([sys.executable, "-m", "nbflow", "analyses"]) 32 | expected = dedent( 33 | """ 34 | { 35 | "analyses/analyze_data.ipynb": { 36 | "sources": [ 37 | "%(path)s/results/data.json" 38 | ], 39 | "targets": [ 40 | "%(path)s/results/stats.json" 41 | ] 42 | }, 43 | "analyses/gen_data.ipynb": { 44 | "sources": [], 45 | "targets": [ 46 | "%(path)s/results/data.json" 47 | ] 48 | } 49 | } 50 | """ % dict(path=os.path.abspath(os.path.realpath(temp_cwd))) 51 | ).lstrip() 52 | assert json.loads(output.decode('UTF-8')) == json.loads(expected) 53 | 54 | # try running scons 55 | output = run_command(["scons"]) 56 | expected = dedent( 57 | """ 58 | scons: Reading SConscript files ... 59 | scons: done reading SConscript files. 60 | scons: Building targets ... 61 | analyses/gen_data.ipynb --> results/data.json 62 | analyses/analyze_data.ipynb --> results/stats.json 63 | scons: done building targets. 64 | """ 65 | ).lstrip() 66 | 67 | assert output == expected.encode('UTF-8') 68 | 69 | # run scons again, make sure it doesn't want to do anything 70 | output = run_command(["scons", "-n"]) 71 | expected = dedent( 72 | """ 73 | scons: Reading SConscript files ... 74 | scons: done reading SConscript files. 75 | scons: Building targets ... 76 | scons: `.' is up to date. 77 | scons: done building targets. 78 | """ 79 | ).lstrip() 80 | 81 | assert output == expected.encode('UTF-8') 82 | 83 | 84 | def test_empty_notebook(temp_cwd, sconstruct): 85 | create_notebook("test.ipynb", [ 86 | "__depends__ = []\n__dest__ = None" 87 | ]) 88 | output = run_command(["scons"]) 89 | expected = dedent( 90 | """ 91 | scons: Reading SConscript files ... 92 | scons: done reading SConscript files. 93 | scons: Building targets ... 94 | test.ipynb --> None 95 | scons: done building targets. 96 | """ 97 | ).lstrip() 98 | 99 | assert output == expected.encode('UTF-8') 100 | 101 | 102 | def test_notebook_with_errors(temp_cwd, sconstruct): 103 | create_notebook("test.ipynb", [ 104 | "__depends__ = []\n__dest__ = None", 105 | "assert False" 106 | ]) 107 | run_command(["scons"], retcode=2) 108 | 109 | 110 | def test_notebook_without_depends(temp_cwd, sconstruct): 111 | create_notebook("test.ipynb", [ 112 | "__dest__ = None" 113 | ]) 114 | output = run_command(["scons"]) 115 | expected = dedent( 116 | """ 117 | scons: Reading SConscript files ... 118 | scons: done reading SConscript files. 119 | scons: Building targets ... 120 | scons: `.' is up to date. 121 | scons: done building targets. 122 | """ 123 | ).lstrip() 124 | 125 | assert output == expected.encode('UTF-8') 126 | 127 | 128 | def test_notebook_without_dest(temp_cwd, sconstruct): 129 | create_notebook("test.ipynb", [ 130 | "__depends__ = []" 131 | ]) 132 | output = run_command(["scons"], retcode=2) 133 | 134 | 135 | def test_multiple_notebooks_with_no_dests(temp_cwd, sconstruct): 136 | create_notebook("test1.ipynb", [ 137 | "__depends__ = []\n__dest__ = []" 138 | ]) 139 | create_notebook("test2.ipynb", [ 140 | "__depends__ = []\n__dest__ = []" 141 | ]) 142 | output = run_command(["scons"]) 143 | expected = dedent( 144 | """ 145 | scons: Reading SConscript files ... 146 | scons: done reading SConscript files. 147 | scons: Building targets ... 148 | test1.ipynb --> None 149 | test2.ipynb --> None 150 | scons: done building targets. 151 | """ 152 | ).lstrip() 153 | 154 | assert output == expected.encode('UTF-8') 155 | 156 | --------------------------------------------------------------------------------