├── setup.cfg ├── docs └── _images │ └── pdn_icon32.png ├── images └── pixiedust_node_schematic.png ├── pixiedust_node ├── package.json ├── __init__.py ├── util.js ├── pixiedustNodeRepl.js └── node.py ├── setup.py ├── .gitignore ├── README.md └── LICENSE.txt /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | 4 | [bdist_wheel] 5 | universal=1 6 | 7 | -------------------------------------------------------------------------------- /docs/_images/pdn_icon32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pixiedust/pixiedust_node/HEAD/docs/_images/pdn_icon32.png -------------------------------------------------------------------------------- /images/pixiedust_node_schematic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pixiedust/pixiedust_node/HEAD/images/pixiedust_node_schematic.png -------------------------------------------------------------------------------- /pixiedust_node/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pixiedust_node", 3 | "version": "0.2.5", 4 | "description": "Run Node.js cells in Jupyter notebooks with Pixiedust", 5 | "main": "pixiedustNodeRepl.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [ 10 | "pixiedust", 11 | "notebook" 12 | ], 13 | "author": "Glynn Bird ", 14 | "license": "Apache-2.0", 15 | "dependencies": { 16 | }, 17 | "repository": { 18 | "type": "git", 19 | "url": "https://github.com/pixiedust/pixiedust_node" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup(name='pixiedust_node', 4 | version='0.2.5', 5 | description='Pixiedust extension for Node.js', 6 | url='https://github.com/pixiedust/pixiedust_node', 7 | install_requires=['pixiedust', 'pandas', 'ipython'], 8 | package_data={ 9 | '': ['*.js','*.json'] 10 | }, 11 | author='David Taieb, Glynn Bird', 12 | author_email='david_taieb@us.ibm.com, glynn.bird@gmail.com', 13 | license='Apache 2.0', 14 | packages=find_packages(), 15 | include_package_data=False, 16 | zip_safe=False) 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # VS Code 7 | .vscode 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *,cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # IPython Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # dotenv 82 | .env 83 | 84 | # virtualenv 85 | venv/ 86 | ENV/ 87 | 88 | # Spyder project settings 89 | .spyderproject 90 | 91 | # Rope project settings 92 | .ropeproject 93 | 94 | .DS_Store 95 | 96 | start.sh 97 | 98 | pixiedust_node/node_modules 99 | -------------------------------------------------------------------------------- /pixiedust_node/__init__.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------- 2 | # Copyright IBM Corp. 2017 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the 'License'); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed unde 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ------------------------------------------------------------------------------- 16 | 17 | from IPython.core.magic import (Magics, magics_class, cell_magic) 18 | from IPython.display import display, HTML 19 | from IPython.core.error import TryNext 20 | import warnings 21 | from .node import Node, Npm 22 | import os 23 | from pixiedust.utils.shellAccess import ShellAccess 24 | 25 | # pixiedust magics to interpret cells starting with %%node 26 | @magics_class 27 | class PixiedustNodeMagics(Magics): 28 | 29 | def __init__(self, shell, node): 30 | super(PixiedustNodeMagics,self).__init__(shell=shell) 31 | display(HTML( 32 | """ 33 |
34 | 35 | 36 | 37 | Pixiedust Node.js 38 |
39 | """ 40 | )) 41 | self.n = node 42 | ShellAccess.npm = Npm() 43 | ShellAccess.node = self.n 44 | 45 | @cell_magic 46 | def node(self, line, cell): 47 | # write the cell contents to the Node.js process 48 | self.n.write(cell) 49 | 50 | # call once when the Kernel shuts down 51 | def shutdown_hook(ipython): 52 | node.terminate() 53 | raise TryNext 54 | 55 | try: 56 | with warnings.catch_warnings(): 57 | warnings.simplefilter("ignore") 58 | ip = get_ipython() 59 | 60 | # start up a Node.js sub-process running a REPL 61 | path = os.path.join(__path__[0], 'pixiedustNodeRepl.js') 62 | node = Node(path) 63 | 64 | # pass the node process to the Node magics 65 | magics = PixiedustNodeMagics(ip, node) 66 | ip.register_magics(magics) 67 | 68 | # register for shutdown hook 69 | ip.set_hook('shutdown_hook', shutdown_hook) 70 | 71 | except NameError: 72 | # IPython not available we must be in a spark executor\ 73 | pass 74 | -------------------------------------------------------------------------------- /pixiedust_node/util.js: -------------------------------------------------------------------------------- 1 | var Util = { 2 | /** 3 | * The iteration cache which will store iterated nodes 4 | */ 5 | _iterCache: [], 6 | 7 | _isArray: function (item) { 8 | return Object.prototype.toString.call(item) === '[object Array]' 9 | }, 10 | 11 | _isObject: function (item) { 12 | return Object.prototype.toString.call(item) === '[object Object]' 13 | }, 14 | 15 | _isUndefined: function (item) { 16 | return typeof item === 'undefined'; 17 | }, 18 | 19 | /** 20 | * Stringify the given item. 21 | * If the item has circular references, the circular references 22 | * will be marked as [CIRCULAR REFERENCE] 23 | */ 24 | _stringify: function (item) { 25 | var cache = []; 26 | 27 | return JSON.stringify(item, function (key, value) { 28 | if (typeof value === 'object' && value !== null) { 29 | if (cache.indexOf(value) !== -1) { 30 | // Found circular reference 31 | return '[CIRCULAR REFERENCE]'; 32 | } 33 | 34 | cache.push(value); 35 | return value; 36 | } 37 | 38 | return value; 39 | }); 40 | }, 41 | 42 | _deepEqualLog: function (title, path, actual, expected) { 43 | title = title || ''; 44 | path = path || []; 45 | return title + ' ' + path.join(" -> ") + ' | actual: ' + Util._stringify(actual) + ', expected: ' + Util._stringify(expected); 46 | }, 47 | 48 | _doDeepEqual: function (actual, expected, notEqualCallback, path) { 49 | var iter; 50 | 51 | notEqualCallback = notEqualCallback || function () {}; 52 | path = path || []; 53 | 54 | if (Util._iterCache.indexOf(actual) !== -1) { 55 | // We have iterated this node before and it passed deep equal check 56 | return true; 57 | } 58 | 59 | // Primitive type 60 | if (actual === expected) { 61 | return true; 62 | } 63 | 64 | // NaN 65 | if (Number.isNaN(actual) && Number.isNaN(expected)) { 66 | return true; 67 | } else if (Number.isNaN(actual) || Number.isNaN(expected)) { 68 | notEqualCallback(Util._deepEqualLog('[Value different]', path, actual, expected)); 69 | return false; 70 | } 71 | 72 | // Array 73 | if (Util._isArray(actual) && Util._isArray(expected)) { 74 | iter = actual.length; 75 | if (iter !== expected.length) { 76 | notEqualCallback(Util._deepEqualLog('[Array with different length]', path, actual, expected)); 77 | return false; 78 | } 79 | 80 | // Mark the actual and expected array has been iterated 81 | Util._iterCache.push(actual); 82 | while (iter--) { 83 | if (!Util._doDeepEqual(actual[iter], expected[iter], notEqualCallback, path.concat(iter))) { 84 | return false; 85 | } 86 | } 87 | 88 | return true; 89 | } else if (Util._isArray(actual) || Util._isArray(expected)) { 90 | notEqualCallback(Util._deepEqualLog('[Different type]', path, actual, expected)); 91 | return false; 92 | } 93 | 94 | // Object 95 | if (Util._isObject(actual) && Util._isObject(expected)) { 96 | if (Object.keys(actual).length !== Object.keys(expected).length) { 97 | notEqualCallback(Util._deepEqualLog('[Object with different keys]', path, actual, expected)); 98 | return false; 99 | } 100 | 101 | // Mark the actual and expected array has been iterated 102 | Util._iterCache.push(actual); 103 | for (iter in actual) { 104 | if (actual.hasOwnProperty(iter)) { 105 | if (!Util._doDeepEqual(actual[iter], expected[iter], notEqualCallback, path.concat(iter))) { 106 | return false; 107 | } 108 | } 109 | } 110 | 111 | return true; 112 | } else if (Util._isObject(actual) || Util._isObject(expected)) { 113 | notEqualCallback(Util._deepEqualLog('[Different type]', path, actual, expected)); 114 | return false; 115 | } 116 | 117 | // Default to false 118 | notEqualCallback(Util._deepEqualLog('[Value different]', path, actual, expected)); 119 | return false; 120 | }, 121 | 122 | /** 123 | * Check the deep equal for primitive type values, array and objects. 124 | * The native assert.deepEqual doesn't work well for NaN case as well 125 | * as +0/-0 case. See https://github.com/substack/node-deep-equal for 126 | * more details 127 | */ 128 | deepEqual: function (actual, expected, notEqualCallback) { 129 | notEqualCallback = notEqualCallback || function () {}; 130 | Util._iterCache = []; 131 | return Util._doDeepEqual(actual, expected, notEqualCallback); 132 | }, 133 | 134 | /** 135 | * Compare and return both the deep equal results, as well as the not equal message 136 | */ 137 | deepEqualWithMessage: function (actual, expected) { 138 | var notEqualMessages = [], 139 | isDeepEqual = Util.deepEqual(actual, expected, function () { 140 | notEqualMessages.push(Array.prototype.slice.call(arguments).join('')); 141 | }); 142 | 143 | return { 144 | isDeepEqual: isDeepEqual, 145 | message: notEqualMessages.join('\n') 146 | }; 147 | } 148 | }; 149 | 150 | module.exports = { 151 | deepEqual: Util.deepEqual, 152 | deepEqualWithMessage: Util.deepEqualWithMessage 153 | }; -------------------------------------------------------------------------------- /pixiedust_node/pixiedustNodeRepl.js: -------------------------------------------------------------------------------- 1 | const repl = require('repl'); 2 | const pkg = require('./package.json'); 3 | const crypto = require('crypto'); 4 | const util = require('./util.js'); 5 | 6 | const startRepl = function(instream, outstream) { 7 | 8 | // check for Node.js global variables and move those values to Python 9 | const globalVariableChecker = function() { 10 | 11 | // get list of global variables 12 | var varlist = Object.getOwnPropertyNames(r.context); 13 | 14 | // trim the list 15 | const cutoff = varlist.indexOf('help') + 1; 16 | varlist.splice(0, cutoff); 17 | 18 | // if there aren't any, we're done 19 | if (varlist.length === 0) return; 20 | 21 | // for each global 22 | for(var i in varlist) { 23 | 24 | // turn it to JSON 25 | const v = varlist[i]; 26 | const j = JSON.stringify(r.context[v]); 27 | 28 | // if it's a string 29 | if (typeof j === 'string' ) { 30 | 31 | // calculate the md5(json) 32 | const h = hash(j); 33 | 34 | // it it's different to what we had last time 35 | if (lastGlobal[v] !== h) { 36 | 37 | // check to see if this is a simple data structure i.e. 38 | // only migrate variables which equal to the JSON.parse'd version of their JSON.stringified selves 39 | // i.e don't migrate objects that contain functions 40 | if (util.deepEqual(JSON.parse(j), r.context[v])) { 41 | 42 | // if we reached here, then we're going to move a variable from Node.js --> Python 43 | 44 | // calculate data type 45 | const datatype = isArray(r.context[v]) && typeof r.context[v][0] === 'object' ? 'array' : typeof r.context[v]; 46 | 47 | // make a special JSON object 48 | const obj = { _pixiedust: true, type: 'variable', key: v, datatype: datatype, value: r.context[v] }; 49 | 50 | // write it to stdout for the Python parser to find 51 | outstream.write('\n' + JSON.stringify(obj) + '\n') 52 | 53 | // store it in our lastGLobal dictionary - so that we only update it when the value changes 54 | lastGlobal[v] = h; 55 | } 56 | } 57 | } 58 | } 59 | }; 60 | 61 | // sync Node.js to Python every 1 second 62 | interval = setInterval(globalVariableChecker, 1000); 63 | interval.unref(); 64 | 65 | // custom writer function that outputs nothing 66 | const writer = function(output) { 67 | globalVariableChecker(); 68 | // don't output anything 69 | return ''; 70 | }; 71 | 72 | const options = { 73 | input: instream, 74 | output: outstream, 75 | prompt: '', 76 | writer: writer 77 | }; 78 | const r = repl.start(options); 79 | var lastGlobal = {}; 80 | var interval = null; 81 | 82 | // generate hash from data 83 | const hash = function(data) { 84 | return crypto.createHash('md5').update(data).digest("hex"); 85 | } 86 | 87 | const isArray = Array.isArray || function(obj) { 88 | return obj && toString.call(obj) === '[object Array]'; 89 | }; 90 | 91 | // custom print function for Notebook interface 92 | const print = function(data) { 93 | // bundle the data into an object 94 | globalVariableChecker(); 95 | const obj = { _pixiedust: true, type: 'print', data: data }; 96 | outstream.write(JSON.stringify(obj) + '\n'); 97 | }; 98 | 99 | // custom display function for Notebook interface 100 | const display = function(data) { 101 | // bundle the data into an object 102 | globalVariableChecker(); 103 | const obj = { _pixiedust: true, type: 'display', data: data }; 104 | outstream.write(JSON.stringify(obj) + '\n'); 105 | 106 | }; 107 | 108 | // custom display function for Notebook interface 109 | const store = function(data, variable) { 110 | globalVariableChecker(); 111 | if (!data && !variable) return; 112 | // bundle the data into an object 113 | const obj = { _pixiedust: true, type: 'store', data: data, variable: variable }; 114 | outstream.write(JSON.stringify(obj) + '\n'); 115 | 116 | }; 117 | 118 | // display html in Notebook cell 119 | const html = function(data) { 120 | // bundle the data into an object 121 | const obj = { _pixiedust: true, type: 'html', data: data}; 122 | outstream.write(JSON.stringify(obj) + '\n'); 123 | globalVariableChecker(); 124 | }; 125 | 126 | // display image in Notebook cell 127 | const image = function(data) { 128 | // bundle the data into an object 129 | const obj = { _pixiedust: true, type: 'image', data: data}; 130 | outstream.write(JSON.stringify(obj) + '\n'); 131 | globalVariableChecker(); 132 | }; 133 | 134 | const help = function() { 135 | console.log(pkg.name, pkg.version); 136 | console.log(pkg.repository.url); 137 | console.log(); 138 | console.log("JavaScript functions:"); 139 | console.log("* print(x) - print out x"); 140 | console.log("* display(x) - turn x into Pandas dataframe and display with Pixiedust"); 141 | console.log("* html(x) - display HTML x in Notebook cell"); 142 | console.log("* image(x) - display image URL x in a Notebook cell"); 143 | console.log("* help() - display help"); 144 | console.log(); 145 | console.log("Python helpers:"); 146 | console.log("* npm.install(x) - install npm package x"); 147 | console.log("* npm.uninstall(x) - remove npm package x"); 148 | console.log("* npm.list() - list installed npm packages"); 149 | console.log("* node.cancel() - cancel Node.js execution"); 150 | console.log("* node.clear() - clear and reset the Node.js engine"); 151 | console.log("* node.help() - view help") 152 | }; 153 | 154 | // add silverlining library and print/display 155 | var resetContext = function() { 156 | r.context.print = print; 157 | r.context.display = display; 158 | r.context.store = store; 159 | r.context.html = html; 160 | r.context.image = image; 161 | r.context.help = help; 162 | lastGlobal = {}; 163 | }; 164 | 165 | // add print/disply/store back in on reset 166 | r.on('reset', resetContext); 167 | 168 | // reset the context 169 | resetContext(); 170 | 171 | return r; 172 | }; 173 | 174 | startRepl(process.stdin, process.stdout); 175 | console.log(pkg.name, pkg.version, "started. Cells starting '%%node' may contain Node.js code."); 176 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pixiedust_node 2 | 3 | PixieDust extension that enable a Jupyter Notebook user to invoke Node.js commands. 4 | 5 | ![schematic](images/pixiedust_node_schematic.png) 6 | 7 | 8 | ## How it works 9 | 10 | The `pixiedust_node` Python module has access to Pixiedust's *display* API to render charts and maps. When `pixiedust_node` is imported into a notebook, a Node.js sub-process is setup and the notebook is configured so that cells beginning with '%%node' may contain JavaScript code: that code is piped to the Node.js sub-process automatically. The output of the Node.js process is parsed by `pixiedust_node` to handle the use of functions display/print/store/html/image. The `pixiedust_node` module also allows npm installs to be initiated from within the notebook. This achieved with further npm sub-processes whose output appears in the notebook. 11 | 12 | ## Prerequisites 13 | 14 | To use `pixiedust_node` you need to be running a Jupyter notebooks with the [Pixedust](https://github.com/pixiedust/pixiedust) extension installed. Notebooks can be run locally by [installing Pixiedust and its prerequisites](https://pixiedust.github.io/pixiedust/install.html). 15 | 16 | 17 | You also need Node.js/npm installed. See the [Node.js downloads](https://nodejs.org/en/download/) page to find an installer for your platform. 18 | 19 | ## Installation 20 | 21 | Inside your Jupyter notebook, install *pixiedust_node* with 22 | 23 | ```python 24 | !pip install pixiedust_node 25 | ``` 26 | 27 | ## Running 28 | 29 | Once installed, a notebook can start up *pixiedust_node* with: 30 | 31 | ```python 32 | import pixiedust_node 33 | ``` 34 | 35 | ## Using %%node 36 | 37 | Use the `%%node` prefix in a notebook cell to indicate that the content that follows is JavaScript. 38 | 39 | ```js 40 | %%node 41 | print(new Date()); 42 | ``` 43 | 44 | ## Installing npm modules 45 | 46 | You can install any [npm](https://www.npmjs.com/) module to use in your Node.js code from your notebook. To install npm modules, in a Python cell: 47 | 48 | ```python 49 | npm.install('silverlining') 50 | ``` 51 | 52 | or install multiple libraries in one go: 53 | 54 | ```python 55 | npm.install( ('request', 'request-promise') ) 56 | ``` 57 | 58 | and then "require" the modules in your Node.js code. 59 | 60 | ```js 61 | %%node 62 | var silverlining = require('silverlining'); 63 | var request = require('request-promise'); 64 | ``` 65 | 66 | You may also do : 67 | 68 | - `npm.uninstall('packagename')` - to remove an npm module (or `npm.remove('packagename')`) 69 | - `npm.list()` - to list the installed modules 70 | 71 | ## Node.js helper functions 72 | 73 | Node.js functions are available to interact with the Notebook 74 | 75 | - `print(x)` - print out the value of variable x 76 | - `display(x)` - use Pixiedust's `display` function to visualise an array of data 77 | - `store(x,'y')` - turn a JavaScript array x into a Pandas data frame and store in Python variable y 78 | - `html(x)` - render HTML string x in a notebook cell 79 | - `image(x)` - render image URL x in a notebook cell 80 | - `help()` - show help 81 | 82 | ### print 83 | 84 | ```js 85 | %%node 86 | // connect to Cloudant using Silverlining 87 | var url = 'https://reader.cloudant.com/cities'; 88 | var cities = silverlining(url); 89 | 90 | // fetch number of cities per country 91 | cities.count('country').then(print); 92 | ``` 93 | 94 | ### display 95 | 96 | ```js 97 | %%node 98 | 99 | // fetch cities called York 100 | cities.query({name: 'York'}).then(display); 101 | ``` 102 | 103 | ### store 104 | 105 | ** This function is deprecated as Node.js global variables are copied to the Python environment automatically ** 106 | 107 | ```js 108 | %%node 109 | 110 | // fetch the data and store in Pandas dataframe called 'x' 111 | cities.all({limit: 2500}).then(function(data) { 112 | store(data, 'x'); 113 | }); 114 | ``` 115 | 116 | The dataframe 'x' is now available to use in a Python cell: 117 | 118 | ```python 119 | x['population'].sum() 120 | ``` 121 | 122 | ### html 123 | 124 | ```js 125 | %%node 126 | var str = 'Sales are up 25%'; 127 | html(str); 128 | ``` 129 | 130 | ### image 131 | 132 | ```js 133 | %%node 134 | var url = 'http://myserver.com/path/to/image.jpg'; 135 | image(url); 136 | ``` 137 | 138 | ### help 139 | 140 | ```js 141 | %%node 142 | help(); 143 | ``` 144 | 145 | ## Node.js-Python bridge 146 | 147 | Any *global* variables that you create in your `%%node` cells will be automatically copied to equivalent variables in Python. e.g if you create some variables in a Node.js cell: 148 | 149 | ``` 150 | %%node 151 | var str = "hello world"; 152 | var n1 = 4.1515; 153 | var n2 = 42; 154 | var tf = true; 155 | var obj = { name:"Frank", age: 42 }; 156 | var array_of_strings = ["hello", "world"]; 157 | var array_of_objects = [{a:1,b:2}, {a:3, b:4}]; 158 | ``` 159 | 160 | Then these variables can be used in Python: 161 | 162 | ``` 163 | # Python cell 164 | print str, n1, n2, tf 165 | print obj 166 | print array_of_strings 167 | print array_of_objects 168 | ``` 169 | 170 | Strings, numbers, booleans and arrays of such are converted to their equivalent in Python. Objects are converted into Python dictionaries and arrays of objects are automatically converted into a Pandas DataFrames. 171 | 172 | Note that only variables declared with `var` are moved to Python, not constants declared with `const`. 173 | 174 | 175 | If you want to move data from an asynchronous Node.js callback, remember to write it to a *global variable*: 176 | 177 | ```js 178 | %%node 179 | var googlehomepage = ''; 180 | request.get('http://www.google.com').then(function(data) { 181 | googlehomepage = data; 182 | print('Fetched Google homepage'); 183 | }); 184 | ``` 185 | 186 | Similarly, Python variables of type `str`, `int`, `float`, `bool`, `unicode`, `dict` or `list` will be moved to Node.js when a cell is executed: 187 | 188 | ``` 189 | # Python cell 190 | a = 'hello' 191 | b = 2 192 | b = 3 193 | c= False 194 | d = {} 195 | d["x"] = 1 196 | d["y"] = 2 197 | e = 3.142 198 | ``` 199 | 200 | The variables can then be used in Node.js: 201 | 202 | ``` 203 | %%node 204 | console.log(a,b,c,d,e); 205 | // hello 3 false { y: 2, x: 1 } 3.142 206 | ``` 207 | 208 | ## Managing the Node.js process 209 | 210 | If enter some invalid syntax into a `%%node` cell, such as code with more opening brackets than closing brackes, then the Node.js interpreter may not think you have finished typing and you receive no output. 211 | 212 | You can cancel execution by running the following command in a Python cell: 213 | 214 | ```python 215 | node.cancel() 216 | ``` 217 | 218 | If you need to clear your Node.js variables and restart from the beginning then issue the following command in an Python cell: 219 | 220 | ```python 221 | node.clear() 222 | ``` 223 | 224 | ## Help 225 | 226 | You can view the help in a Python cell: 227 | 228 | ```python 229 | node.help() 230 | ``` 231 | -------------------------------------------------------------------------------- /pixiedust_node/node.py: -------------------------------------------------------------------------------- 1 | 2 | import json 3 | import os 4 | import sys 5 | import platform 6 | import subprocess 7 | import hashlib 8 | from functools import partial 9 | from threading import Thread, Event 10 | 11 | import IPython 12 | import pandas 13 | from pixiedust.display import display 14 | from pixiedust.utils.environment import Environment 15 | from pixiedust.utils.shellAccess import ShellAccess 16 | 17 | RESERVED = ['true', 'false','self','this','In','Out'] 18 | 19 | try: 20 | VARIABLE_TYPES = (str, int, float, bool, unicode, dict, list) 21 | except: 22 | # Python 3 => no unicode type 23 | VARIABLE_TYPES = (str, int, float, bool, dict, list) 24 | 25 | class VarWatcher(object): 26 | """ 27 | this class watches for cell "post_execute" events. When one occurs, it examines 28 | the IPython shell for variables that have been set (only numbers and strings). 29 | New or changed variables are moved over to the JavaScript environment. 30 | """ 31 | 32 | def __init__(self, ip, ps): 33 | self.shell = ip 34 | self.ps = ps 35 | ip.events.register('post_execute', self.post_execute) 36 | self.clearCache() 37 | 38 | def clearCache(self): 39 | self.cache = {} 40 | 41 | # the cache contains the key (variable name) against an MD5 of the 42 | # JSON form of the value. This makes the cache more compact. 43 | def setCache(self, key, val): 44 | self.cache[key] = hashlib.md5(json.dumps(val).encode("utf8")).hexdigest() 45 | 46 | # check whether key is in cache and whether it equals val by comparing 47 | # the hash of its JSON value with what's in the cache 48 | def inCache(self, key, val): 49 | hash = hashlib.md5(json.dumps(val).encode("utf8")).hexdigest() 50 | return (key in self.cache and self.cache[key] == hash) 51 | 52 | def post_execute(self): 53 | for key in self.shell.user_ns: 54 | v = self.shell.user_ns[key] 55 | t = type(v) 56 | # if this is one of our varables, is a number or a string or a float 57 | if not key.startswith('_') and (not key in RESERVED) and (t in VARIABLE_TYPES): 58 | # if it's not in our cache or it is an its value has changed 59 | if not key in self.cache or not self.inCache(key, v): 60 | # move it to JavaScript land and add it to our cache 61 | self.ps.stdin.write("var " + key + " = " + json.dumps(v) + ";\r\n") 62 | self.setCache(key, v) 63 | 64 | class NodeStdReader(Thread): 65 | """ 66 | Thread class that is given a process in the constructor 67 | the thead listens to each line coming out of the 68 | process's stdout and checks to see if it is JSON. 69 | if it is, and it's a special Pixiedust command, 70 | then the pixiedust display/print function is called 71 | """ 72 | 73 | def __init__(self, ps, vw): 74 | super(NodeStdReader, self).__init__() 75 | self._stop_event = Event() 76 | self.ps = ps 77 | self.vw = vw 78 | self.daemon = True 79 | self.start() 80 | 81 | def stop(self): 82 | self._stop_event.set() 83 | 84 | def run(self): 85 | 86 | # forever 87 | while not self._stop_event.is_set(): 88 | # read line from Node's stdout 89 | line = self.ps.stdout.readline() 90 | 91 | # see if it parses as JSON 92 | obj = None 93 | try: 94 | if line: 95 | obj = json.loads(line) 96 | # if it does and is a pixiedust object 97 | if obj and isinstance(obj, dict) and obj['_pixiedust']: 98 | if obj['type'] == 'display': 99 | pdf = pandas.DataFrame(obj['data']) 100 | ShellAccess.pdf = pdf 101 | display(pdf) 102 | elif obj['type'] == 'print': 103 | print(json.dumps(obj['data'])) 104 | elif obj['type'] == 'store': 105 | print('!!! Warning: store is now deprecated - Node.js global variables are automatically propagated to Python !!!') 106 | variable = 'pdf' 107 | if 'variable' in obj: 108 | variable = obj['variable'] 109 | ShellAccess[variable] = pandas.DataFrame(obj['data']) 110 | elif obj['type'] == 'html': 111 | IPython.display.display(IPython.display.HTML(obj['data'])) 112 | elif obj['type'] == 'image': 113 | IPython.display.display(IPython.display.HTML(''.format(obj['data']))) 114 | elif obj['type'] == 'variable': 115 | ShellAccess[obj['key']] = obj['value'] 116 | if self.vw: 117 | self.vw.setCache(obj['key'], obj['value']) 118 | else: 119 | print(line) 120 | except Exception as e: 121 | # output the original line when we don't have JSON 122 | line = line.strip() 123 | if len(line) > 0: 124 | print(line) 125 | 126 | 127 | 128 | class NodeBase(object): 129 | """ 130 | Node base class with common tasks for Node.js and NPM process runs. 131 | """ 132 | @staticmethod 133 | def is_exe(fpath): 134 | return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 135 | 136 | @staticmethod 137 | def which(program): 138 | fpath, fname = os.path.split(program) 139 | if fpath: 140 | if NodeBase.is_exe(program): 141 | return program 142 | else: 143 | for path in os.environ["PATH"].split(os.pathsep): 144 | path = path.strip('"') 145 | exe_file = os.path.join(path, program) 146 | if NodeBase.is_exe(exe_file): 147 | return exe_file 148 | 149 | return None 150 | 151 | def __init__(self): 152 | """ 153 | Establishes the Node's home directories and executable paths 154 | """ 155 | # get the home directory 156 | home = Environment.pixiedustHome 157 | 158 | # Node home directory 159 | self.node_home = os.path.join(home, 'node') 160 | if not os.path.exists(self.node_home): 161 | os.makedirs(self.node_home) 162 | 163 | # Node modules home directory 164 | self.node_modules = os.path.join(self.node_home, 'node_modules') 165 | if not os.path.exists(self.node_modules): 166 | os.makedirs(self.node_modules) 167 | 168 | self.node_prog = 'node' 169 | self.npm_prog = 'npm' 170 | if platform.system() == 'Windows': 171 | self.node_prog += '.exe' 172 | self.npm_prog += '.cmd' 173 | 174 | self.node_path = NodeBase.which(self.node_prog) 175 | if self.node_path is None: 176 | print('ERROR: Cannot find Node.js executable') 177 | raise FileNotFoundError('node executable not found in path') 178 | 179 | self.npm_path = NodeBase.which(self.npm_prog) 180 | if self.npm_path is None: 181 | print('ERROR: Cannot find npm executable') 182 | raise FileNotFoundError('npm executable not found in path') 183 | 184 | # Create popen partial, that will be used later 185 | popen_kwargs = { 186 | 'stdin': subprocess.PIPE, 187 | 'stdout': subprocess.PIPE, 188 | 'stderr': subprocess.STDOUT, 189 | 'cwd': self.node_home 190 | } 191 | if sys.version_info.major == 3: 192 | popen_kwargs['encoding'] = 'utf-8' 193 | self.popen = partial(subprocess.Popen, **popen_kwargs) 194 | 195 | 196 | class Node(NodeBase): 197 | """ 198 | Class runs a Node sub-process and starts a NodeStdReader thread 199 | to listen to its stdout. 200 | """ 201 | def __init__(self, path): 202 | """ 203 | Constructor runs a JavaScript script (path) with "node" 204 | :param path: JavaScript path 205 | """ 206 | super(Node, self).__init__() 207 | 208 | # process that runs the Node.js code 209 | args = (self.node_path, path) 210 | self.ps = self.popen(args) 211 | #print ("Node process id", self.ps.pid) 212 | 213 | # watch Python variables for changes 214 | self.vw = VarWatcher(get_ipython(), self.ps) 215 | 216 | # create thread to read this process's output 217 | NodeStdReader(self.ps, self.vw) 218 | 219 | def terminate(self): 220 | self.ps.terminate() 221 | 222 | def write(self, s): 223 | self.ps.stdin.write(s) 224 | self.ps.stdin.write("\r\n") 225 | self.ps.stdin.flush() 226 | 227 | def cancel(self): 228 | self.write("\r\n.break") 229 | 230 | def clear(self): 231 | self.write("\r\n.clear") 232 | self.vw.clearCache() 233 | 234 | def help(self): 235 | self.cancel() 236 | self.write("help()\r\n") 237 | 238 | 239 | class Npm(NodeBase): 240 | """ 241 | npm helper class 242 | allows npm modules to be installed, removed and listed 243 | """ 244 | def __init__(self): 245 | super(Npm, self).__init__() 246 | 247 | # run an npm command 248 | def cmd(self, command, module): 249 | args = [self.npm_path, command, '-s'] 250 | if module: 251 | if isinstance(module, str): 252 | args.append(module) 253 | else: 254 | args.extend(module) 255 | print(' '.join(args)) 256 | ps = self.popen(args) 257 | 258 | # create thread to read this process's output 259 | t = NodeStdReader(ps, None) 260 | 261 | # wait for the sub-process to exit 262 | ps.wait() 263 | 264 | # tell the thread reading its output to stop too, to prevent 100% CPU usage 265 | t.stop() 266 | 267 | def install(self, module): 268 | self.cmd('install', module) 269 | 270 | def remove(self, module): 271 | self.cmd('uninstall', module) 272 | 273 | def uninstall(self, module): 274 | self.cmd('uninstall', module) 275 | 276 | def list(self): 277 | self.cmd('list', None) 278 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | 204 | --------------------------------------------------------------------------------