7 | 8 | Learn: [http://blaze.pydata.org/](http://blaze.pydata.org/) 9 | 10 | Try: `conda install blaze` 11 | 12 | Contribute: 13 | [http://github.com/blaze/blaze/](http://github.com/blaze/blaze) 14 | -------------------------------------------------------------------------------- /blaze/tests/test_core.py: -------------------------------------------------------------------------------- 1 | from blaze import into, drop, create_index 2 | from blaze.compute.core import compute_down, compute_up 3 | from multipledispatch.conflict import ambiguities 4 | from blaze.expr.arithmetic import scalar_coerce 5 | 6 | 7 | def test_no_dispatch_ambiguities(): 8 | for func in [into, compute_up, compute_down, drop, create_index, 9 | scalar_coerce]: 10 | assert not ambiguities(func.funcs) 11 | -------------------------------------------------------------------------------- /blaze/expr/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | 3 | from .core import * 4 | from .expressions import * 5 | from .literal import * 6 | from .strings import * 7 | from .datetime import * 8 | from .math import * 9 | from .arithmetic import * 10 | from .broadcast import * 11 | from .collections import * 12 | from .split_apply_combine import * 13 | from .reductions import * 14 | from .arrays import * 15 | from . import math 16 | -------------------------------------------------------------------------------- /docs/source/_static/presentations/markdown/frontbackends.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | 11 | 12 | 13 | 17 | -------------------------------------------------------------------------------- /blaze/tests/dummydata.csv: -------------------------------------------------------------------------------- 1 | Name,RegistrationDate,ZipCode,Consts 2 | Cynthia Oneal,2006-10-07,57003,3.14159 3 | James Eaton,2011-07-07,71409,6.022141 4 | Mickey Boggs,2010-05-31,58326,9.274009 5 | Cornell Horne,2003-06-03,61249,5.29000 6 | Jami Riggs,2010-02-25,24123,1.380648 7 | Steven Dillon,2012-01-27,21303,2.765000 8 | Tad Sharp,2013-01-24,35689,8.987551 9 | Eugenia Maynard,2006-05-10,18220,.37 10 | Emmett Mullen,2009-12-06,41855,.73 11 | Thomas Sexton,2009-01-02,19211,.2701 -------------------------------------------------------------------------------- /docs/source/_static/presentations/plugin/multiplex/client.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | var multiplex = Reveal.getConfig().multiplex; 3 | var socketId = multiplex.id; 4 | var socket = io.connect(multiplex.url); 5 | 6 | socket.on(multiplex.id, function(data) { 7 | // ignore data from sockets that aren't ours 8 | if (data.socketId !== socketId) { return; } 9 | if( window.location.host === 'localhost:1947' ) return; 10 | 11 | Reveal.slide(data.indexh, data.indexv, data.indexf, 'remote'); 12 | }); 13 | }()); 14 | -------------------------------------------------------------------------------- /blaze/compute/tests/test_pmap.py: -------------------------------------------------------------------------------- 1 | from blaze import compute, data, symbol, discover 2 | from blaze.utils import example 3 | 4 | 5 | flag = [False] 6 | 7 | def mymap(func, *args): 8 | flag[0] = True 9 | return map(func, *args) 10 | 11 | 12 | def test_map_called_on_data_star(): 13 | r = data(example('accounts_*.csv')) 14 | s = symbol('s', discover(r)) 15 | flag[0] = False 16 | a = compute(s.count(), r) 17 | b = compute(s.count(), r, map=mymap) 18 | assert a == b 19 | assert flag[0] 20 | -------------------------------------------------------------------------------- /docs/source/_static/presentations/images/dask.ones.dot: -------------------------------------------------------------------------------- 1 | strict digraph G { 2 | "('wrapped_1', 1)" [shape=box]; 3 | "('wrapped_1', 2)" [shape=box]; 4 | "('wrapped_1', 0)" [shape=box]; 5 | "((
10 |
11 |
12 | ## `dask.dataframe` is...
13 |
14 | Very new.
15 |
16 | Ready for use but known failures popping up.
17 |
18 | Needs to be exercised by the community.
19 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/markdown/start.md:
--------------------------------------------------------------------------------
1 | ## Blaze - an interface
2 |
3 | While we're waiting you may want to try the following
4 | 5 | 6 | Docs -- [http://blaze.pydata.org/](http://blaze.pydata.org/) 7 | 8 | Install -- `conda install blaze` 9 | 10 | Source -- 11 | [http://github.com/blaze/blaze/](http://github.com/blaze/blaze) 12 | 13 | 14 | ## TL;DR 15 | 16 | Blaze is an extensible interface for data analytics. 17 | 18 | It feels like NumPy/Pandas to users but drives other data systems. 19 | 20 | We achieve performance through accessibility 21 | -------------------------------------------------------------------------------- /docs/source/_static/presentations/markdown/icml-mloss.md: -------------------------------------------------------------------------------- 1 | ## Python and Parallelism 2 | 3 | *or* 4 | 5 | ## Dask 6 | 7 | *Matthew Rocklin* 8 | 9 | Continuum Analytics 10 | 11 | 12 | ## Outline 13 | 14 | * . 15 | * Dask - Dynamic Task Scheduling 16 | * Dask.array - out-of-core, parallel NumPy 17 | * Dask with other workloads 18 | * . 19 | 20 | 21 | ## Outline 22 | 23 | * Numeric Python and Parallelism 24 | * Dask - Dynamic Task Scheduling 25 | * Dask.array - out-of-core, parallel NumPy 26 | * Dask with other workloads 27 | * Parallelism and Machine Learning 28 | -------------------------------------------------------------------------------- /RELEASE_PROCEDURE.md: -------------------------------------------------------------------------------- 1 | Blaze Release Procedure 2 | ======================= 3 | 4 | 1. Run the conda build recipes across all platforms (Linux, OS X, 5 | Windows) including both 32 and 64-bit architectures. 6 | 7 | 2. Tag the release version. 8 | 9 | ```sh 10 | git tag -a x.x.x -m 'Version x.x.x' 11 | ``` 12 | 13 | 3. Push those tags to `blaze` master 14 | 15 | ```sh 16 | git push --tags upstream master 17 | ``` 18 | 19 | 3. Upload a tarball to PyPI 20 | 21 | ```sh 22 | python setup.py sdist upload # from within your clone of blaze 23 | ``` 24 | 25 | 4. Release email to blaze-dev@continuum.io. 26 | -------------------------------------------------------------------------------- /docs/source/whatsnew/0.11.2.txt: -------------------------------------------------------------------------------- 1 | Release 0.11.2 2 | ----------------- 3 | 4 | :Release: 0.11.2 5 | 6 | New Expressions 7 | ^^^^^^^^^^^^^^^ 8 | 9 | None 10 | 11 | Improved Expressions 12 | ^^^^^^^^^^^^^^^^^^^^ 13 | 14 | None 15 | 16 | New Backends 17 | ^^^^^^^^^^^^ 18 | 19 | None 20 | 21 | Improved Backends 22 | ^^^^^^^^^^^^^^^^^ 23 | 24 | None 25 | 26 | Experimental Features 27 | ^^^^^^^^^^^^^^^^^^^^^ 28 | 29 | None 30 | 31 | API Changes 32 | ^^^^^^^^^^^ 33 | 34 | None 35 | 36 | Bug Fixes 37 | ^^^^^^^^^ 38 | 39 | * `blaze.data()` now respects an explicit `dshape` argument in all cases 40 | (:issue:`1589`). 41 | -------------------------------------------------------------------------------- /docs/source/people.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributors 3 | ============ 4 | 5 | Current Core Developers 6 | ``````````````````````` 7 | * Phillip Cloud 8 | * Joe Jevnik 9 | * Matt Rocklin 10 | * Kurt Smith 11 | 12 | Contributors 13 | ```````````` 14 | * Andy R. Terrel 15 | * Mark Wiebe 16 | * Majid alDosari 17 | * Francesc Alted 18 | * Tyler Alumbaugh 19 | * Dav Clark 20 | * Stephen Diehl 21 | * Christine Doig 22 | * Mark Florisson 23 | * Damien Garaud 24 | * Valentin Haenel 25 | * Lila Hickey 26 | * Maggie Mari 27 | * Travis Oliphant 28 | * Milos Popovic 29 | * Stan Seibert 30 | * Hugo Shi 31 | * Oscar Villellas Guillén 32 | * Peter Wang 33 | * Matt Westcott 34 | * Ben Zaitlen 35 | -------------------------------------------------------------------------------- /blaze/compute/literal.py: -------------------------------------------------------------------------------- 1 | from odo import append, drop 2 | 3 | from ..compute.core import compute 4 | from ..dispatch import dispatch 5 | from ..expr.expressions import Expr 6 | from ..expr.literal import BoundSymbol 7 | 8 | 9 | @dispatch(Expr, BoundSymbol) 10 | def compute_down(expr, data, **kwargs): 11 | return compute(expr, data.data, **kwargs) 12 | 13 | 14 | @dispatch(Expr, BoundSymbol) 15 | def pre_compute(expr, data, **kwargs): 16 | return pre_compute(expr, data.data, **kwargs) 17 | 18 | 19 | @dispatch(BoundSymbol, object) 20 | def append(a, b, **kwargs): 21 | return append(a.data, b, **kwargs) 22 | 23 | 24 | @dispatch(BoundSymbol) 25 | def drop(d): 26 | return drop(d.data) 27 | -------------------------------------------------------------------------------- /blaze/compute/json.py: -------------------------------------------------------------------------------- 1 | from .core import pre_compute 2 | from ..dispatch import dispatch 3 | from ..expr import Expr 4 | from odo.backends.json import JSON, JSONLines 5 | from odo import into 6 | from collections import Iterator 7 | from odo.utils import records_to_tuples 8 | 9 | 10 | __all__ = ['pre_compute'] 11 | 12 | 13 | @dispatch(Expr, JSON) 14 | def pre_compute(expr, data, **kwargs): 15 | seq = into(list, data, **kwargs) 16 | leaf = expr._leaves()[0] 17 | return list(records_to_tuples(leaf.dshape, seq)) 18 | 19 | 20 | @dispatch(Expr, JSONLines) 21 | def pre_compute(expr, data, **kwargs): 22 | seq = into(Iterator, data, **kwargs) 23 | leaf = expr._leaves()[0] 24 | return records_to_tuples(leaf.dshape, seq) 25 | -------------------------------------------------------------------------------- /blaze/types.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import decimal 3 | 4 | import numpy as np 5 | import pandas as pd 6 | 7 | CORE_SCALAR_TYPES = ( 8 | float, 9 | decimal.Decimal, 10 | int, 11 | bool, 12 | str, 13 | pd.Timestamp, 14 | datetime.date, 15 | datetime.timedelta 16 | ) 17 | CORE_SEQUENCE_TYPES = ( 18 | list, 19 | dict, 20 | tuple, 21 | set, 22 | pd.Series, 23 | pd.DataFrame, 24 | np.ndarray 25 | ) 26 | CORE_TYPES = CORE_SCALAR_TYPES + CORE_SEQUENCE_TYPES 27 | 28 | 29 | def iscorescalar(x): 30 | return isinstance(x, CORE_SCALAR_TYPES) 31 | 32 | 33 | def iscoresequence(x): 34 | return isinstance(x, CORE_SEQUENCE_TYPES) 35 | 36 | 37 | def iscoretype(x): 38 | return isinstance(x, CORE_TYPES) 39 | -------------------------------------------------------------------------------- /docs/source/timeseries.rst: -------------------------------------------------------------------------------- 1 | Time Series Operations 2 | ====================== 3 | 4 | Blaze is starting to grow some time series related expressions. 5 | These are generally inspired by pandas API, but are different in cases 6 | where the pandas API is too big or doesn't work well with the way 7 | Blaze works. 8 | 9 | A notable difference is that blaze's time series operations are *not* 10 | centered around the notion of an index. 11 | 12 | .. note:: 13 | 14 | Time series functionality is very new. Please keep this in mind 15 | when writing production code that uses these new features. 16 | 17 | 18 | So far, we have :func:`~blaze.expr.collections.shift` since PR :issue:`1266`. 19 | 20 | On the short-term roadmap are rolling aggregates and resample for both the 21 | Pandas and SQL backends. 22 | -------------------------------------------------------------------------------- /blaze/server/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | 3 | from .server import Server, to_tree, from_tree, api, expr_md5 4 | from .spider import data_spider, from_yaml 5 | from .client import Client 6 | from .serialization import ( 7 | SerializationFormat, 8 | all_formats, 9 | fastmsgpack as fastmsgpack_format, 10 | json as json_format, 11 | msgpack as msgpack_format, 12 | pickle as pickle_format, 13 | ) 14 | 15 | 16 | __all__ = [ 17 | 'Client', 18 | 'SerializationFormat', 19 | 'Server', 20 | 'all_formats', 21 | 'api', 22 | 'data_spider', 23 | 'expr_md5', 24 | 'fastmsgpack_format', 25 | 'from_tree', 26 | 'from_yaml', 27 | 'json_format', 28 | 'msgpack_format', 29 | 'pickle_format', 30 | 'to_tree', 31 | ] 32 | -------------------------------------------------------------------------------- /binstar/all/.binstar.yml: -------------------------------------------------------------------------------- 1 | package: blaze 2 | user: blaze 3 | 4 | platform: 5 | - linux-64 6 | - linux-32 7 | - osx-64 8 | - win-32 9 | - win-64 10 | engine: 11 | - python=2.6 12 | - python=2.7 13 | - python=3.3 14 | - python=3.4 15 | 16 | install: 17 | - conda config --add channels blaze 18 | - conda config --add channels numba 19 | - conda config --set always_yes true 20 | 21 | before_script: 22 | - python -V 23 | 24 | script: 25 | - conda build ../../all.recipe 26 | 27 | iotimeout: 600 28 | 29 | build_targets: conda 30 | 31 | notifications: 32 | email: 33 | recipients: ['ben.zaitlen@continuum.io', 'mrocklin@continuum.io', 'phillip.cloud@continuum.io'] 34 | 35 | --- 36 | platform: win-32 37 | engine: python=2.6 38 | exclude: true 39 | --- 40 | platform: win-64 41 | engine: python=2.6 42 | exclude: true 43 | -------------------------------------------------------------------------------- /binstar/core/.binstar.yml: -------------------------------------------------------------------------------- 1 | package: blaze-core 2 | user: blaze 3 | 4 | platform: 5 | - linux-64 6 | - linux-32 7 | - osx-64 8 | - win-32 9 | - win-64 10 | engine: 11 | - python=2.6 12 | - python=2.7 13 | - python=3.3 14 | - python=3.4 15 | 16 | install: 17 | - conda config --add channels blaze 18 | - conda config --add channels numba 19 | - conda config --set always_yes true 20 | 21 | before_script: 22 | - python -V 23 | 24 | script: 25 | - conda build ../../core.recipe 26 | 27 | iotimeout: 600 28 | 29 | build_targets: conda 30 | 31 | notifications: 32 | email: 33 | recipients: ['ben.zaitlen@continuum.io', 'mrocklin@continuum.io', 'phillip.cloud@continuum.io'] 34 | 35 | --- 36 | platform: win-32 37 | engine: python=2.6 38 | exclude: true 39 | --- 40 | platform: win-64 41 | engine: python=2.6 42 | exclude: true 43 | -------------------------------------------------------------------------------- /blaze/compute/tests/test_json.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | 3 | from blaze.utils import example 4 | from odo.backends.json import JSON, JSONLines 5 | from odo import Chunks 6 | from blaze import symbol, discover, compute, into, data 7 | 8 | 9 | js = JSON(example('accounts.json')) 10 | jss = JSONLines(example('accounts-streaming.json')) 11 | 12 | s = symbol('s', discover(js)) 13 | 14 | 15 | def test_simple(): 16 | assert compute(s.count(), js) == 5 17 | assert compute(s.count(), jss) == 5 18 | 19 | 20 | def test_less_simple(): 21 | assert compute(s.amount.sum(), js) == 100 22 | assert compute(s.amount.sum(), jss) == 100 23 | 24 | 25 | def test_chunks_json(): 26 | r = data(example('accounts-streaming*.json')) 27 | assert isinstance(r.data, Chunks) 28 | assert compute(s.amount.sum(), r) == 200 29 | -------------------------------------------------------------------------------- /etc/requirements_ci.txt: -------------------------------------------------------------------------------- 1 | bcolz==0.12.1 2 | click==6.6 3 | cloudpickle==0.2.1 4 | Cython==0.25.1 5 | dask==0.10.2 6 | decorator==4.0.10 7 | Flask==0.11.1 8 | Flask-Cors==3.0.2 9 | future==0.16.0 10 | h5py==2.6.0 11 | itsdangerous==0.24 12 | Jinja2==2.8 13 | llvmlite==0.14.0 14 | locket==0.2.0 15 | MarkupSafe==0.23 16 | multipledispatch==0.4.9 17 | networkx==1.11 18 | numba==0.29.0 # please update the .travis.yml file! 19 | numexpr==2.6.1 20 | numpy==1.11.2 # please update the .travis.yml file! 21 | pandas==0.19.1 # please update the .travis.yml file! 22 | partd==0.3.6 23 | psutil==4.4.2 24 | psycopg2==2.7.3 25 | PyHive==0.2.1 26 | pymongo==2.9.4 27 | PyMySQL==0.7.9 28 | pytest==3.0.3 29 | pytest-cov==2.0.0 30 | python-dateutil==2.5.3 31 | pytz==2016.7 32 | pyyaml==3.11 33 | requests>=2.11.1 34 | six==1.10.0 35 | SQLAlchemy==1.1.3 36 | tables==3.3.0 37 | toolz==0.8.0 38 | Werkzeug==0.11.11 39 | -------------------------------------------------------------------------------- /blaze/compute/tests/test_optimize_compute.py: -------------------------------------------------------------------------------- 1 | from blaze.expr import Expr, symbol, Cast 2 | from blaze.compute import compute_up 3 | from blaze.compute.varargs import register_varargs_arity 4 | from blaze import compute 5 | 6 | 7 | class Foo(object): 8 | def __init__(self, data): 9 | self.data = data 10 | 11 | @compute_up.register((Expr, Cast), Foo) 12 | def compute_up(expr, data, **kwargs): 13 | return data 14 | 15 | # We need to resolve this ambiguity for test_no_dispatch_ambiguities 16 | register_varargs_arity(1, Foo) 17 | 18 | 19 | def optimize(expr, data): 20 | """ Renames leaf """ 21 | leaf = expr._leaves()[0] 22 | return expr._subs({leaf: symbol('newname', leaf.dshape)}) 23 | 24 | 25 | def test_scope_gets_updated_after_optimize_call(): 26 | a = symbol('a', 'int') 27 | result = compute(a + 1, Foo('foo'), optimize=optimize) 28 | assert result.data == 'foo' 29 | -------------------------------------------------------------------------------- /blaze/server/serialization/json_dumps_trusted.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | 3 | from collections import Callable 4 | 5 | from blaze.dispatch import dispatch 6 | from blaze.compatibility import u8 7 | from functools import partial 8 | 9 | from .json_dumps import json_dumps 10 | 11 | 12 | json_dumps_trusted_ns = dict() 13 | dispatch = partial(dispatch, namespace=json_dumps_trusted_ns) 14 | 15 | 16 | @dispatch(Callable) 17 | def json_dumps_trusted(f): 18 | # let the server serialize any callable - this is only used for testing 19 | # at present - do the error handling when json comes from client so in 20 | # object_hook, catch anything that is not pandas_numpy 21 | fcn = u8(".".join([f.__module__, f.__name__])) 22 | return {u'__!callable': fcn} 23 | 24 | 25 | for tps, func in json_dumps.funcs.items(): 26 | json_dumps_trusted.register(tps)(func) 27 | -------------------------------------------------------------------------------- /docs/source/_static/presentations/markdown/pydata-berlin.md: -------------------------------------------------------------------------------- 1 | ## Dask Arrays 2 | 3 | *or* 4 | 5 | ## PyData's Relationship with Parallelism 6 | 7 | *Matthew Rocklin* 8 | 9 | Continuum Analytics 10 | 11 | 12 | ## Outline 13 | 14 | * . 15 | * Dask.array 16 | * Multicore parallelism with blocked algorithms 17 | * Out-of-core execution with task scheduling 18 | * . 19 | * . 20 | 21 | 22 | ## Outline 23 | 24 | * PyData's uneasy relationship with parallelism 25 | * Dask.array 26 | * Multicore parallelism with blocked algorithms 27 | * Out-of-core execution with task scheduling 28 | * Dask.core 29 | * Extend parallelism to other contexts 30 | * PyData and the GIL 31 | 32 | 33 | ## Parallelism and Data 34 | 35 | * Gigabyte - Fits in memory, need one core (laptop) 36 | * Terabyte - Fits on disk, need ten cores (workstation) 37 | * Petabyte - Fits on many disks, need 1000 cores (cluster) 38 | -------------------------------------------------------------------------------- /docs/source/_static/presentations/markdown/mloss/finish.md: -------------------------------------------------------------------------------- 1 | ## Final Thoughts 2 | 3 | * Python and Parallelism 4 | * Most data is small 5 | * Storage, representation, streaming, sampling offer bigger gains 6 | * That being said, please [release the GIL](https://github.com/scikit-image/scikit-image/pull/1519) 7 | 8 | * Dask: Dynamic task scheduling yields sane parallelism 9 | * Simple library to enable parallelism 10 | * Dask.array/dataframe demonstrate ability 11 | * Rarely optimal performance (Theano is far smarter) 12 | * Scheduling necessary for composed algorithms 13 | 14 | * Questions: 15 | * Appropriate class of problems in ML? 16 | * What is the right API for algorithm builders? 17 | 18 | 19 | ## Questions? 20 | 21 | [http://dask.pydata.org](http://dask.pydata.org) 22 | 23 |
24 |
25 | 
26 |
--------------------------------------------------------------------------------
/blaze/expr/tests/test_functions.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import, division, print_function
2 |
3 | from blaze.expr.functions import *
4 | from blaze.expr import Expr, symbol
5 | from blaze.expr import math
6 | import numpy as np
7 |
8 |
9 | def test_reductions():
10 | assert max([1, 2, 3]) == 3
11 | assert max(np.array([1, 2, 3])) == 3
12 | assert isinstance(max(symbol('t', 'var * {x: int, y: int}').x), Expr)
13 | assert all([True, True, True]) is True
14 | assert all([True, True, False]) is False
15 |
16 |
17 | def test_math():
18 | assert sin(0) == 0
19 | assert isinstance(sin(0), float)
20 | assert isinstance(sin(np.int32(0)), np.float)
21 | assert isinstance(sin(symbol('x', 'real')), Expr)
22 |
23 |
24 | def test_abs():
25 | assert abs(-1) == 1
26 | assert (abs(np.array([-1, 2, 3])) == np.array([1, 2, 3])).all()
27 | x = symbol('x', 'int')
28 | assert isinstance(abs(x), math.abs)
29 |
--------------------------------------------------------------------------------
/docs/source/whatsnew/0.12.0.txt:
--------------------------------------------------------------------------------
1 | Release 0.12.0
2 | -----------------
3 |
4 | :Release: 0.12.0
5 |
6 | New Expressions
7 | ^^^^^^^^^^^^^^^
8 |
9 | None
10 |
11 | Improved Expressions
12 | ^^^^^^^^^^^^^^^^^^^^
13 |
14 | * Adds support for variadic expressions of known
15 | types. :func:`~blaze.expr.expressions.merge` is now implemented as a variadic
16 | expression (:issue:`1611`).
17 |
18 | New Backends
19 | ^^^^^^^^^^^^
20 |
21 | None
22 |
23 | Improved Backends
24 | ^^^^^^^^^^^^^^^^^
25 |
26 | None
27 |
28 | Experimental Features
29 | ^^^^^^^^^^^^^^^^^^^^^
30 |
31 | None
32 |
33 | API Changes
34 | ^^^^^^^^^^^
35 |
36 | * Adds a ``drop_field`` method to tabular expressions for taking subsets of the
37 | fields of an expression while preserving order.
38 |
39 | Bug Fixes
40 | ^^^^^^^^^
41 |
42 | * The ``flask.ext.cors`` import was updated to resolve a ``DeprecationWarning``
43 | (:issue:`1556`).
44 |
45 | Miscellaneous
46 | ^^^^^^^^^^^^^
47 |
48 | None
49 |
--------------------------------------------------------------------------------
/blaze/deprecation.py:
--------------------------------------------------------------------------------
1 | from __future__ import print_function, division, absolute_import
2 | import warnings
3 | from functools import wraps
4 |
5 |
6 | def deprecated(version, replacement=None):
7 | """Define a deprecation decorator.
8 | An optional `replacement` should refer to the new API to be used instead.
9 |
10 | Example:
11 | @deprecated('1.1')
12 | def old_func(): ...
13 |
14 | @deprecated('1.1', 'new_func')
15 | def old_func(): ..."""
16 |
17 | def decorator(func):
18 | def wrapper(*args, **kwargs):
19 | msg = "\"{}\" has been deprecated in version {} and will be removed in a future version."
20 | if replacement:
21 | msg += "\n Use \"{}\" instead."
22 | warnings.warn(msg.format(func.__name__, version, replacement),
23 | category=DeprecationWarning, stacklevel=2)
24 | return func(*args, **kwargs)
25 |
26 | return wraps(func)(wrapper)
27 |
28 | return decorator
29 |
--------------------------------------------------------------------------------
/blaze/compute/tests/test_chunks.py:
--------------------------------------------------------------------------------
1 | from odo import chunks
2 | from blaze import discover, into, compute, symbol
3 | from datashape.predicates import iscollection
4 |
5 |
6 | L = [1, 2, 3, 4, 5, 6]
7 | cL = chunks(list)([[1., 2., 3.], [4., 5., 6.]])
8 | s = symbol('s', discover(cL))
9 |
10 |
11 | def test_chunks_compute():
12 | exprs = [s, s + 1, s.max(), s.mean() + 1, s.head()]
13 | for e in exprs:
14 | result = compute(e, {s: cL})
15 | expected = compute(e, {s: L})
16 | if iscollection(e.dshape):
17 | result = into(list, result)
18 | expected = into(list, expected)
19 | assert result == expected
20 |
21 |
22 | def test_chunks_head():
23 | assert compute(s.head(2), cL) == (1., 2.)
24 |
25 |
26 | def test_pmap_default():
27 | flag = [0]
28 |
29 | def mymap(func, seq):
30 | flag[0] = True
31 | return map(func, seq)
32 |
33 | from blaze import set_default_pmap
34 | set_default_pmap(mymap)
35 |
36 | compute(s + 1, cL)
37 |
38 | assert flag[0] is True
39 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/css/theme/template/settings.scss:
--------------------------------------------------------------------------------
1 | // Base settings for all themes that can optionally be
2 | // overridden by the super-theme
3 |
4 | // Background of the presentation
5 | $backgroundColor: #2b2b2b;
6 |
7 | // Primary/body text
8 | $mainFont: 'Lato', sans-serif;
9 | $mainFontSize: 36px;
10 | $mainColor: #eee;
11 |
12 | // Headings
13 | $headingMargin: 0 0 20px 0;
14 | $headingFont: 'League Gothic', Impact, sans-serif;
15 | $headingColor: #eee;
16 | $headingLineHeight: 0.9em;
17 | $headingLetterSpacing: 0.02em;
18 | $headingTextTransform: uppercase;
19 | $headingTextShadow: 0px 0px 6px rgba(0,0,0,0.2);
20 | $heading1TextShadow: $headingTextShadow;
21 |
22 | // Links and actions
23 | $linkColor: #13DAEC;
24 | $linkColorHover: lighten( $linkColor, 20% );
25 |
26 | // Text selection
27 | $selectionBackgroundColor: #FF5E99;
28 | $selectionColor: #fff;
29 |
30 | // Generates the presentation background, can be overridden
31 | // to return a background image or gradient
32 | @mixin bodyBackground() {
33 | background: $backgroundColor;
34 | }
--------------------------------------------------------------------------------
/docs/source/_static/presentations/plugin/print-pdf/print-pdf.js:
--------------------------------------------------------------------------------
1 | /**
2 | * phantomjs script for printing presentations to PDF.
3 | *
4 | * Example:
5 | * phantomjs print-pdf.js "http://lab.hakim.se/reveal-js?print-pdf" reveal-demo.pdf
6 | *
7 | * By Manuel Bieh (https://github.com/manuelbieh)
8 | */
9 |
10 | // html2pdf.js
11 | var page = new WebPage();
12 | var system = require( 'system' );
13 |
14 | page.viewportSize = {
15 | width: 1024,
16 | height: 768
17 | };
18 |
19 | page.paperSize = {
20 | format: 'letter',
21 | orientation: 'landscape',
22 | margin: {
23 | left: '0',
24 | right: '0',
25 | top: '0',
26 | bottom: '0'
27 | }
28 | };
29 |
30 | var revealFile = system.args[1] || 'index.html?print-pdf';
31 | var slideFile = system.args[2] || 'slides.pdf';
32 |
33 | if( slideFile.match( /\.pdf$/gi ) === null ) {
34 | slideFile += '.pdf';
35 | }
36 |
37 | console.log( 'Printing PDF...' );
38 |
39 | page.open( revealFile, function( status ) {
40 | console.log( 'Printed succesfully' );
41 | page.render( slideFile );
42 | phantom.exit();
43 | } );
44 |
45 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/markdown/expressions.md:
--------------------------------------------------------------------------------
1 | # Expressions
2 |
3 |
4 | * SymPy expressions...
5 | * encode mathematical equations as Python objects
6 | * generate numeric, high-performance, code
7 | * enable mathematical reasoning
8 |
9 | 10 | 11 | We combine high-level reasoning and low-level performance. 12 | 13 | 14 | * Blaze expressions... 15 | * encode relational and linear algebra, rather than calculus and trigonometry 16 | * interpret to other systems 17 | * enable data reasoning 18 | * are extensible (in a way that keeps you sane) 19 | 20 |
21 | 22 | We combine high-level reasoning and low-level performance. 23 | 24 | 25 | ## Take-aways 26 | 27 | * Separating expressions from computation ... 28 | 1. enables users to transition easily between compute backends (right tool for 29 | the job) 30 | 2. enables developers to focus on user interface 31 | 3. enables developers to focus on computational backends 32 | 4. shares reasoning and query optimizations across backends 33 | 5. eases growth of Blaze to new backends when they arise 34 | -------------------------------------------------------------------------------- /blaze/tests/test_cached.py: -------------------------------------------------------------------------------- 1 | from blaze.cached import CachedDataset 2 | from blaze import symbol, discover, compute, into 3 | import pandas as pd 4 | from collections import Iterator 5 | 6 | 7 | df = pd.DataFrame([['Alice', 100, 1], 8 | ['Bob', 200, 2], 9 | ['Alice', 50, 3]], 10 | columns=['name', 'amount', 'id']) 11 | 12 | 13 | t = symbol('t', discover(df)) 14 | 15 | 16 | def test_dataset(): 17 | ns = {'t': df, 'x': 10} 18 | cache=dict() 19 | d = CachedDataset(ns, cache=cache) 20 | 21 | assert discover(d) == discover(ns) 22 | 23 | s = symbol('s', discover(d)) 24 | compute(s.x * 2, d) == 20 25 | cache == {s.x * 2: 20} 26 | 27 | 28 | def test_streaming(): 29 | seq = [{'name': 'Alice', 'x': 1}, 30 | {'name': 'Bob', 'x': 1}] 31 | ns = {'t': seq, 'x': 10} 32 | cache=dict() 33 | d = CachedDataset(ns, cache=cache) 34 | 35 | s = symbol('s', discover(d)) 36 | expr = s.t.x * 2 37 | result = compute(expr, d) 38 | 39 | assert not isinstance(d.cache[expr], Iterator) 40 | assert into(list, d.cache[expr]) == [2, 2] 41 | -------------------------------------------------------------------------------- /docs/source/whatsnew/0.10.1.txt: -------------------------------------------------------------------------------- 1 | Release 0.10.1 2 | -------------- 3 | 4 | :Release: 0.10.1 5 | :Date: TBD 6 | 7 | New Expressions 8 | ^^^^^^^^^^^^^^^ 9 | 10 | None 11 | 12 | Improved Expressions 13 | ^^^^^^^^^^^^^^^^^^^^ 14 | 15 | None 16 | 17 | New Backends 18 | ^^^^^^^^^^^^ 19 | 20 | None 21 | 22 | Improved Backends 23 | ^^^^^^^^^^^^^^^^^ 24 | 25 | * Blaze server's ``/add`` endpoint was enhanced to take a more general payload 26 | (:issue:`1481`). 27 | * Adds consistency check to blaze server at startup for YAML file and dynamic 28 | addition options (:issue:`1491`). 29 | 30 | Experimental Features 31 | ^^^^^^^^^^^^^^^^^^^^^ 32 | 33 | * The ``str_cat()`` expression was added, mirroring Pandas' 34 | ``Series.str.cat()`` API (:issue:`1496`). 35 | 36 | API Changes 37 | ^^^^^^^^^^^ 38 | 39 | None 40 | 41 | 42 | Bug Fixes 43 | ^^^^^^^^^ 44 | 45 | * The content type specification parsing was improved to accept more elaborate 46 | headers (:issue:`1490`). 47 | * The discoverablility consistency check is done before a dataset is 48 | dynamically added to the server (:issue:`1498`). 49 | 50 | Miscellaneous 51 | ^^^^^^^^^^^^^ 52 | 53 | None 54 | -------------------------------------------------------------------------------- /docs/source/_static/presentations/css/theme/source/night.scss: -------------------------------------------------------------------------------- 1 | /** 2 | * Black theme for reveal.js. 3 | * 4 | * Copyright (C) 2011-2012 Hakim El Hattab, http://hakim.se 5 | */ 6 | 7 | 8 | // Default mixins and settings ----------------- 9 | @import "../template/mixins"; 10 | @import "../template/settings"; 11 | // --------------------------------------------- 12 | 13 | 14 | // Include theme-specific fonts 15 | @import url(https://fonts.googleapis.com/css?family=Montserrat:700); 16 | @import url(https://fonts.googleapis.com/css?family=Open+Sans:400,700,400italic,700italic); 17 | 18 | 19 | // Override theme settings (see ../template/settings.scss) 20 | $backgroundColor: #111; 21 | 22 | $mainFont: 'Open Sans', sans-serif; 23 | $linkColor: #e7ad52; 24 | $linkColorHover: lighten( $linkColor, 20% ); 25 | $headingFont: 'Montserrat', Impact, sans-serif; 26 | $headingTextShadow: none; 27 | $headingLetterSpacing: -0.03em; 28 | $headingTextTransform: none; 29 | $selectionBackgroundColor: #e7ad52; 30 | $mainFontSize: 30px; 31 | 32 | 33 | // Theme template ------------------------------ 34 | @import "../template/theme"; 35 | // --------------------------------------------- -------------------------------------------------------------------------------- /docs/source/_static/presentations/plugin/postmessage/postmessage.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | simple postmessage plugin 4 | 5 | Useful when a reveal slideshow is inside an iframe. 6 | It allows to call reveal methods from outside. 7 | 8 | Example: 9 | var reveal = window.frames[0]; 10 | 11 | // Reveal.prev(); 12 | reveal.postMessage(JSON.stringify({method: 'prev', args: []}), '*'); 13 | // Reveal.next(); 14 | reveal.postMessage(JSON.stringify({method: 'next', args: []}), '*'); 15 | // Reveal.slide(2, 2); 16 | reveal.postMessage(JSON.stringify({method: 'slide', args: [2,2]}), '*'); 17 | 18 | Add to the slideshow: 19 | 20 | dependencies: [ 21 | ... 22 | { src: 'plugin/postmessage/postmessage.js', async: true, condition: function() { return !!document.body.classList; } } 23 | ] 24 | 25 | */ 26 | 27 | (function (){ 28 | 29 | window.addEventListener( "message", function ( event ) { 30 | var data = JSON.parse( event.data ), 31 | method = data.method, 32 | args = data.args; 33 | 34 | if( typeof Reveal[method] === 'function' ) { 35 | Reveal[method].apply( Reveal, data.args ); 36 | } 37 | }, false); 38 | 39 | }()); 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /docs/source/_static/presentations/REVEAL_LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2014 Hakim El Hattab, http://hakim.se 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /blaze/expr/tests/test_method_dispatch.py: -------------------------------------------------------------------------------- 1 | from blaze.expr.method_dispatch import select_functions, name, partial 2 | 3 | 4 | def iseven(n): 5 | return isinstance(n, int) and n % 2 == 0 6 | 7 | def inc(x): 8 | return Foo(x.data + 1) 9 | 10 | def dec(x): 11 | return Foo(x.data - 1) 12 | 13 | def lower(x): 14 | return Foo(x.data.lower()) 15 | 16 | def upper(x): 17 | return Foo(x.data.upper()) 18 | 19 | def halve(x): 20 | return Foo(x.data // 2) 21 | 22 | def isnine(x): 23 | return True 24 | 25 | 26 | methods = [(int, set([inc, dec])), 27 | (str, set([lower, upper])), 28 | (iseven, set([halve])), 29 | (9, set([isnine]))] 30 | 31 | 32 | def test_select_functions(): 33 | assert select_functions(methods, 3) == {'inc': inc, 'dec': dec} 34 | assert select_functions(methods, 4) == {'inc': inc, 'dec': dec, 'halve': halve} 35 | assert select_functions(methods, 'A') == {'lower': lower, 'upper': upper} 36 | assert select_functions(methods, 9) == {'inc': inc, 'dec': dec, 'isnine': 37 | isnine} 38 | 39 | 40 | def test_name(): 41 | assert name(inc) == 'inc' 42 | assert name(partial(inc)) == name(inc) 43 | -------------------------------------------------------------------------------- /docs/source/_static/presentations/css/theme/source/serif.scss: -------------------------------------------------------------------------------- 1 | /** 2 | * A simple theme for reveal.js presentations, similar 3 | * to the default theme. The accent color is brown. 4 | * 5 | * This theme is Copyright (C) 2012-2013 Owen Versteeg, http://owenversteeg.com - it is MIT licensed. 6 | */ 7 | 8 | 9 | // Default mixins and settings ----------------- 10 | @import "../template/mixins"; 11 | @import "../template/settings"; 12 | // --------------------------------------------- 13 | 14 | 15 | 16 | // Override theme settings (see ../template/settings.scss) 17 | $mainFont: 'Palatino Linotype', 'Book Antiqua', Palatino, FreeSerif, serif; 18 | $mainColor: #000; 19 | $headingFont: 'Palatino Linotype', 'Book Antiqua', Palatino, FreeSerif, serif; 20 | $headingColor: #383D3D; 21 | $headingTextShadow: none; 22 | $headingTextTransform: none; 23 | $backgroundColor: #F0F1EB; 24 | $linkColor: #51483D; 25 | $linkColorHover: lighten( $linkColor, 20% ); 26 | $selectionBackgroundColor: #26351C; 27 | 28 | .reveal a:not(.image) { 29 | line-height: 1.3em; 30 | } 31 | 32 | 33 | // Theme template ------------------------------ 34 | @import "../template/theme"; 35 | // --------------------------------------------- 36 | -------------------------------------------------------------------------------- /blaze/compute/hdfstore.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | 3 | from collections import namedtuple 4 | 5 | from .core import pre_compute 6 | from ..expr import Expr, Field 7 | from ..dispatch import dispatch 8 | 9 | from odo import into, chunks 10 | 11 | import pandas as pd 12 | 13 | 14 | @dispatch(Expr, pd.io.pytables.AppendableFrameTable) 15 | def pre_compute(expr, data, **kwargs): 16 | return into(chunks(pd.DataFrame), data, **kwargs) 17 | 18 | 19 | @dispatch(Expr, pd.io.pytables.Fixed) 20 | def pre_compute(expr, data, **kwargs): 21 | return into(pd.DataFrame, data, **kwargs) 22 | 23 | 24 | @dispatch(Field, pd.HDFStore) 25 | def compute_up(expr, data, **kwargs): 26 | key = '/' + expr._name 27 | if key in data.keys(): 28 | return data.get_storer(key) 29 | else: 30 | return HDFGroup(data, key) 31 | 32 | 33 | HDFGroup = namedtuple('HDFGroup', 'parent,datapath') 34 | 35 | 36 | @dispatch(Field, HDFGroup) 37 | def compute_up(expr, data, **kwargs): 38 | key = data.datapath + '/' + expr._name 39 | if key in data.parent.keys(): 40 | return data.parent.get_storer(key) 41 | else: 42 | return HDFGroup(data.parent, key) 43 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # See the docstring in versioneer.py for instructions. Note that you must 2 | # re-run 'versioneer.py setup' after changing this section, and commit the 3 | # resulting files. 4 | 5 | [versioneer] 6 | VCS = git 7 | style = pep440 8 | versionfile_source = blaze/_version.py 9 | versionfile_build = blaze/_version.py 10 | tag_prefix = 11 | parentdir_prefix = blaze- 12 | 13 | [flake8] 14 | # References: 15 | # http://flake8.readthedocs.org/en/latest/config.html 16 | # http://flake8.readthedocs.org/en/latest/warnings.html#error-codes 17 | # 18 | # Style checks turned on: 19 | # F - all pyflake errors except F811 due to multipledispatch multiple defines. 20 | # E101 - indentation contains mixed spaces and tabs 21 | # E111 - indentation is not a multiple of four 22 | # E131 - continuation line unaligned for hanging indent 23 | # E133 - closing bracket is missing indentation 24 | # E501 - line too long (see max-line-length) 25 | 26 | # Note: there cannot be spaces after commas here 27 | exclude = __init__.py,compatibility.py 28 | select = F4,F812,F82,F831,F841,E101,E111,E501 29 | ignore = F811,E,W 30 | max-line-length = 160 31 | 32 | # indentation style stuff 33 | # select = E131,E133 34 | # hang-closing = 1 35 | -------------------------------------------------------------------------------- /docs/source/_static/presentations/plugin/postmessage/example.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 |
7 |
8 |
9 |
10 |
11 |
12 |
37 |
38 |
39 |
40 |
--------------------------------------------------------------------------------
/blaze/expr/tests/test_array.py:
--------------------------------------------------------------------------------
1 | from blaze.expr import *
2 | from blaze.expr.arrays import *
3 | from blaze.utils import raises
4 |
5 |
6 | w = symbol('w', '5 * 6 * int32')
7 | x = symbol('x', '4 * 5 * int32')
8 | y = symbol('y', '5 * int32')
9 | z = symbol('z', '4 * int32')
10 |
11 | def test_shape_of_binop_of_differently_shaped_arrays():
12 | assert (x + y).shape == x.shape
13 |
14 |
15 | def test_transpose():
16 | assert ndim(x.dshape) == 2
17 | assert x.T.shape == (5, 4)
18 |
19 |
20 | def test_default_transpose_axes():
21 | a = symbol('a', '10 * 20 * 30 * 40 * int32')
22 | assert transpose(a).shape == a.shape[::-1]
23 |
24 |
25 | def test_dot():
26 | assert x.dot(y).shape == (4,)
27 | assert x.dot(w).shape == (4, 6)
28 |
29 |
30 | def test_tensordot():
31 | expr = tensordot(x, z, axes=[0, 0])
32 | assert expr.shape == (5,)
33 |
34 | expr = tensordot(x, z, axes=[[0], [0]])
35 | assert expr.shape == (5,)
36 |
37 |
38 | def test_tensordot_schema_matches_mul():
39 | x = symbol('x', '5 * int64')
40 | y = symbol('y', '5 * float32')
41 |
42 | assert tensordot(x, y).dshape.measure == (x + y).dshape.measure
43 |
44 |
45 | def test_shapes_raise_errors():
46 | assert raises(ValueError, lambda: w + x)
47 |
--------------------------------------------------------------------------------
/docs/source/whatsnew/0.11.1.txt:
--------------------------------------------------------------------------------
1 | Release 0.11.1
2 | -----------------
3 |
4 | :Release: 0.11.1
5 |
6 | New Expressions
7 | ^^^^^^^^^^^^^^^
8 |
9 | None
10 |
11 | Improved Expressions
12 | ^^^^^^^^^^^^^^^^^^^^
13 |
14 | None
15 |
16 | New Backends
17 | ^^^^^^^^^^^^
18 |
19 | None
20 |
21 | Improved Backends
22 | ^^^^^^^^^^^^^^^^^
23 |
24 | * The CSV backend was refactored to use ``dask.dataframe``. Previously it used
25 | ``odo.chunks`` (:issue:`1549`).
26 | * Configurable blaze server logging was added (:issue:`1521`).
27 |
28 | Experimental Features
29 | ^^^^^^^^^^^^^^^^^^^^^
30 |
31 | None
32 |
33 | API Changes
34 | ^^^^^^^^^^^
35 |
36 | None
37 |
38 | Bug Fixes
39 | ^^^^^^^^^
40 |
41 | * Fixed a testing regression introduced by the latest pymysql version
42 | (:issue:`1571`).
43 | * Fixed unicode conversion issues when using a Python 3 blaze server and a
44 | Python 2 client (:issue:`1572`), (:issue:`1566`).
45 | * Fixed ``dayofweek`` and ``dayofyear`` expressions for the SQL backend. The
46 | ``dayofweek`` expression uses monday = 0 like :class:`datetime.datetime` and
47 | :class:`pandas.Timestamp` instead of the SQL convention of sunday = 0.
48 | (:issue:`1585`).
49 |
50 | Miscellaneous
51 | ^^^^^^^^^^^^^
52 |
53 | * Support for Python 3.5 added.
54 |
--------------------------------------------------------------------------------
/blaze/server/serialization/object_hook_trusted.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import, division, print_function
2 |
3 | from .object_hook import object_hook, register
4 | import werkzeug.exceptions as wz_ex
5 | from blaze.compatibility import builtins, reduce
6 | import numpy as np
7 | import pandas as pd
8 |
9 | _converters_trusted = object_hook._converters.copy()
10 |
11 |
12 | def object_hook_trusted(ob, _converters=_converters_trusted):
13 | return object_hook(ob, _converters=_converters)
14 | object_hook_trusted._converters = _converters_trusted
15 | object_hook_trusted.register = register(converters=_converters_trusted)
16 | del _converters_trusted
17 |
18 |
19 | @object_hook_trusted.register('callable')
20 | def numpy_pandas_function_from_str(f):
21 | """
22 | reconstruct function from string representation
23 | """
24 | if f.startswith(np.__name__):
25 | mod = np
26 | elif f.startswith(pd.__name__):
27 | mod = pd
28 | elif f.startswith(builtins.__name__):
29 | mod = builtins
30 | else:
31 | msg = ("Function {} not recognized; only numpy, pandas, or builtin "
32 | "functions are supported.")
33 | raise wz_ex.NotImplemented(msg.format(f))
34 | fcn = reduce(getattr, f.split('.')[1:], mod)
35 | return fcn
36 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "reveal.js",
3 | "version": "2.6.2",
4 | "description": "The HTML Presentation Framework",
5 | "homepage": "http://lab.hakim.se/reveal-js",
6 | "subdomain": "revealjs",
7 | "scripts": {
8 | "test": "grunt test",
9 | "start": ""
10 | },
11 | "author": {
12 | "name": "Hakim El Hattab",
13 | "email": "hakim.elhattab@gmail.com",
14 | "web": "http://hakim.se"
15 | },
16 | "repository": {
17 | "type": "git",
18 | "url": "git://github.com/hakimel/reveal.js.git"
19 | },
20 | "engines": {
21 | "node": "~0.8.0"
22 | },
23 | "dependencies": {
24 | "underscore": "~1.5.1",
25 | "express": "~2.5.9",
26 | "mustache": "~0.7.2",
27 | "socket.io": "~0.9.13"
28 | },
29 | "devDependencies": {
30 | "grunt-contrib-qunit": "~0.2.2",
31 | "grunt-contrib-jshint": "~0.6.4",
32 | "grunt-contrib-cssmin": "~0.4.1",
33 | "grunt-contrib-uglify": "~0.2.4",
34 | "grunt-contrib-watch": "~0.5.3",
35 | "grunt-contrib-sass": "~0.5.0",
36 | "grunt-contrib-connect": "~0.4.1",
37 | "grunt-zip": "~0.7.0",
38 | "grunt": "~0.4.0"
39 | },
40 | "licenses": [
41 | {
42 | "type": "MIT",
43 | "url": "https://github.com/hakimel/reveal.js/blob/master/LICENSE"
44 | }
45 | ]
46 | }
47 |
--------------------------------------------------------------------------------
/blaze/server/tests/test_serialization.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import, division, print_function
2 |
3 | import pytest
4 | import os
5 | import json
6 | from datetime import datetime, timedelta
7 |
8 | import pandas as pd
9 | from pytz import utc
10 |
11 | from blaze.server.serialization import (json_dumps, json_dumps_trusted,
12 | object_hook, object_hook_trusted)
13 |
14 |
15 | @pytest.mark.parametrize('serializers', [(json_dumps, object_hook),
16 | (json_dumps_trusted, object_hook_trusted)])
17 | @pytest.mark.parametrize('input_,serialized', (
18 | ([1, datetime(2000, 1, 1, 12, 30, 0, 0, utc)],
19 | '[1, {"__!datetime": "2000-01-01T12:30:00+00:00"}]'),
20 | ([1, datetime(2000, 1, 1, 12, 30, 0, 0)],
21 | '[1, {"__!datetime": "2000-01-01T12:30:00"}]'),
22 | ([1, pd.NaT], '[1, {"__!datetime": "NaT"}]'),
23 | ([1, frozenset([1, 2, 3])], '[1, {"__!frozenset": [1, 2, 3]}]'),
24 | ([1, timedelta(seconds=5)], '[1, {"__!timedelta": 5.0}]'),
25 | ))
26 | def test_json_encoder(serializers, input_, serialized):
27 | serializer, deserializer = serializers
28 | result = json.dumps(input_, default=serializer)
29 | assert result == serialized
30 | assert json.loads(result, object_hook=deserializer) == input_
31 |
--------------------------------------------------------------------------------
/blaze/compute/tests/test_pydatetime.py:
--------------------------------------------------------------------------------
1 | from blaze.compute.pydatetime import truncate
2 | from datetime import datetime, date, timedelta
3 |
4 | def test_hour():
5 | dts = [datetime(2000, 6, 20, 1, 00, 00),
6 | datetime(2000, 6, 20, 12, 59, 59),
7 | datetime(2000, 6, 20, 12, 00, 00),
8 | datetime(2000, 6, 20, 11, 59, 59)]
9 |
10 | assert [truncate(dt, 1, 'hour') for dt in dts] == \
11 | [datetime(2000, 6, 20, 1, 0),
12 | datetime(2000, 6, 20, 12, 0),
13 | datetime(2000, 6, 20, 12, 0),
14 | datetime(2000, 6, 20, 11, 0)]
15 |
16 |
17 | def test_month():
18 | dts = [datetime(2000, 7, 1),
19 | datetime(2000, 6, 30),
20 | datetime(2000, 6, 1),
21 | datetime(2000, 5, 31)]
22 |
23 | assert [truncate(dt, 1, 'month') for dt in dts] == \
24 | [date(2000, 7, 1),
25 | date(2000, 6, 1),
26 | date(2000, 6, 1),
27 | date(2000, 5, 1),]
28 |
29 | assert truncate(datetime(2000, 12, 1), 1, 'month') == \
30 | date(2000, 12, 1)
31 |
32 |
33 | def test_week():
34 | d = date(2014, 11, 8)
35 | assert truncate(d, 1, 'week').isoweekday() == 7
36 | assert (d - truncate(d, 1, 'week')) < timedelta(days=7)
37 | assert (d - truncate(d, 1, 'week')) > timedelta(days=0)
38 |
--------------------------------------------------------------------------------
/blaze/server/serialization/json_dumps.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import, division, print_function
2 |
3 | import datetime
4 | from functools import partial
5 |
6 | from datashape import Mono, DataShape
7 | import pandas as pd
8 |
9 | from blaze.dispatch import dispatch
10 | from blaze.compatibility import unicode
11 |
12 |
13 | json_dumps_ns = dict()
14 | dispatch = partial(dispatch, namespace=json_dumps_ns)
15 |
16 |
17 | @dispatch(datetime.datetime)
18 | def json_dumps(dt):
19 | if dt is pd.NaT:
20 | # NaT has an isoformat but it is totally invalid.
21 | # This keeps the parsing on the client side simple.
22 | s = u'NaT'
23 | else:
24 | s = dt.isoformat()
25 | if isinstance(s, bytes):
26 | s = s.decode('utf-8')
27 |
28 | return {u'__!datetime': s}
29 |
30 |
31 | @dispatch(frozenset)
32 | def json_dumps(ds):
33 | return {u'__!frozenset': list(ds)}
34 |
35 |
36 | @dispatch(datetime.timedelta)
37 | def json_dumps(ds):
38 | return {u'__!timedelta': ds.total_seconds()}
39 |
40 |
41 | @dispatch(Mono)
42 | def json_dumps(m):
43 | return {u'__!mono': unicode(m)}
44 |
45 |
46 | @dispatch(DataShape)
47 | def json_dumps(ds):
48 | return {u'__!datashape': unicode(ds)}
49 |
50 |
51 | @dispatch(object)
52 | def json_dumps(ob):
53 | return pd.io.packers.encode(ob)
54 |
--------------------------------------------------------------------------------
/docs/README:
--------------------------------------------------------------------------------
1 | Blaze Documentation
2 | ===================
3 |
4 | The blaze documentation is split over two repositories:
5 |
6 | The user and developer documentation:
7 |
8 | https://github.com/blaze/blaze/tree/master/docs (the 'docs' directory)
9 |
10 | The webpage, blaze.pydata.org:
11 |
12 | https://github.com/blaze/blaze/tree/gh-pages
13 |
14 | User and Developer Documentation
15 | ================================
16 | The documentation is under the 'docs' directory of the blaze repo (this directory).
17 |
18 | To build the documentation, you need the basicstrap theme and sphinxjp.themecore:
19 |
20 | $ pip install sphinxjp.themes.basicstrap
21 | $ pip install numpydoc
22 |
23 | You can edit the source files under docs/source/, after which you can build and
24 | check the documentation:
25 |
26 | $ make html
27 | $ open build/html/index.html
28 |
29 | This documentation can be uploaded to http://blaze.pydata.org/en/latest/index.html
30 | using the gh-pages.py script under docs/:
31 |
32 | $ python gh-pages.py
33 |
34 | then verify the repository under the 'gh-pages' directory and use 'git push'.
35 |
36 | Webpage
37 | =======
38 | The blaze webpage on blaze.pydata.org can be fetched from here: https://github.com/blaze/blaze/tree/gh-pages
39 | The documentation is linked to in the 'dev' subdirectory.
40 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/css/theme/source/simple.scss:
--------------------------------------------------------------------------------
1 | /**
2 | * A simple theme for reveal.js presentations, similar
3 | * to the default theme. The accent color is darkblue.
4 | *
5 | * This theme is Copyright (C) 2012 Owen Versteeg, https://github.com/StereotypicalApps. It is MIT licensed.
6 | * reveal.js is Copyright (C) 2011-2012 Hakim El Hattab, http://hakim.se
7 | */
8 |
9 |
10 | // Default mixins and settings -----------------
11 | @import "../template/mixins";
12 | @import "../template/settings";
13 | // ---------------------------------------------
14 |
15 |
16 |
17 | // Include theme-specific fonts
18 | @import url(https://fonts.googleapis.com/css?family=News+Cycle:400,700);
19 | @import url(https://fonts.googleapis.com/css?family=Lato:400,700,400italic,700italic);
20 |
21 |
22 | // Override theme settings (see ../template/settings.scss)
23 | $mainFont: 'Lato', sans-serif;
24 | $mainColor: #000;
25 | $headingFont: 'News Cycle', Impact, sans-serif;
26 | $headingColor: #000;
27 | $headingTextShadow: none;
28 | $headingTextTransform: none;
29 | $backgroundColor: #fff;
30 | $linkColor: #00008B;
31 | $linkColorHover: lighten( $linkColor, 20% );
32 | $selectionBackgroundColor: rgba(0, 0, 0, 0.99);
33 |
34 |
35 |
36 | // Theme template ------------------------------
37 | @import "../template/theme";
38 | // ---------------------------------------------
--------------------------------------------------------------------------------
/docs/source/_static/presentations/markdown/parallel-options.md:
--------------------------------------------------------------------------------
1 | ### Python's options for Parallelism
2 |
3 | Explicit control -- Fast but hard
4 |
5 | * Threads/Processes/MPI/ZeroMQ
6 | * Concurrent.futures/Joblib/...
7 | * .
8 | * .
9 | * .
10 | * IPython parallel
11 | * Luigi
12 | * PySpark
13 | * Hadoop (mrjob)
14 | * SQL: Hive, Pig, Impala
15 |
16 | Implicit control -- Restrictive but easy
17 |
18 |
19 | ### Python's options for Parallelism
20 |
21 | Explicit control -- Fast but hard
22 |
23 | * Threads/Processes/MPI/ZeroMQ
24 | * Concurrent.futures/Joblib/...
25 | * .
26 | * . <-- I need this
27 | * .
28 | * IPython parallel
29 | * Luigi
30 | * PySpark
31 | * Hadoop (mrjob)
32 | * SQL: Hive, Pig, Impala
33 |
34 | Implicit control -- Restrictive but easy
35 |
36 |
37 | ### Scale
38 |
39 | * Single four-core laptop (Gigabyte scale)
40 | * Single thirty-core workstation (Terabyte scale)
41 | * Distributed thousand-core cluster (Petabyte Scale)
42 |
43 |
44 | ### Scale
45 |
46 | * **Single four-core laptop (Gigabyte scale)**
47 | * **Single thirty-core workstation (Terabyte scale)**
48 | * Distributed thousand-core cluster (Petabyte Scale)
49 |
50 |
51 | ### Upcoming Outline
52 |
53 | * Dask.array - parallel array library using dask
54 | * Dask internals - dynamic task scheduling library
55 | * Beyond arrays - How can we extend task parallelism
56 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/css/theme/source/sky.scss:
--------------------------------------------------------------------------------
1 | /**
2 | * Sky theme for reveal.js.
3 | *
4 | * Copyright (C) 2011-2012 Hakim El Hattab, http://hakim.se
5 | */
6 |
7 |
8 | // Default mixins and settings -----------------
9 | @import "../template/mixins";
10 | @import "../template/settings";
11 | // ---------------------------------------------
12 |
13 |
14 |
15 | // Include theme-specific fonts
16 | @import url(https://fonts.googleapis.com/css?family=Quicksand:400,700,400italic,700italic);
17 | @import url(https://fonts.googleapis.com/css?family=Open+Sans:400italic,700italic,400,700);
18 |
19 |
20 | // Override theme settings (see ../template/settings.scss)
21 | $mainFont: 'Open Sans', sans-serif;
22 | $mainColor: #333;
23 | $headingFont: 'Quicksand', sans-serif;
24 | $headingColor: #333;
25 | $headingLetterSpacing: -0.08em;
26 | $headingTextShadow: none;
27 | $backgroundColor: #f7fbfc;
28 | $linkColor: #3b759e;
29 | $linkColorHover: lighten( $linkColor, 20% );
30 | $selectionBackgroundColor: #134674;
31 |
32 | // Fix links so they are not cut off
33 | .reveal a:not(.image) {
34 | line-height: 1.3em;
35 | }
36 |
37 | // Background generator
38 | @mixin bodyBackground() {
39 | @include radial-gradient( #add9e4, #f7fbfc );
40 | }
41 |
42 |
43 |
44 | // Theme template ------------------------------
45 | @import "../template/theme";
46 | // ---------------------------------------------
47 |
--------------------------------------------------------------------------------
/blaze/mongo.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import, division, print_function
2 |
3 | from .compatibility import basestring
4 | from .compute.mongo import dispatch
5 |
6 | try:
7 | from pymongo.collection import Collection
8 | from pymongo import ASCENDING
9 | except ImportError:
10 | Collection = type(None)
11 |
12 |
13 | __all__ = ['create_index']
14 |
15 |
16 | @dispatch(object)
17 | def scrub_keys(o):
18 | """Add an ascending sort key when pass a string, to make the MongoDB
19 | interface similar to SQL.
20 | """
21 | raise NotImplementedError("scrub_keys not implemented for type %r" %
22 | type(o).__name__)
23 |
24 |
25 | @dispatch(basestring)
26 | def scrub_keys(s):
27 | return s, ASCENDING
28 |
29 |
30 | @dispatch(tuple)
31 | def scrub_keys(t):
32 | return t
33 |
34 |
35 | @dispatch(list)
36 | def scrub_keys(seq):
37 | for el in seq:
38 | if not isinstance(el, (tuple, basestring)):
39 | raise TypeError('indexing keys must be a string or pair of '
40 | '(18 | 19 | from dask.threaded import get 20 | get(dsk, keys) 21 | 22 | 23 | ## Cross Validation 24 | 25 | 26 |
28 |
29 |
30 |
31 | ## Cross Validation
32 |
33 | This killed the small-memory-footprint heuristics in the dask scheduler.
34 | Parallel profile/trace to get a sense of the problem.
35 |
36 | # from dask.threaded import get
37 | # get(dsk, keys)
38 |
39 | from dask.diagnostics import thread_prof
40 | thread_prof.get(dsk, keys)
41 | thread_prof.visualize()
42 |
43 | Fixing with small amounts of [static scheduling (PR)](https://github.com/blaze/dask/pull/403).
44 |
45 | [Profile Link (1.7MB)](https://rawgit.com/mrocklin/8ec0443c94da553fe00c/raw/ff7d8d0754d07f35086b08c0d21865a03b3edeac/profile.html)
46 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/images/dask.split-apply-aggregate.dot:
--------------------------------------------------------------------------------
1 | strict digraph G {
2 | "-5559165775281348128" [shape=circle, label=apply];
3 | "('f-12', 0)" [shape=box];
4 | 584190624031756547 [shape=circle, label=apply];
5 | "-7268493500755381654" [shape=circle, label=apply];
6 | 97023219449674268 [shape=circle, label=DataFrame];
7 | "('f-11', 0)" [shape=box, label="('f-11', 0)"];
8 | "('f-11', 1)" [shape=box, label="('f-11', 1)"];
9 | "('f-11', 2)" [shape=box, label="('f-11', 2)"];
10 | "-6292191960271350542" [shape=circle, label=DataFrame];
11 | "('from-array-1', 2)" [shape=box, label="('from-array-1', 2)"];
12 | "-6833178345205854086" [shape=circle, label=DataFrame];
13 | "('from-array-1', 1)" [shape=box, label="('from-array-1', 1)"];
14 | 6680825370834289340 [shape=circle, label=5 | 6 | Docs -- [http://blaze.pydata.org/](http://blaze.pydata.org/) 7 | 8 | Install -- `conda install -c blaze blaze` 9 | 10 | Source -- 11 | [http://github.com/blaze/blaze/](http://github.com/blaze/blaze) 12 | 13 | 14 | ## TL;DR 15 | 16 | Blaze is an extensible interface for data analytics. 17 | 18 | It feels like NumPy/Pandas. It drives other data systems. 19 | 20 | Blaze expressions enable high-level reasoning. 21 | 22 | 23 | 24 | ## Motivation 25 | 26 | NumPy and Pandas serve as the foundation for the PyData stack 27 | 28 | ... but they are limited to memory 29 | 30 | *The state of biggish-data analysis is still wild* 31 | 32 | 33 | ## Approach 34 | 35 | * Blaze is a user interface 36 | * ... and something like a compiler 37 | * ... for analytic processing (like Pandas, not like Redis) 38 | * It isn't a database or a Pandas replacement 39 | * It complements pre-existing systems by improving user access 40 | 41 | 42 | ## Play time 43 | 44 | *In which we pray to the demo gods* 45 | 46 |
47 | 48 | * Main points: Blaze ... 49 | * has a familiar interface 50 | * directs other mature projects to do your analysis 51 | * extends beyond Python/Pandas 52 | * Supports Pandas-like and NumPy-like computation 53 | 54 |
55 | 56 | Also, in case you missed it 57 | 58 | conda install -c blaze blaze 59 | 60 | [http://blaze.pydata.org/presentations/](http://blaze.pydata.org/presentations/) 61 | -------------------------------------------------------------------------------- /blaze/expr/method_dispatch.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | 3 | from functools import partial 4 | from multipledispatch import Dispatcher 5 | 6 | __all__ = 'select_functions', 7 | 8 | def match(condition, data): 9 | """ Does the condition match the data 10 | 11 | >>> match(1, 1) 12 | True 13 | >>> match(int, 1) 14 | True 15 | >>> match(lambda x: x > 0, 1) 16 | True 17 | 18 | Use tuples for many possibilities 19 | 20 | >>> match((1, 2, 3), 1) 21 | True 22 | >>> match((int, float), 1) 23 | True 24 | """ 25 | return ((condition == data) or 26 | (isinstance(condition, type) 27 | and isinstance(data, condition)) or 28 | (not isinstance(condition, type) 29 | and callable(condition) 30 | and condition(data)) or 31 | (isinstance(condition, tuple) 32 | and any(match(c, data) for c in condition))) 33 | 34 | 35 | def name(func): 36 | """ Name of a function, robust to partials """ 37 | if isinstance(func, partial): 38 | return name(func.func) 39 | elif isinstance(func, Dispatcher): 40 | return func.name 41 | else: 42 | return func.__name__ 43 | 44 | 45 | def select_functions(methods, data): 46 | """ 47 | Select appropriate functions given types and predicates 48 | """ 49 | s = set() 50 | for condition, funcs in methods: 51 | if match(condition, data): 52 | s |= funcs 53 | return dict((name(func), func) for func in s) 54 | -------------------------------------------------------------------------------- /conda.recipe/meta.yaml: -------------------------------------------------------------------------------- 1 | package: 2 | name: blaze 3 | version: {{ environ.get('GIT_DESCRIBE_TAG', '')}} 4 | 5 | build: 6 | number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} 7 | {% if environ.get('GIT_DESCRIBE_NUMBER', '0') == '0' %}string: py{{ environ.get('PY_VER').replace('.', '') }}_0 8 | {% else %}string: py{{ environ.get('PY_VER').replace('.', '') }}_{{ environ.get('GIT_BUILD_STR', 'GIT_STUB') }}{% endif %} 9 | entry_points: 10 | - blaze-server = blaze.server.spider:_main 11 | 12 | source: 13 | git_url: https://github.com/blaze/blaze.git 14 | 15 | requirements: 16 | build: 17 | - python 18 | - setuptools 19 | - contextlib2 # [py2k] 20 | - dask 21 | - flask >=0.10.1 22 | - flask-cors 23 | - odo >=0.5.0 24 | - datashape >=0.5.3 25 | - psutil ==4.0 26 | - sqlalchemy >=0.8.0 27 | - toolz 28 | run: 29 | - python 30 | - contextlib2 # [py2k] 31 | - cytoolz 32 | - dask 33 | - flask >=0.10.1 34 | - flask-cors 35 | - h5py 36 | - numba 37 | - odo >=0.5.0 38 | - datashape >=0.5.3 39 | - psutil ==4.0 40 | - pytables >=3.0.0 41 | - pyyaml 42 | - requests 43 | - sqlalchemy >=0.8.0 44 | 45 | test: 46 | imports: 47 | - blaze 48 | requires: 49 | - pytest >=2.7.0 50 | - psycopg2 # [unix] 51 | - pymongo >=3.0 # [unix] 52 | - bcolz # [unix] 53 | - pymysql 54 | - sqlite 55 | commands: 56 | - py.test -vx --doctest-modules --doctest-ignore-import-errors --pyargs blaze -rsxX 57 | 58 | about: 59 | home: http://blaze.pydata.org 60 | license: BSD 61 | -------------------------------------------------------------------------------- /docs/source/_static/presentations/lib/js/classList.js: -------------------------------------------------------------------------------- 1 | /*! @source http://purl.eligrey.com/github/classList.js/blob/master/classList.js*/ 2 | if(typeof document!=="undefined"&&!("classList" in document.createElement("a"))){(function(j){var a="classList",f="prototype",m=(j.HTMLElement||j.Element)[f],b=Object,k=String[f].trim||function(){return this.replace(/^\s+|\s+$/g,"")},c=Array[f].indexOf||function(q){var p=0,o=this.length;for(;p
18 |
19 |
25 |
26 |
27 |
28 |
29 |
30 |
49 |
50 |
51 |
52 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/dask-frames.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
18 |
19 |
25 |
26 |
27 |
28 |
29 |
30 |
49 |
50 |
51 |
52 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/dask-graphs.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
18 |
19 |
25 |
26 |
27 |
28 |
29 |
30 |
49 |
50 |
51 |
52 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/strata-nyc-2015.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
18 |
19 |
25 |
26 |
27 |
28 |
29 |
30 |
49 |
50 |
51 |
52 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/css/theme/source/default.scss:
--------------------------------------------------------------------------------
1 | /**
2 | * Default theme for reveal.js.
3 | *
4 | * Copyright (C) 2011-2012 Hakim El Hattab, http://hakim.se
5 | */
6 |
7 |
8 | // Default mixins and settings -----------------
9 | @import "../template/mixins";
10 | @import "../template/settings";
11 | // ---------------------------------------------
12 |
13 |
14 |
15 | // Include theme-specific fonts
16 | @font-face {
17 | font-family: 'League Gothic';
18 | src: url('../../lib/font/league_gothic-webfont.eot');
19 | src: url('../../lib/font/league_gothic-webfont.eot?#iefix') format('embedded-opentype'),
20 | url('../../lib/font/league_gothic-webfont.woff') format('woff'),
21 | url('../../lib/font/league_gothic-webfont.ttf') format('truetype'),
22 | url('../../lib/font/league_gothic-webfont.svg#LeagueGothicRegular') format('svg');
23 |
24 | font-weight: normal;
25 | font-style: normal;
26 | }
27 |
28 | @import url(https://fonts.googleapis.com/css?family=Lato:400,700,400italic,700italic);
29 |
30 | // Override theme settings (see ../template/settings.scss)
31 | $heading1TextShadow: 0 1px 0 #ccc, 0 2px 0 #c9c9c9, 0 3px 0 #bbb, 0 4px 0 #b9b9b9, 0 5px 0 #aaa, 0 6px 1px rgba(0,0,0,.1), 0 0 5px rgba(0,0,0,.1), 0 1px 3px rgba(0,0,0,.3), 0 3px 5px rgba(0,0,0,.2), 0 5px 10px rgba(0,0,0,.25), 0 20px 20px rgba(0,0,0,.15);
32 |
33 | // Background generator
34 | @mixin bodyBackground() {
35 | @include radial-gradient( rgba(28,30,32,1), rgba(85,90,95,1) );
36 | }
37 |
38 |
39 |
40 | // Theme template ------------------------------
41 | @import "../template/theme";
42 | // ---------------------------------------------
--------------------------------------------------------------------------------
/docs/source/_static/presentations/dask-databricks-2015.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
18 |
19 |
25 |
26 |
27 |
28 |
29 |
30 |
49 |
50 |
51 |
52 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/plugin/multiplex/master.js:
--------------------------------------------------------------------------------
1 | (function() {
2 | // Don't emit events from inside of notes windows
3 | if ( window.location.search.match( /receiver/gi ) ) { return; }
4 |
5 | var multiplex = Reveal.getConfig().multiplex;
6 |
7 | var socket = io.connect(multiplex.url);
8 |
9 | var notify = function( slideElement, indexh, indexv, origin ) {
10 | if( typeof origin === 'undefined' && origin !== 'remote' ) {
11 | var nextindexh;
12 | var nextindexv;
13 |
14 | var fragmentindex = Reveal.getIndices().f;
15 | if (typeof fragmentindex == 'undefined') {
16 | fragmentindex = 0;
17 | }
18 |
19 | if (slideElement.nextElementSibling && slideElement.parentNode.nodeName == 'SECTION') {
20 | nextindexh = indexh;
21 | nextindexv = indexv + 1;
22 | } else {
23 | nextindexh = indexh + 1;
24 | nextindexv = 0;
25 | }
26 |
27 | var slideData = {
28 | indexh : indexh,
29 | indexv : indexv,
30 | indexf : fragmentindex,
31 | nextindexh : nextindexh,
32 | nextindexv : nextindexv,
33 | secret: multiplex.secret,
34 | socketId : multiplex.id
35 | };
36 |
37 | socket.emit('slidechanged', slideData);
38 | }
39 | }
40 |
41 | Reveal.addEventListener( 'slidechanged', function( event ) {
42 | notify( event.currentSlide, event.indexh, event.indexv, event.origin );
43 | } );
44 |
45 | var fragmentNotify = function( event ) {
46 | notify( Reveal.getCurrentSlide(), Reveal.getIndices().h, Reveal.getIndices().v, event.origin );
47 | };
48 |
49 | Reveal.addEventListener( 'fragmentshown', fragmentNotify );
50 | Reveal.addEventListener( 'fragmenthidden', fragmentNotify );
51 | }());
--------------------------------------------------------------------------------
/docs/source/_static/presentations/plugin/remotes/remotes.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Touch-based remote controller for your presentation courtesy
3 | * of the folks at http://remotes.io
4 | */
5 |
6 | (function(window){
7 |
8 | /**
9 | * Detects if we are dealing with a touch enabled device (with some false positives)
10 | * Borrowed from modernizr: https://github.com/Modernizr/Modernizr/blob/master/feature-detects/touch.js
11 | */
12 | var hasTouch = (function(){
13 | return ('ontouchstart' in window) || window.DocumentTouch && document instanceof DocumentTouch;
14 | })();
15 |
16 | /**
17 | * Detects if notes are enable and the current page is opened inside an /iframe
18 | * this prevents loading Remotes.io several times
19 | */
20 | var isNotesAndIframe = (function(){
21 | return window.RevealNotes && !(self == top);
22 | })();
23 |
24 | if(!hasTouch && !isNotesAndIframe){
25 | head.ready( 'remotes.ne.min.js', function() {
26 | new Remotes("preview")
27 | .on("swipe-left", function(e){ Reveal.right(); })
28 | .on("swipe-right", function(e){ Reveal.left(); })
29 | .on("swipe-up", function(e){ Reveal.down(); })
30 | .on("swipe-down", function(e){ Reveal.up(); })
31 | .on("tap", function(e){ Reveal.next(); })
32 | .on("zoom-out", function(e){ Reveal.toggleOverview(true); })
33 | .on("zoom-in", function(e){ Reveal.toggleOverview(false); })
34 | ;
35 | } );
36 |
37 | head.js('https://hakim-static.s3.amazonaws.com/reveal-js/remotes.ne.min.js');
38 | }
39 | })(window);
--------------------------------------------------------------------------------
/blaze/compute/tests/test_literal_compute.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pytest
3 |
4 | from datashape import dshape
5 | from odo import into
6 |
7 | from blaze.compute import compute
8 | from blaze.expr import data, literal, symbol
9 |
10 |
11 | tdata = (('Alice', 100),
12 | ('Bob', 200))
13 |
14 | L = [[1, 'Alice', 100],
15 | [2, 'Bob', -200],
16 | [3, 'Charlie', 300],
17 | [4, 'Denis', 400],
18 | [5, 'Edith', -500]]
19 |
20 | l = literal(tdata)
21 | nl = literal(tdata, name='amounts')
22 | t = data(tdata, fields=['name', 'amount'])
23 |
24 |
25 | def test_resources_fail():
26 | t = symbol('t', 'var * {x: int, y: int}')
27 | d = t[t['x'] > 100]
28 | with pytest.raises(ValueError):
29 | compute(d)
30 |
31 |
32 | def test_compute_on_Data_gives_back_data():
33 | assert compute(data([1, 2, 3])) == [1, 2, 3]
34 |
35 |
36 | def test_compute_on_literal_gives_back_data():
37 | assert compute(literal([1, 2, 3])) == [1, 2, 3]
38 |
39 |
40 | def test_compute():
41 | assert list(compute(t['amount'] + 1)) == [101, 201]
42 |
43 |
44 | def test_create_with_schema():
45 | t = data(tdata, schema='{name: string, amount: float32}')
46 | assert t.schema == dshape('{name: string, amount: float32}')
47 |
48 |
49 | def test_create_with_raw_data():
50 | t = data(tdata, fields=['name', 'amount'])
51 | assert t.schema == dshape('{name: string, amount: int64}')
52 | assert t.name
53 | assert t.data == tdata
54 |
55 |
56 | def test_iter():
57 | x = np.ones(4)
58 | d = data(x)
59 | assert list(d + 1) == [2, 2, 2, 2]
60 |
61 |
62 | def test_into():
63 | assert into(list, t) == into(list, tdata)
64 |
--------------------------------------------------------------------------------
/blaze/compute/tests/test_hdfstore.py:
--------------------------------------------------------------------------------
1 | import pytest
2 | tables = pytest.importorskip('tables')
3 |
4 | from blaze.utils import tmpfile
5 | from blaze import symbol, discover, compute, pre_compute
6 | import pandas as pd
7 | from datetime import datetime
8 | from odo import Chunks, odo
9 |
10 |
11 | @pytest.fixture
12 | def df():
13 | return pd.DataFrame([['a', 1, 10., datetime(2000, 1, 1)],
14 | ['ab', 2, 20., datetime(2000, 2, 2)],
15 | ['abc', 3, 30., datetime(2000, 3, 3)],
16 | ['abcd', 4, 40., datetime(2000, 4, 4)]],
17 | columns=['name', 'a', 'b', 'time'])
18 |
19 |
20 | @pytest.fixture
21 | def s(hdf):
22 | return symbol('s', discover(hdf))
23 |
24 |
25 | @pytest.yield_fixture(params=['fixed', 'table'])
26 | def hdf(df, request):
27 | with tmpfile('.hdf5') as fn:
28 | df.to_hdf(fn, '/data', format=request.param)
29 | df.to_hdf(fn, '/nested/data', format=request.param)
30 | with pd.HDFStore(fn, mode='r') as r:
31 | yield r
32 |
33 |
34 | def test_basic_compute(hdf, s):
35 | result = compute(s.data, hdf)
36 | types = (
37 | pd.DataFrame,
38 | pd.io.pytables.Fixed,
39 | pd.io.pytables.AppendableFrameTable,
40 | Chunks
41 | )
42 | assert isinstance(result, types)
43 |
44 |
45 | def test_pre_compute(hdf, s):
46 | result = pre_compute(s, hdf.get_storer('data'))
47 | assert isinstance(result, (pd.DataFrame, Chunks))
48 |
49 |
50 | def test_groups(hdf, df, s):
51 | assert discover(hdf) == discover(dict(data=df, nested=dict(data=df)))
52 | assert odo(compute(s.nested.data.a, hdf), list) == [1, 2, 3, 4]
53 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/dask.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
18 |
19 |
26 |
27 |
28 |
29 |
30 |
31 |
50 |
51 |
52 |
53 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/into.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
18 |
19 |
26 |
27 |
28 |
29 |
30 |
31 |
50 |
51 |
52 |
53 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/blaze.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
18 |
19 |
27 |
28 |
29 |
30 |
31 |
32 |
51 |
52 |
53 |
54 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/css/theme/template/mixins.scss:
--------------------------------------------------------------------------------
1 | @mixin vertical-gradient( $top, $bottom ) {
2 | background: $top;
3 | background: -moz-linear-gradient( top, $top 0%, $bottom 100% );
4 | background: -webkit-gradient( linear, left top, left bottom, color-stop(0%,$top), color-stop(100%,$bottom) );
5 | background: -webkit-linear-gradient( top, $top 0%, $bottom 100% );
6 | background: -o-linear-gradient( top, $top 0%, $bottom 100% );
7 | background: -ms-linear-gradient( top, $top 0%, $bottom 100% );
8 | background: linear-gradient( top, $top 0%, $bottom 100% );
9 | }
10 |
11 | @mixin horizontal-gradient( $top, $bottom ) {
12 | background: $top;
13 | background: -moz-linear-gradient( left, $top 0%, $bottom 100% );
14 | background: -webkit-gradient( linear, left top, right top, color-stop(0%,$top), color-stop(100%,$bottom) );
15 | background: -webkit-linear-gradient( left, $top 0%, $bottom 100% );
16 | background: -o-linear-gradient( left, $top 0%, $bottom 100% );
17 | background: -ms-linear-gradient( left, $top 0%, $bottom 100% );
18 | background: linear-gradient( left, $top 0%, $bottom 100% );
19 | }
20 |
21 | @mixin radial-gradient( $outer, $inner, $type: circle ) {
22 | background: $outer;
23 | background: -moz-radial-gradient( center, $type cover, $inner 0%, $outer 100% );
24 | background: -webkit-gradient( radial, center center, 0px, center center, 100%, color-stop(0%,$inner), color-stop(100%,$outer) );
25 | background: -webkit-radial-gradient( center, $type cover, $inner 0%, $outer 100% );
26 | background: -o-radial-gradient( center, $type cover, $inner 0%, $outer 100% );
27 | background: -ms-radial-gradient( center, $type cover, $inner 0%, $outer 100% );
28 | background: radial-gradient( center, $type cover, $inner 0%, $outer 100% );
29 | }
--------------------------------------------------------------------------------
/docs/source/_static/presentations/ucar-sea-2015.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
18 |
19 |
28 |
29 |
30 |
31 |
32 |
33 |
52 |
53 |
54 |
55 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/plugin/multiplex/index.js:
--------------------------------------------------------------------------------
1 | var express = require('express');
2 | var fs = require('fs');
3 | var io = require('socket.io');
4 | var crypto = require('crypto');
5 |
6 | var app = express.createServer();
7 | var staticDir = express.static;
8 |
9 | io = io.listen(app);
10 |
11 | var opts = {
12 | port: 1948,
13 | baseDir : __dirname + '/../../'
14 | };
15 |
16 | io.sockets.on('connection', function(socket) {
17 | socket.on('slidechanged', function(slideData) {
18 | if (typeof slideData.secret == 'undefined' || slideData.secret == null || slideData.secret === '') return;
19 | if (createHash(slideData.secret) === slideData.socketId) {
20 | slideData.secret = null;
21 | socket.broadcast.emit(slideData.socketId, slideData);
22 | };
23 | });
24 | });
25 |
26 | app.configure(function() {
27 | [ 'css', 'js', 'plugin', 'lib' ].forEach(function(dir) {
28 | app.use('/' + dir, staticDir(opts.baseDir + dir));
29 | });
30 | });
31 |
32 | app.get("/", function(req, res) {
33 | res.writeHead(200, {'Content-Type': 'text/html'});
34 | fs.createReadStream(opts.baseDir + '/index.html').pipe(res);
35 | });
36 |
37 | app.get("/token", function(req,res) {
38 | var ts = new Date().getTime();
39 | var rand = Math.floor(Math.random()*9999999);
40 | var secret = ts.toString() + rand.toString();
41 | res.send({secret: secret, socketId: createHash(secret)});
42 | });
43 |
44 | var createHash = function(secret) {
45 | var cipher = crypto.createCipher('blowfish', secret);
46 | return(cipher.final('hex'));
47 | };
48 |
49 | // Actually listen
50 | app.listen(opts.port || null);
51 |
52 | var brown = '\033[33m',
53 | green = '\033[32m',
54 | reset = '\033[0m';
55 |
56 | console.log( brown + "reveal.js:" + reset + " Multiplex running on port " + green + opts.port + reset );
--------------------------------------------------------------------------------
/blaze/tests/test_types.py:
--------------------------------------------------------------------------------
1 | import dask.array as da
2 | import numpy as np
3 | import pandas as pd
4 | import pytest
5 |
6 | from blaze.compute.core import into
7 | from blaze.types import (
8 | iscorescalar,
9 | iscoresequence,
10 | iscoretype,
11 | )
12 |
13 |
14 | @pytest.mark.parametrize('data,res',
15 | [(1, True),
16 | (1.1, True),
17 | ("foo", True),
18 | ([1, 2], False),
19 | ((1, 2), False),
20 | (pd.Series([1, 2]), False)])
21 | def test_iscorescalar(data, res):
22 | assert iscorescalar(data) == res
23 |
24 |
25 | @pytest.mark.parametrize('data,res',
26 | [(1, False),
27 | ("foo", False),
28 | ([1, 2], True),
29 | ((1, 2), True),
30 | (pd.Series([1, 2]), True),
31 | (pd.DataFrame([[1, 2], [3, 4]]), True),
32 | (np.ndarray([1, 2]), True),
33 | (into(da.core.Array, [1, 2], chunks=(10,)), False)])
34 | def test_iscoresequence(data, res):
35 | assert iscoresequence(data) == res
36 |
37 |
38 | @pytest.mark.parametrize('data,res',
39 | [(1, True),
40 | ("foo", True),
41 | ([1, 2], True),
42 | ((1, 2), True),
43 | (pd.Series([1, 2]), True),
44 | (pd.DataFrame([[1, 2], [3, 4]]), True),
45 | (np.ndarray([1, 2]), True),
46 | (into(da.core.Array, [1, 2], chunks=(10,)), False)])
47 | def test_iscoretype(data, res):
48 | assert iscoretype(data) == res
49 |
--------------------------------------------------------------------------------
/blaze/index.py:
--------------------------------------------------------------------------------
1 | from .dispatch import dispatch
2 | from .compatibility import basestring
3 | from blaze.expr.literal import BoundSymbol, data as bz_data
4 |
5 |
6 | @dispatch(object, (basestring, list, tuple))
7 | def create_index(t, column_name_or_names, name=None):
8 | """Create an index on a column.
9 |
10 | Parameters
11 | ----------
12 | o : table-like
13 | index_name : str
14 | The name of the index to create
15 | column_name_or_names : string, list, tuple
16 | A column name to index on, or a list or tuple for a composite index
17 |
18 | Examples
19 | --------
20 | >>> # Using SQLite
21 | >>> from blaze import SQL
22 | >>> # create a table called 'tb', in memory
23 | >>> sql = SQL('sqlite:///:memory:', 'tb',
24 | ... schema='{id: int64, value: float64, categ: string}')
25 | >>> dta = [(1, 2.0, 'a'), (2, 3.0, 'b'), (3, 4.0, 'c')]
26 | >>> sql.extend(dta)
27 | >>> # create an index on the 'id' column (for SQL we must provide a name)
28 | >>> sql.table.indexes
29 | set()
30 | >>> create_index(sql, 'id', name='id_index')
31 | >>> sql.table.indexes
32 | {Index('id_index', Column('id', BigInteger(), table=
49 |
50 |
51 | ### Ghosting (shared boundaries)
52 |
53 | >>> x = da.ones(100, chunks=(10,))
54 | >>> g = da.ghost.ghost(x, depth={0: 2}, boundary={0: np.nan})
55 |
56 | 
57 |
58 |
59 |
60 | ### Two Dimensional Algorithms
61 |
62 | >>> x = da.ones((15, 15), chunks=(5, 5))
63 |
64 |
65 | ### Partial Reductions
66 |
67 | >>> x.mean(axis=0)
68 |
69 | 
70 |
71 |
72 | ### Transpose
73 |
74 | >>> x + x.T
75 |
76 | 
77 |
78 |
79 | ### Matrix Multiply (index contraction)
80 |
81 | >>> x.dot(x.T)
82 |
83 | 
84 |
85 |
86 | ### Compound ad naseum
87 |
88 | >>> x.dot(x.T + 1) - x.mean(axis=1)
89 |
90 | 
91 |
92 | We can compound these operations forever. Constructing larger and larger
93 | graphs before we hand off the work to a scheduler to execute.
94 |
--------------------------------------------------------------------------------
/blaze/expr/tests/test_slicing.py:
--------------------------------------------------------------------------------
1 | from blaze.expr import symbol
2 | import numpy as np
3 | from datashape import dshape, isscalar
4 |
5 |
6 | def test_array_dshape():
7 | x = symbol('x', '5 * 3 * float32')
8 | assert x.shape == (5, 3)
9 | assert x.schema == dshape('float32')
10 | assert len(x) == 5
11 | assert x.ndim == 2
12 |
13 |
14 | def test_element():
15 | x = symbol('x', '5 * 3 * float32')
16 | assert isscalar(x[1, 2].dshape)
17 | assert x[1, 2].dshape == dshape('float32')
18 |
19 | assert str(x[1, 2]) == 'x[1, 2]'
20 |
21 | x = symbol('x', '5 * float32')
22 | assert isscalar(x[3].dshape)
23 |
24 |
25 | def test_slice():
26 | x = symbol('x', '5 * 3 * {name: string, amount: float32}')
27 | assert x[2:, 0].dshape == dshape('3 * {name: string, amount: float32}')
28 |
29 | assert x[2:].dshape == x[2:, :].dshape
30 |
31 | # Make sure that these are hashable
32 | hash(x[:2])
33 | hash(x[0, :2])
34 |
35 | assert str(x[1]) == 'x[1]'
36 | assert str(x[:2]) == 'x[:2]'
37 | assert str(x[0, :2]) == 'x[0, :2]'
38 | assert str(x[1:4:2, :2]) == 'x[1:4:2, :2]'
39 |
40 |
41 | def test_negative_slice():
42 | x = symbol('x', '10 * 10 * int32')
43 | assert x[:5, -3:].shape == (5, 3)
44 |
45 |
46 | def test_None_slice():
47 | x = symbol('x', '10 * 10 * int32')
48 | assert x[:5, None, -3:].shape == (5, 1, 3)
49 |
50 |
51 | def test_list_slice():
52 | x = symbol('x', '10 * 10 * int32')
53 | assert x[[1, 2, 3], [4, 5]].shape == (3, 2)
54 |
55 |
56 | def test_list_slice_string():
57 | x = symbol('x', '10 * 10 * int32')
58 | assert str(x[[1, 2, 3]]) == "x[[1, 2, 3]]"
59 |
60 |
61 | def test_slice_with_boolean_list():
62 | x = symbol('x', '5 * int32')
63 | expr = x[[True, False, False, True, False]]
64 | assert expr.index == ([0, 3],)
65 |
66 |
67 | def test_slice_with_numpy_array():
68 | x = symbol('x', '2 * int32')
69 | assert x[np.array([True, False])].isidentical(x[[True, False]])
70 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/css/theme/source/moon.scss:
--------------------------------------------------------------------------------
1 | /**
2 | * Solarized Dark theme for reveal.js.
3 | * Author: Achim Staebler
4 | */
5 |
6 |
7 | // Default mixins and settings -----------------
8 | @import "../template/mixins";
9 | @import "../template/settings";
10 | // ---------------------------------------------
11 |
12 |
13 |
14 | // Include theme-specific fonts
15 | @font-face {
16 | font-family: 'League Gothic';
17 | src: url('../../lib/font/league_gothic-webfont.eot');
18 | src: url('../../lib/font/league_gothic-webfont.eot?#iefix') format('embedded-opentype'),
19 | url('../../lib/font/league_gothic-webfont.woff') format('woff'),
20 | url('../../lib/font/league_gothic-webfont.ttf') format('truetype'),
21 | url('../../lib/font/league_gothic-webfont.svg#LeagueGothicRegular') format('svg');
22 |
23 | font-weight: normal;
24 | font-style: normal;
25 | }
26 |
27 | @import url(https://fonts.googleapis.com/css?family=Lato:400,700,400italic,700italic);
28 |
29 | /**
30 | * Solarized colors by Ethan Schoonover
31 | */
32 | html * {
33 | color-profile: sRGB;
34 | rendering-intent: auto;
35 | }
36 |
37 | // Solarized colors
38 | $base03: #002b36;
39 | $base02: #073642;
40 | $base01: #586e75;
41 | $base00: #657b83;
42 | $base0: #839496;
43 | $base1: #93a1a1;
44 | $base2: #eee8d5;
45 | $base3: #fdf6e3;
46 | $yellow: #b58900;
47 | $orange: #cb4b16;
48 | $red: #dc322f;
49 | $magenta: #d33682;
50 | $violet: #6c71c4;
51 | $blue: #268bd2;
52 | $cyan: #2aa198;
53 | $green: #859900;
54 |
55 | // Override theme settings (see ../template/settings.scss)
56 | $mainColor: $base1;
57 | $headingColor: $base2;
58 | $headingTextShadow: none;
59 | $backgroundColor: $base03;
60 | $linkColor: $blue;
61 | $linkColorHover: lighten( $linkColor, 20% );
62 | $selectionBackgroundColor: $magenta;
63 |
64 |
65 |
66 | // Theme template ------------------------------
67 | @import "../template/theme";
68 | // ---------------------------------------------
69 |
--------------------------------------------------------------------------------
/blaze/compute/tests/test_numba.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | nb = pytest.importorskip('numba')
4 |
5 | from numba import float64, int64, float32
6 | from numba.types import NPDatetime as datetime64, NPTimedelta as timedelta64
7 | from numba.types import CharSeq as char, UnicodeCharSeq as unichar
8 | from blaze.compute.numba import compute_signature, get_numba_type
9 | from blaze import symbol
10 | import datashape
11 |
12 |
13 | def test_compute_signature():
14 | s = symbol('s', 'int64')
15 | t = symbol('t', 'float32')
16 | d = symbol('d', 'datetime')
17 |
18 | assert compute_signature(s + t) == float64(int64, float32)
19 | assert (compute_signature(d.truncate(days=1)) ==
20 | datetime64('D')(datetime64('us')))
21 | assert compute_signature(d.day + 1) == int64(datetime64('us'))
22 |
23 |
24 | def test_get_numba_type():
25 | assert get_numba_type(datashape.bool_) == nb.bool_
26 | assert get_numba_type(datashape.date_) == datetime64('D')
27 | assert get_numba_type(datashape.datetime_) == datetime64('us')
28 | assert get_numba_type(datashape.timedelta_) == timedelta64('us')
29 | assert get_numba_type(datashape.TimeDelta('D')) == timedelta64('D')
30 | assert get_numba_type(datashape.int64) == int64
31 | assert get_numba_type(datashape.String(7, "A")) == char(7)
32 | assert get_numba_type(datashape.String(None, "A")) == nb.types.string
33 | assert get_numba_type(datashape.String(7)) == unichar(7)
34 |
35 |
36 | def test_fail_on_object_type():
37 | with pytest.raises(TypeError):
38 | get_numba_type(datashape.object_)
39 |
40 |
41 | @pytest.mark.xfail(raises=TypeError,
42 | reason="Cannot infer variable length string type yet")
43 | def test_get_numba_type_failures():
44 | get_numba_type(datashape.string)
45 |
46 |
47 | @pytest.mark.xfail(raises=TypeError,
48 | reason='Cannot infer type of record dshapes yet')
49 | def test_get_record_type():
50 | get_numba_type(datashape.dshape('10 * {a: int64}'))
51 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/plugin/notes-server/index.js:
--------------------------------------------------------------------------------
1 | var express = require('express');
2 | var fs = require('fs');
3 | var io = require('socket.io');
4 | var _ = require('underscore');
5 | var Mustache = require('mustache');
6 |
7 | var app = express.createServer();
8 | var staticDir = express.static;
9 |
10 | io = io.listen(app);
11 |
12 | var opts = {
13 | port : 1947,
14 | baseDir : __dirname + '/../../'
15 | };
16 |
17 | io.sockets.on('connection', function(socket) {
18 | socket.on('slidechanged', function(slideData) {
19 | socket.broadcast.emit('slidedata', slideData);
20 | });
21 | socket.on('fragmentchanged', function(fragmentData) {
22 | socket.broadcast.emit('fragmentdata', fragmentData);
23 | });
24 | });
25 |
26 | app.configure(function() {
27 | [ 'css', 'js', 'images', 'plugin', 'lib' ].forEach(function(dir) {
28 | app.use('/' + dir, staticDir(opts.baseDir + dir));
29 | });
30 | });
31 |
32 | app.get("/", function(req, res) {
33 | res.writeHead(200, {'Content-Type': 'text/html'});
34 | fs.createReadStream(opts.baseDir + '/index.html').pipe(res);
35 | });
36 |
37 | app.get("/notes/:socketId", function(req, res) {
38 |
39 | fs.readFile(opts.baseDir + 'plugin/notes-server/notes.html', function(err, data) {
40 | res.send(Mustache.to_html(data.toString(), {
41 | socketId : req.params.socketId
42 | }));
43 | });
44 | // fs.createReadStream(opts.baseDir + 'notes-server/notes.html').pipe(res);
45 | });
46 |
47 | // Actually listen
48 | app.listen(opts.port || null);
49 |
50 | var brown = '\033[33m',
51 | green = '\033[32m',
52 | reset = '\033[0m';
53 |
54 | var slidesLocation = "http://localhost" + ( opts.port ? ( ':' + opts.port ) : '' );
55 |
56 | console.log( brown + "reveal.js - Speaker Notes" + reset );
57 | console.log( "1. Open the slides at " + green + slidesLocation + reset );
58 | console.log( "2. Click on the link your JS console to go to the notes page" );
59 | console.log( "3. Advance through your slides and your notes will advance automatically" );
60 |
--------------------------------------------------------------------------------
/blaze/compute/chunks.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import, division, print_function
2 |
3 | from multipledispatch import MDNotImplementedError
4 | from odo import Chunks, convert, into
5 | from collections import Iterator, Iterable
6 | from toolz import curry, concat
7 | from datashape.dispatch import dispatch
8 |
9 | import pandas as pd
10 | import numpy as np
11 |
12 | from ..expr import Head, ElemWise, Distinct, Symbol, Expr, path
13 | from ..expr.split import split
14 | from .core import compute
15 | from .pmap import get_default_pmap
16 |
17 |
18 | __all__ = ['Cheap', 'compute_chunk', 'compute_down']
19 |
20 |
21 | Cheap = (Head, ElemWise, Distinct, Symbol)
22 |
23 |
24 | @dispatch(Head, Chunks)
25 | def pre_compute(expr, data, **kwargs):
26 | leaf = expr._leaves()[0]
27 | if all(isinstance(e, Cheap) for e in path(expr, leaf)):
28 | return convert(Iterator, data)
29 | else:
30 | raise MDNotImplementedError()
31 |
32 |
33 | def compute_chunk(chunk, chunk_expr, part):
34 | return compute(chunk_expr, {chunk: part})
35 |
36 |
37 | @dispatch(Expr, Chunks)
38 | def compute_down(expr, data, map=None, **kwargs):
39 | if map is None:
40 | map = get_default_pmap()
41 |
42 | leaf = expr._leaves()[0]
43 |
44 | (chunk, chunk_expr), (agg, agg_expr) = split(leaf, expr)
45 |
46 | parts = list(map(curry(compute_chunk, chunk, chunk_expr), data))
47 |
48 | if isinstance(parts[0], np.ndarray):
49 | intermediate = np.concatenate(parts)
50 | elif isinstance(parts[0], pd.DataFrame):
51 | intermediate = pd.concat(parts)
52 | elif isinstance(parts[0], (Iterable, Iterator)):
53 | intermediate = list(concat(parts))
54 |
55 | return compute(agg_expr, {agg: intermediate})
56 |
57 |
58 | Cheap = (Head, ElemWise, Distinct, Symbol)
59 |
60 | @dispatch(Head, Chunks)
61 | def compute_down(expr, data, **kwargs):
62 | leaf = expr._leaves()[0]
63 | if all(isinstance(e, Cheap) for e in path(expr, leaf)):
64 | return compute(expr, {leaf: into(Iterator, data)}, **kwargs)
65 | else:
66 | raise MDNotImplementedError()
67 |
--------------------------------------------------------------------------------
/blaze/server/tests/test_spider.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import os
3 | import json
4 |
5 | import pytest
6 |
7 | import h5py
8 | import numpy as np
9 | import pandas as pd
10 |
11 | from datashape import dshape
12 |
13 | from blaze import data_spider, discover
14 |
15 |
16 | @pytest.fixture
17 | def data(tmpdir):
18 | csvf = tmpdir.join('foo.csv')
19 | csvf.write('a,b\n1,2\n3,4')
20 | h5f = tmpdir.join('foo.hdf5')
21 | data = np.random.randn(10, 2)
22 | with h5py.File(str(h5f)) as f:
23 | f.create_dataset(name='fooh5', shape=data.shape,
24 | dtype=data.dtype, data=data)
25 | jsonf = tmpdir.mkdir('sub').join('foo.json')
26 | jsonf.write(json.dumps([{'a': 2,
27 | 'b': 3.14,
28 | 'c': str(pd.Timestamp('now'))},
29 | {'a': 2,
30 | 'b': 4.2,
31 | 'c': None,
32 | 'd': 'foobar'}]))
33 | return tmpdir
34 |
35 |
36 | @pytest.fixture
37 | def data_with_cycle(data):
38 | data.join('cycle').mksymlinkto(data)
39 | return data
40 |
41 |
42 | def test_data_spider(data):
43 | result = data_spider(str(data))
44 | ss = """{
45 | %r: {
46 | 'foo.csv': var * {a: int64, b: int64},
47 | 'foo.hdf5': {fooh5: 10 * 2 * float64},
48 | sub: {'foo.json': 2 * {a: int64, b: float64, c: ?datetime, d: ?string}}
49 | }
50 | }""" % os.path.basename(str(data))
51 | assert dshape(discover(result)) == dshape(ss)
52 |
53 |
54 | @pytest.mark.skipif(sys.platform == 'win32',
55 | reason='Windows does not have symlinks')
56 | def test_data_spider_cycle(data_with_cycle):
57 | result = data_spider(str(data_with_cycle), followlinks=True)
58 | ss = """{
59 | %r: {
60 | 'foo.csv': var * {a: int64, b: int64},
61 | 'foo.hdf5': {fooh5: 10 * 2 * float64},
62 | sub: {'foo.json': 2 * {a: int64, b: float64, c: ?datetime, d: ?string}}
63 | }
64 | }""" % os.path.basename(str(data_with_cycle))
65 | assert dshape(discover(result)) != dshape(ss)
66 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/css/theme/source/solarized.scss:
--------------------------------------------------------------------------------
1 | /**
2 | * Solarized Light theme for reveal.js.
3 | * Author: Achim Staebler
4 | */
5 |
6 |
7 | // Default mixins and settings -----------------
8 | @import "../template/mixins";
9 | @import "../template/settings";
10 | // ---------------------------------------------
11 |
12 |
13 |
14 | // Include theme-specific fonts
15 | @font-face {
16 | font-family: 'League Gothic';
17 | src: url('../../lib/font/league_gothic-webfont.eot');
18 | src: url('../../lib/font/league_gothic-webfont.eot?#iefix') format('embedded-opentype'),
19 | url('../../lib/font/league_gothic-webfont.woff') format('woff'),
20 | url('../../lib/font/league_gothic-webfont.ttf') format('truetype'),
21 | url('../../lib/font/league_gothic-webfont.svg#LeagueGothicRegular') format('svg');
22 |
23 | font-weight: normal;
24 | font-style: normal;
25 | }
26 |
27 | @import url(https://fonts.googleapis.com/css?family=Lato:400,700,400italic,700italic);
28 |
29 |
30 | /**
31 | * Solarized colors by Ethan Schoonover
32 | */
33 | html * {
34 | color-profile: sRGB;
35 | rendering-intent: auto;
36 | }
37 |
38 | // Solarized colors
39 | $base03: #002b36;
40 | $base02: #073642;
41 | $base01: #586e75;
42 | $base00: #657b83;
43 | $base0: #839496;
44 | $base1: #93a1a1;
45 | $base2: #eee8d5;
46 | $base3: #fdf6e3;
47 | $yellow: #b58900;
48 | $orange: #cb4b16;
49 | $red: #dc322f;
50 | $magenta: #d33682;
51 | $violet: #6c71c4;
52 | $blue: #268bd2;
53 | $cyan: #2aa198;
54 | $green: #859900;
55 |
56 | // Override theme settings (see ../template/settings.scss)
57 | $mainColor: $base00;
58 | $headingColor: $base01;
59 | $headingTextShadow: none;
60 | $backgroundColor: $base3;
61 | $linkColor: $blue;
62 | $linkColorHover: lighten( $linkColor, 20% );
63 | $selectionBackgroundColor: $magenta;
64 |
65 | // Background generator
66 | // @mixin bodyBackground() {
67 | // @include radial-gradient( rgba($base3,1), rgba(lighten($base3, 20%),1) );
68 | // }
69 |
70 |
71 |
72 | // Theme template ------------------------------
73 | @import "../template/theme";
74 | // ---------------------------------------------
75 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/markdown/pydata-berlin-fin.md:
--------------------------------------------------------------------------------
1 | ### Scikit Image Case Study
2 |
3 | * Setup
4 | * Scikit image has sophisticated single-threaded algorithms
5 | * Dask.array parallelizes map on slighty overlapping blocks
6 |
7 | 
8 |
9 | * Timeline
10 | * [Blake Griffith](http://github.com/cowlicks/)
11 | creates
12 | [parallel apply function](https://github.com/scikit-image/scikit-image/pull/1493)
13 | in scikit-image (1 week part time)
14 | * People try it out; it's not much faster
15 | * [Johannes Schönberger](http://www.cs.unc.edu/~jsch/) [releases the GIL](https://github.com/scikit-image/scikit-image/pull/1519/files) (few days)
16 | * Scikit image + dask.array sees
17 | [2x-3x speedups](https://github.com/blaze/dask/blob/master/notebooks/parallelize_image_filtering_workload.ipynb)
18 | over Scikit image alone (experiments by [@arve0](http://arve0.github.io/))
19 |
20 |
21 | ### Momentum
22 |
23 | * Jeff Reback has [a nogil Pandas branch](https://github.com/pydata/pandas/pull/10199)
24 |
25 | This morning: *I updated this. works for all groupbys now.*
26 |
27 | * [Bottleneck issue](https://github.com/kwgoodman/bottleneck)
28 |
29 |
30 | ### Final thoughts
31 |
32 | [http://dask.pydata.org](http://dask.pydata.org)
33 |
34 | * Most data is small (*you should ignore this talk*)
35 | * PyData has room to grow in parallelism (GIL is not an issue)
36 | * Dask.array -- a multi-core on-disk numpy clone
37 | * Dask.core -- an option for parallelism
38 |
39 |
40 |
41 |
42 | ### Finally: Parallelism is rarely important
43 |
44 | * Most data is small
45 | * For moderate data, think about storage and representation
46 | * Pandas categoricals are possibly the biggest improvement to PyData performance in
47 | the last year
48 |
49 |
50 | ### Ignore everything I just said
51 |
52 | * Most data is small
53 | * For moderate data, think about storage and representation
54 | * Pandas categoricals are possibly the biggest improvement to PyData performance in
55 | the last year
56 |
57 |
58 | ### Questions?
59 |
60 | [http://dask.pydata.org](http://dask.pydata.org)
61 |
62 | 
63 |
64 | 
65 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/sfpython-aug-2014.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
18 |
19 |
39 |
40 |
41 |
42 |
43 |
44 |
63 |
64 |
65 |
66 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/pydata-nyc-2014.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
18 |
19 |
42 |
43 |
44 |
45 |
46 |
47 |
66 |
67 |
68 |
69 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/lib/css/zenburn.css:
--------------------------------------------------------------------------------
1 | /*
2 |
3 | Zenburn style from voldmar.ru (c) Vladimir Epifanov
18 |
19 |
40 |
41 |
42 |
43 |
44 |
45 |
64 |
65 |
66 |
67 |
--------------------------------------------------------------------------------
/blaze/sql.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import, division, print_function
2 |
3 | import sqlalchemy as sa
4 |
5 | from .compatibility import basestring
6 | from .dispatch import dispatch
7 |
8 | __all__ = ()
9 |
10 |
11 | @dispatch(sa.Table, basestring)
12 | def create_index(s, column, **kwargs):
13 | """Create an index for a single column.
14 |
15 | Parameters
16 | ----------
17 | s : sa.Table
18 | The table to create the index on.
19 | column : str
20 | The name of the column to create an index on.
21 | name : str
22 | The name of the created index.
23 | unique : bool, optional
24 | Should the index be unique.
25 | ignore_existing : bool, optional
26 | Should this supress exceptions from the index already existing.
27 | concurrently : bool, optional
28 | Should the index be created without holding a lock. This feature is
29 | postgres specific.
30 | """
31 | return create_index(s, (column,), **kwargs)
32 |
33 |
34 | @dispatch(sa.Table, (list, tuple))
35 | def create_index(s,
36 | columns,
37 | name=None,
38 | unique=False,
39 | ignore_existing=False,
40 | concurrently=False):
41 | """Create an index for a single column.
42 |
43 | Parameters
44 | ----------
45 | s : sa.Table
46 | The table to create the index on.
47 | columns : list[str] or tuple[str]
48 | The names of the columns to create an index on.
49 | name : str
50 | The name of the created index.
51 | unique : bool, optional
52 | Should the index be unique.
53 | ignore_existing : bool, optional
54 | Should this supress exceptions from the index already existing.
55 | concurrently : bool, optional
56 | Should the index be created without holding a lock. This feature is
57 | postgres specific.
58 | """
59 |
60 | if name is None:
61 | raise ValueError('SQL indexes must have a name')
62 | try:
63 | sa.Index(
64 | name,
65 | *(s.c[c] for c in columns),
66 | unique=unique,
67 | **{'postgresql_concurrently': concurrently} if concurrently else {}
68 | ).create(s.bind)
69 | except (sa.exc.ProgrammingError, sa.exc.OperationalError):
70 | # sqlite raises OperationalError but postgres raises ProgrammingError
71 | # thanks fam
72 | if not ignore_existing:
73 | raise
74 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/css/theme/source/blood.scss:
--------------------------------------------------------------------------------
1 | /**
2 | * Blood theme for reveal.js
3 | * Author: Walther http://github.com/Walther
4 | *
5 | * Designed to be used with highlight.js theme
6 | * "monokai_sublime.css" available from
7 | * https://github.com/isagalaev/highlight.js/
8 | *
9 | * For other themes, change $codeBackground accordingly.
10 | *
11 | */
12 |
13 | // Default mixins and settings -----------------
14 | @import "../template/mixins";
15 | @import "../template/settings";
16 | // ---------------------------------------------
17 |
18 | // Include theme-specific fonts
19 |
20 | @import url(https://fonts.googleapis.com/css?family=Ubuntu:300,700,300italic,700italic);
21 |
22 | // Colors used in the theme
23 | $blood: #a23;
24 | $coal: #222;
25 | $codeBackground: #23241f;
26 |
27 | // Main text
28 | $mainFont: Ubuntu, 'sans-serif';
29 | $mainFontSize: 36px;
30 | $mainColor: #eee;
31 |
32 | // Headings
33 | $headingFont: Ubuntu, 'sans-serif';
34 | $headingTextShadow: 2px 2px 2px $coal;
35 |
36 | // h1 shadow, borrowed humbly from
37 | // (c) Default theme by Hakim El Hattab
38 | $heading1TextShadow: 0 1px 0 #ccc, 0 2px 0 #c9c9c9, 0 3px 0 #bbb, 0 4px 0 #b9b9b9, 0 5px 0 #aaa, 0 6px 1px rgba(0,0,0,.1), 0 0 5px rgba(0,0,0,.1), 0 1px 3px rgba(0,0,0,.3), 0 3px 5px rgba(0,0,0,.2), 0 5px 10px rgba(0,0,0,.25), 0 20px 20px rgba(0,0,0,.15);
39 |
40 | // Links
41 | $linkColor: $blood;
42 | $linkColorHover: lighten( $linkColor, 20% );
43 |
44 | // Text selection
45 | $selectionBackgroundColor: $blood;
46 | $selectionColor: #fff;
47 |
48 | // Background generator
49 | @mixin bodyBackground() {
50 | @include radial-gradient( $coal, lighten( $coal, 25% ) );
51 | }
52 |
53 | // Theme template ------------------------------
54 | @import "../template/theme";
55 | // ---------------------------------------------
56 |
57 | // some overrides after theme template import
58 |
59 | .reveal p {
60 | font-weight: 300;
61 | text-shadow: 1px 1px $coal;
62 | }
63 |
64 | .reveal h1,
65 | .reveal h2,
66 | .reveal h3,
67 | .reveal h4,
68 | .reveal h5,
69 | .reveal h6 {
70 | font-weight: 700;
71 | }
72 |
73 | .reveal a:not(.image),
74 | .reveal a:not(.image):hover {
75 | text-shadow: 2px 2px 2px #000;
76 | }
77 |
78 | .reveal small a:not(.image),
79 | .reveal small a:not(.image):hover {
80 | text-shadow: 1px 1px 1px #000;
81 | }
82 |
83 | .reveal p code {
84 | background-color: $codeBackground;
85 | display: inline-block;
86 | border-radius: 7px;
87 | }
88 |
89 | .reveal small code {
90 | vertical-align: baseline;
91 | }
--------------------------------------------------------------------------------
/blaze/compute/utils.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import, division, print_function
2 |
3 | from datetime import datetime
4 | from decimal import Decimal
5 |
6 | import sqlalchemy as sa
7 | import sqlalchemy.orm
8 | from toolz import curry
9 | from datashape.predicates import isrecord
10 | from ..expr import Field
11 | from odo.backends.sql import dshape_to_alchemy
12 |
13 | # This was taken from the following StackOverflow post
14 | # http://stackoverflow.com/questions/5631078/sqlalchemy-print-the-actual-query
15 | # answer by bukzor http://stackoverflow.com/users/146821/bukzor
16 |
17 |
18 | def literalquery(statement, dialect=None):
19 | """Generate an SQL expression string with bound parameters rendered inline
20 | for the given SQLAlchemy statement.
21 |
22 | WARNING: This method of escaping is insecure, incomplete, and for debugging
23 | purposes only. Executing SQL statements with inline-rendered user values is
24 | extremely insecure.
25 | """
26 | if isinstance(statement, sqlalchemy.orm.Query):
27 | if dialect is None:
28 | dialect = statement.session.get_bind(
29 | statement._mapper_zero_or_none()
30 | ).dialect
31 | statement = statement.statement
32 | if dialect is None:
33 | dialect = getattr(statement.bind, 'dialect', None)
34 | if dialect is None:
35 | from sqlalchemy.dialects import mysql
36 | dialect = mysql.dialect()
37 |
38 | Compiler = type(statement._compiler(dialect))
39 |
40 | class LiteralCompiler(Compiler):
41 | visit_bindparam = Compiler.render_literal_bindparam
42 |
43 | def render_literal_value(self, value, type_):
44 | if isinstance(value, (Decimal, long)):
45 | return str(value)
46 | elif isinstance(value, datetime):
47 | return repr(str(value))
48 | else: # fallback
49 | value = super(LiteralCompiler, self).render_literal_value(
50 | value, type_,
51 | )
52 | if isinstance(value, unicode):
53 | return value.encode('UTF-8')
54 | else:
55 | return value
56 |
57 | return LiteralCompiler(dialect, statement)
58 |
59 |
60 | def make_sqlalchemy_table(expr):
61 | return sa.Table(expr._name, sa.MetaData(), *dshape_to_alchemy(expr.dshape))
62 |
63 |
64 | @curry
65 | def istable(db, t):
66 | return (isinstance(t, Field) and
67 | isrecord(t.dshape.measure) and
68 | t._child.isidentical(db))
69 |
--------------------------------------------------------------------------------
/docs/source/_static/presentations/markdown/dask-array-meteorology.md:
--------------------------------------------------------------------------------
1 | ## Meteorological data
2 |
3 | We have a pile of NetCDF files
4 |
5 | $ ls
6 | 2014-01-01.nc 2014-03-18.nc 2014-06-02.nc 2014-08-17.nc 2014-11-01.nc
7 | 2014-01-02.nc 2014-03-19.nc 2014-06-03.nc 2014-08-18.nc 2014-11-02.nc
8 | 2014-01-03.nc 2014-03-20.nc 2014-06-04.nc 2014-08-19.nc 2014-11-03.nc
9 | 2014-01-04.nc 2014-03-21.nc 2014-06-05.nc 2014-08-20.nc 2014-11-04.nc
10 | ... ... ... ... ...
11 |
12 | Four measurements per day, quarter degree resolution, for 2014
13 |
14 | >>> import netCDF4
15 | >>> t = netCDF4.Dataset('2014-01-01.nc').variables['t2m']
16 | >>> t.shape
17 | (4, 721, 1440)
18 |
19 |
20 | ## Meteorological data
21 |
22 | Point to a bunch of NetCDF datasets
23 |
24 | >>> filenames = sorted(glob('2014-*.nc'))
25 | >>> temps = [netCDF4.Dataset(fn).variables['t2m'] for fn in filenames]
26 |
27 | Wrap each with `dask.array`
28 |
29 | >>> import dask.array as da
30 | >>> arrays = [da.from_array(t, chunks=(4, 200, 200)) for t in temps]
31 |
32 | Manipulate arrays with numpy syntax
33 |
34 | >>> x = da.concatenate(arrays, axis=0)
35 | >>> x.shape
36 | (1464, 721, 1440)
37 |
38 |
39 | ## Meteorological data
40 |
41 | Interact with the ecosystem
42 |
43 | >>> from matplotlib import imshow
44 | >>> imshow(x.mean(axis=0), cmap='bone')
45 |
46 | 
47 |
48 |
49 | ## Meteorological data
50 |
51 | Interact with the ecosystem
52 |
53 | >>> from matplotlib import imshow
54 | >>> imshow(x[1000] - x.mean(axis=0), cmap='RdBu_r')
55 |
56 | 
57 |
58 |
59 | ## Meteorological data
60 |
61 | Interact with the ecosystem
62 |
63 | >>> from matplotlib import imshow
64 | >>> imshow(x[::4].mean(axis=0) - x[2::4].mean(axis=0), cmap='RdBu_r')
65 |
66 | 
67 |
68 |
69 |
70 | ## XRay
71 |
72 | 
73 |
74 | Dask.array integrates with XRay.
75 | [http://xray.readthedocs.org](http://xray.readthedocs.org)
76 |
77 | 78 | 79 | * Implements the netCDF model 80 | * Set of associated ndarrays / variables 81 | * Pandas index along each axis 82 | * Index and reason using named axes with labels 83 | * NumPy -- `x[40:100].mean(axis=2)` 84 | * XRay -- `ds.sel(time='2014-04').mean('time')` 85 | 86 |
87 | 88 | Written by Stephan Hoyer (@shoyer) at Climate Corp 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /docs/source/whatsnew/0.10.2.txt: -------------------------------------------------------------------------------- 1 | Release 0.10.2 2 | ----------------- 3 | 4 | :Release: 0.10.2 5 | 6 | New Expressions 7 | ^^^^^^^^^^^^^^^ 8 | 9 | None 10 | 11 | Improved Expressions 12 | ^^^^^^^^^^^^^^^^^^^^ 13 | 14 | * Adds support for ``any`` and ``all`` to the sql backend (:issue:`1511`). 15 | 16 | New Backends 17 | ^^^^^^^^^^^^ 18 | 19 | None 20 | 21 | Improved Backends 22 | ^^^^^^^^^^^^^^^^^ 23 | 24 | * To allow access to the ``map`` and ``apply`` expressions in client / server 25 | interactions when in a trusted environment, new ``_trusted`` versions of the 26 | several default ``SerializationFormat`` instances were added. These trusted 27 | variants allow (de)serialization of builtin functions, NumPy functions, and 28 | Pandas functions. They are intentially kept separate from the default 29 | versions to ensure they are not accidentally enabled in untrusted 30 | environments (:issue:`1497` :issue:`1504`). 31 | 32 | Experimental Features 33 | ^^^^^^^^^^^^^^^^^^^^^ 34 | 35 | None 36 | 37 | API Changes 38 | ^^^^^^^^^^^ 39 | 40 | None 41 | 42 | Bug Fixes 43 | ^^^^^^^^^ 44 | 45 | * Fixed a bug with ``to_tree()`` and ``slice`` objects. Have to change the 46 | order of cases in ``to_tree()`` to ensure ``slice`` objects are handled 47 | before lookups inside the ``names`` namespace (:issue:`1516`). 48 | * Perform more input validation for ``sort()`` expression arguments 49 | (:issue:`1517`). 50 | * Fixes issue with string and datetime coercions on Pandas objects 51 | (:issue:`1519` :issue:`1524`). 52 | * Fixed a bug with ``isin`` and ``Selection``\s on sql selectables 53 | (:issue:`1528`). 54 | 55 | Miscellaneous 56 | ^^^^^^^^^^^^^ 57 | 58 | Expression Identity Rework 59 | """""""""""""""""""""""""" 60 | 61 | Expression are now memoized by their inputs. This means that two identical 62 | expressions will always be the same object, or that ``a.isidentical(b)`` is the 63 | same as ``a is b``. ``isidentical`` is called hundreds of thousands of times in 64 | a normal blaze workload. Moving more work to expression construction time has 65 | been shown to dramatically improve compute times when the expressions grow in 66 | complexity or size. In the past, blaze was spending linear time relative to the 67 | expression size to compare expressions because it needed to recurse through the 68 | entire expression tree but now it can do ``isidentical`` in constant time. 69 | 70 | Users should still use ``a.isidentical(b)`` instead of ``a is b`` because we 71 | reserve the right to add more arguments or change the implementation of 72 | ``isidentical`` in the future. 73 | -------------------------------------------------------------------------------- /docs/source/overview.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Overview 3 | ======== 4 | 5 | Blaze Abstracts Computation and Storage 6 | --------------------------------------- 7 | 8 | .. image:: svg/numpy_plus.png 9 | :align: center 10 | 11 | 12 | Several projects provide rich and performant data analytics. Competition 13 | between these projects gives rise to a vibrant and dynamic ecosystem. 14 | Blaze augments this ecosystem with a uniform and adaptable interface. Blaze 15 | orchestrates computation and data access among these external projects. It 16 | provides a consistent backdrop to build standard interfaces usable by the 17 | current Python community. 18 | 19 | 20 | Demonstration 21 | ------------- 22 | 23 | Blaze separates the computations that we want to perform: 24 | 25 | .. code-block:: python 26 | 27 | >>> from blaze import * 28 | >>> accounts = symbol('accounts', 'var * {id: int, name: string, amount: int}') 29 | 30 | >>> deadbeats = accounts[accounts.amount < 0].name 31 | 32 | From the representation of data 33 | 34 | .. code-block:: python 35 | 36 | >>> L = [[1, 'Alice', 100], 37 | ... [2, 'Bob', -200], 38 | ... [3, 'Charlie', 300], 39 | ... [4, 'Denis', 400], 40 | ... [5, 'Edith', -500]] 41 | 42 | Blaze enables users to solve data-oriented problems 43 | 44 | .. code-block:: python 45 | 46 | >>> list(compute(deadbeats, L)) 47 | ['Bob', 'Edith'] 48 | 49 | But the separation of expression from data allows us to switch between 50 | different backends. 51 | 52 | Here we solve the same problem using Pandas instead of Pure Python. 53 | 54 | .. code-block:: python 55 | 56 | >>> df = DataFrame(L, columns=['id', 'name', 'amount']) 57 | 58 | >>> compute(deadbeats, df) 59 | 1 Bob 60 | 4 Edith 61 | Name: name, dtype: object 62 | 63 | Blaze doesn't compute these results, Blaze intelligently drives other projects 64 | to compute them instead. These projects range from simple Pure Python 65 | iterators to powerful distributed Spark clusters. Blaze is built to be 66 | extended to new systems as they evolve. 67 | 68 | Scope 69 | ----- 70 | 71 | Blaze speaks Python and Pandas as seen above and also several other 72 | technologies, including NumPy, SQL, Mongo, Spark, PyTables, etc.. Blaze is 73 | built to make connecting to a new technology easy. 74 | 75 | Blaze currently targets database and array technologies used for analytic 76 | queries. It strives to orchestrate and provide interfaces on top of and in 77 | between other computational systems. We provide performance by providing data 78 | scientists with intuitive access to a variety of tools. 79 | -------------------------------------------------------------------------------- /blaze/expr/tests/test_literal_expr.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import pytest 4 | import re 5 | 6 | 7 | from datashape import dshape 8 | 9 | from blaze import discover, transform 10 | from blaze.expr import data, literal 11 | from blaze.utils import normalize 12 | 13 | 14 | tdata = (('Alice', 100), 15 | ('Bob', 200)) 16 | 17 | L = [[1, 'Alice', 100], 18 | [2, 'Bob', -200], 19 | [3, 'Charlie', 300], 20 | [4, 'Denis', 400], 21 | [5, 'Edith', -500]] 22 | 23 | l = literal(tdata) 24 | nl = literal(tdata, name='amounts') 25 | t = data(tdata, fields=['name', 'amount']) 26 | 27 | 28 | def test_discover_on_data(): 29 | assert discover(t) == dshape("2 * {name: string, amount: int64}") 30 | 31 | 32 | def test_discover_on_literal(): 33 | assert discover(l) == dshape("2 * (string, int64)") 34 | 35 | 36 | def test_table_raises_on_inconsistent_inputs(): 37 | with pytest.raises(ValueError) as excinfo: 38 | data(tdata, schema='{name: string, amount: float32}', 39 | dshape=dshape("{name: string, amount: float32}")) 40 | assert "specify one of schema= or dshape= keyword" in str(excinfo.value) 41 | 42 | 43 | def test_resources(): 44 | assert t._resources() == {t: t.data} 45 | 46 | 47 | def test_literal_repr(): 48 | assert "(('Alice', 100), ('Bob', 200))" == repr(l) 49 | 50 | 51 | def test_literal_name_repr(): 52 | assert "amounts" == repr(nl) 53 | 54 | 55 | def test_data_repr(): 56 | def replace_name(s): 57 | 58 | # Account for autogenerated name. 59 | return re.sub(r"name='(.*)'", "name='_0'", s) 60 | expected = "<'tuple' data; _name='.+', dshape='2 * {name: string, amount: int64}'>" # noqa 61 | assert replace_name(expected) == replace_name(repr(t)) 62 | 63 | 64 | def test_str_does_not_repr(): 65 | # see GH issue #1240. 66 | d = data( 67 | [('aa', 1), ('b', 2)], 68 | name="ZZZ", 69 | dshape='2 * {a: string, b: int64}', 70 | ) 71 | expr = transform(d, c=d.a.str.len() + d.b) 72 | assert ( 73 | normalize(str(expr)) == 74 | normalize(""" 75 | Merge( 76 | args=(ZZZ, label(len(_child=ZZZ.a) + ZZZ.b, 'c')), 77 | _varargsexpr=VarArgsExpr( 78 | _inputs=(ZZZ, label(len(_child=ZZZ.a) + ZZZ.b, 'c')) 79 | ), 80 | _shape=(2,) 81 | ) 82 | """) 83 | ) 84 | 85 | 86 | def test_isidentical_regr(): 87 | # regression test for #1387 88 | tdata = np.array([(np.nan,), (np.nan,)], dtype=[('a', 'float64')]) 89 | ds = data(tdata) 90 | assert ds.a.isidentical(ds.a) 91 | -------------------------------------------------------------------------------- /blaze/tests/test_partition.py: -------------------------------------------------------------------------------- 1 | from blaze.partition import * 2 | from blaze.expr import shape 3 | 4 | import numpy as np 5 | 6 | x = np.arange(24).reshape(4, 6) 7 | 8 | 9 | def eq(a, b): 10 | if isinstance(a == b, bool): 11 | return a == b 12 | if isinstance(a, np.ndarray) or isinstance(b, np.ndarray): 13 | return (a == b).all() 14 | else: 15 | return a == b 16 | 17 | 18 | def test_partition_get(): 19 | assert eq(partition_get(x, (0, slice(0, None)), chunksize=(1, 6)), 20 | x[0, :]) 21 | assert eq(partition_get(x, (slice(0, None), 0), chunksize=(4, 1)), 22 | x[:, 0]) 23 | assert eq(partition_get(x, (slice(2, 4), slice(0, 2)), chunksize=(2, 2)), 24 | x[2:4, 0:2]) 25 | 26 | 27 | def test_partition_set(): 28 | x = np.arange(24).reshape(4, 6) 29 | partition_set(x, 30 | (slice(0, 2), slice(0, 2)), np.array([[1, 1], [1, 1]]), 31 | chunksize=(2, 2)) 32 | assert (x[:2, :2] == 1).all() 33 | 34 | 35 | def test_partition_set_1d(): 36 | x = np.arange(24).reshape(4, 6) 37 | partition_set(x, 38 | (slice(0, 4), 0), np.array([[1], [1], [1], [1]]), 39 | chunksize=(4, 1), 40 | keepdims=False) 41 | assert (x[:4, 0] == 1).all() 42 | 43 | 44 | def test_partitions(): 45 | assert list(partitions(x, chunksize=(1, 6))) == \ 46 | [(i, slice(0, 6)) for i in range(4)] 47 | assert list(partitions(x, chunksize=(4, 1))) == \ 48 | [(slice(0, 4), i) for i in range(6)] 49 | assert list(partitions(x, chunksize=(2, 3))) == [ 50 | (slice(0, 2), slice(0, 3)), (slice(0, 2), slice(3, 6)), 51 | (slice(2, 4), slice(0, 3)), (slice(2, 4), slice(3, 6))] 52 | 53 | 54 | def dont_test_partitions_flat(): 55 | assert list(partitions(x, chunksize=(2, 3))) == [ 56 | (slice(0, 2), slice(0, 3)), (slice(0, 2), slice(3, 6)), 57 | (slice(2, 4), slice(0, 3)), (slice(2, 4), slice(3, 6))] 58 | 59 | 60 | def test_uneven_partitions(): 61 | x = np.arange(10*12).reshape(10, 12) 62 | parts = list(partitions(x, chunksize=(7, 7))) 63 | 64 | assert len(parts) == 2 * 2 65 | 66 | assert parts == [(slice(0, 7), slice(0, 7)), (slice(0, 7), slice(7, 12)), 67 | (slice(7, 10), slice(0, 7)), (slice(7, 10), slice(7, 12))] 68 | 69 | x = np.arange(20*24).reshape(20, 24) 70 | parts = list(partitions(x, chunksize=(7, 7))) 71 | 72 | 73 | def test_3d_partitions(): 74 | x = np.arange(4*4*6).reshape(4, 4, 6) 75 | parts = list(partitions(x, chunksize=(2, 2, 3))) 76 | assert len(parts) == 2 * 2 * 2 77 | -------------------------------------------------------------------------------- /docs/source/_static/presentations/plugin/notes/notes.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Handles opening of and synchronization with the reveal.js 3 | * notes window. 4 | */ 5 | var RevealNotes = (function() { 6 | 7 | function openNotes() { 8 | var jsFileLocation = document.querySelector('script[src$="notes.js"]').src; // this js file path 9 | jsFileLocation = jsFileLocation.replace(/notes\.js(\?.*)?$/, ''); // the js folder path 10 | var notesPopup = window.open( jsFileLocation + 'notes.html', 'reveal.js - Notes', 'width=1120,height=850' ); 11 | 12 | // Fires when slide is changed 13 | Reveal.addEventListener( 'slidechanged', post ); 14 | 15 | // Fires when a fragment is shown 16 | Reveal.addEventListener( 'fragmentshown', post ); 17 | 18 | // Fires when a fragment is hidden 19 | Reveal.addEventListener( 'fragmenthidden', post ); 20 | 21 | /** 22 | * Posts the current slide data to the notes window 23 | */ 24 | function post() { 25 | var slideElement = Reveal.getCurrentSlide(), 26 | slideIndices = Reveal.getIndices(), 27 | messageData; 28 | 29 | var notes = slideElement.querySelector( 'aside.notes' ), 30 | nextindexh, 31 | nextindexv; 32 | 33 | if( slideElement.nextElementSibling && slideElement.parentNode.nodeName == 'SECTION' ) { 34 | nextindexh = slideIndices.h; 35 | nextindexv = slideIndices.v + 1; 36 | } else { 37 | nextindexh = slideIndices.h + 1; 38 | nextindexv = 0; 39 | } 40 | 41 | messageData = { 42 | notes : notes ? notes.innerHTML : '', 43 | indexh : slideIndices.h, 44 | indexv : slideIndices.v, 45 | indexf : slideIndices.f, 46 | nextindexh : nextindexh, 47 | nextindexv : nextindexv, 48 | markdown : notes ? typeof notes.getAttribute( 'data-markdown' ) === 'string' : false 49 | }; 50 | 51 | notesPopup.postMessage( JSON.stringify( messageData ), '*' ); 52 | } 53 | 54 | // Navigate to the current slide when the notes are loaded 55 | notesPopup.addEventListener( 'load', function( event ) { 56 | post(); 57 | }, false ); 58 | } 59 | 60 | // If the there's a 'notes' query set, open directly 61 | if( window.location.search.match( /(\?|\&)notes/gi ) !== null ) { 62 | openNotes(); 63 | } 64 | 65 | // Open the notes when the 's' key is hit 66 | document.addEventListener( 'keydown', function( event ) { 67 | // Disregard the event if the target is editable or a 68 | // modifier is present 69 | if ( document.querySelector( ':focus' ) !== null || event.shiftKey || event.altKey || event.ctrlKey || event.metaKey ) return; 70 | 71 | if( event.keyCode === 83 ) { 72 | event.preventDefault(); 73 | openNotes(); 74 | } 75 | }, false ); 76 | 77 | return { open: openNotes }; 78 | })(); 79 | --------------------------------------------------------------------------------