├── .gitignore ├── autograd ├── __init__.py ├── numpy │ ├── __init__.py │ ├── numpy_wrapper.py │ ├── numpy_vjps.py │ └── numpy_boxes.py ├── differential_operators.py ├── util.py ├── core.py └── tracer.py ├── notebooks ├── tanh.png ├── Autograd Exploration.ipynb └── .ipynb_checkpoints │ └── Autograd Exploration-checkpoint.ipynb ├── examples └── tanh.py ├── license.txt └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /autograd/__init__.py: -------------------------------------------------------------------------------- 1 | from .differential_operators import make_vjp, grad 2 | -------------------------------------------------------------------------------- /notebooks/tanh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mattjj/autodidact/HEAD/notebooks/tanh.png -------------------------------------------------------------------------------- /autograd/numpy/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from .numpy_wrapper import * 3 | from . import numpy_boxes 4 | from . import numpy_vjps 5 | -------------------------------------------------------------------------------- /examples/tanh.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import autograd.numpy as np 3 | import matplotlib.pyplot as plt 4 | from autograd import grad 5 | 6 | # We could use np.tanh, but let's write our own as an example. 7 | def tanh(x): 8 | return (1.0 - np.exp(-x)) / (1.0 + np.exp(-x)) 9 | 10 | x = np.linspace(-7, 7, 200) 11 | plt.plot(x, tanh(x), 12 | x, grad(tanh)(x), # first derivative 13 | x, grad(grad(tanh))(x), # second derivative 14 | x, grad(grad(grad(tanh)))(x), # third derivative 15 | x, grad(grad(grad(grad(tanh))))(x), # fourth derivative 16 | x, grad(grad(grad(grad(grad(tanh)))))(x), # fifth derivative 17 | x, grad(grad(grad(grad(grad(grad(tanh))))))(x)) # sixth derivative 18 | 19 | plt.axis('off') 20 | plt.savefig("tanh.png") 21 | plt.show() 22 | -------------------------------------------------------------------------------- /autograd/differential_operators.py: -------------------------------------------------------------------------------- 1 | """Convenience functions built on top of `make_vjp`.""" 2 | 3 | import numpy as np 4 | 5 | from .core import make_vjp 6 | from .util import subval 7 | 8 | def grad(fun, argnum=0): 9 | """Constructs gradient function. 10 | 11 | Given a function fun(x), returns a function fun'(x) that returns the 12 | gradient of fun(x) wrt x. 13 | 14 | Args: 15 | fun: single-argument function. ndarray -> ndarray. 16 | argnum: integer. Index of argument to take derivative wrt. 17 | 18 | Returns: 19 | gradfun: function that takes same args as fun(), but returns the gradient 20 | wrt to fun()'s argnum-th argument. 21 | """ 22 | def gradfun(*args, **kwargs): 23 | # Replace args[argnum] with x. Define a single-argument function to 24 | # compute derivative wrt. 25 | unary_fun = lambda x: fun(*subval(args, argnum, x), **kwargs) 26 | 27 | # Construct vector-Jacobian product 28 | vjp, ans = make_vjp(unary_fun, args[argnum]) 29 | return vjp(np.ones_like(ans)) 30 | return gradfun 31 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 by the President and Fellows of Harvard University 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Autodidact: a pedagogical implementation of [Autograd](https://github.com/hips/autograd) 2 | 3 | This is a tutorial implementation based on [the full version of 4 | Autograd](https://github.com/hips/autograd). 5 | 6 | Example use: 7 | 8 | ```python 9 | >>> import autograd.numpy as np # Thinly-wrapped numpy 10 | >>> from autograd import grad # The only autograd function you may ever need 11 | >>> 12 | >>> def tanh(x): # Define a function 13 | ... y = np.exp(-2.0 * x) 14 | ... return (1.0 - y) / (1.0 + y) 15 | ... 16 | >>> grad_tanh = grad(tanh) # Obtain its gradient function 17 | >>> grad_tanh(1.0) # Evaluate the gradient at x = 1.0 18 | 0.41997434161402603 19 | >>> (tanh(1.0001) - tanh(0.9999)) / 0.0002 # Compare to finite differences 20 | 0.41997434264973155 21 | ``` 22 | 23 | We can continue to differentiate as many times as we like, and use numpy's 24 | vectorization of scalar-valued functions across many different input values: 25 | 26 | ```python 27 | >>> import matplotlib.pyplot as plt 28 | >>> x = np.linspace(-7, 7, 200) 29 | >>> plt.plot(x, tanh(x), 30 | ... x, grad(tanh)(x), # first derivative 31 | ... x, grad(grad(tanh))(x), # second derivative 32 | ... x, grad(grad(grad(tanh)))(x), # third derivative 33 | ... x, grad(grad(grad(grad(tanh))))(x), # fourth derivative 34 | ... x, grad(grad(grad(grad(grad(tanh)))))(x), # fifth derivative 35 | ... x, grad(grad(grad(grad(grad(grad(tanh))))))(x)) # sixth derivative 36 | >>> plt.show() 37 | ``` 38 | 39 | Autograd was written by [Dougal Maclaurin](https://dougalmaclaurin.com), 40 | [David Duvenaud](https://www.cs.toronto.edu/~duvenaud/) 41 | and [Matt Johnson](http://people.csail.mit.edu/mattjj/). 42 | See [the main page](https://github.com/hips/autograd) for more information. 43 | -------------------------------------------------------------------------------- /autograd/util.py: -------------------------------------------------------------------------------- 1 | """Utility functions.""" 2 | 3 | 4 | def subvals(x, ivs): 5 | """Replace the i-th value of x with v. 6 | 7 | Args: 8 | x: iterable of items. 9 | ivs: list of (int, value) pairs. 10 | 11 | Returns: 12 | x modified appropriately. 13 | """ 14 | x_ = list(x) 15 | for i, v in ivs: 16 | x_[i] = v 17 | return tuple(x_) 18 | 19 | def subval(x, i, v): 20 | """Replace the i-th value of x with v.""" 21 | x_ = list(x) 22 | x_[i] = v 23 | return tuple(x_) 24 | 25 | def toposort(end_node): 26 | child_counts = {} 27 | stack = [end_node] 28 | while stack: 29 | node = stack.pop() 30 | if node in child_counts: 31 | child_counts[node] += 1 32 | else: 33 | child_counts[node] = 1 34 | stack.extend(node.parents) 35 | 36 | childless_nodes = [end_node] 37 | while childless_nodes: 38 | node = childless_nodes.pop() 39 | yield node 40 | for parent in node.parents: 41 | if child_counts[parent] == 1: 42 | childless_nodes.append(parent) 43 | else: 44 | child_counts[parent] -= 1 45 | 46 | def wraps(fun, namestr="{fun}", docstr="{doc}", **kwargs): 47 | """Decorator for a function wrapping another. 48 | 49 | Used when wrapping a function to ensure its name and docstring get copied 50 | over. 51 | 52 | Args: 53 | fun: function to be wrapped 54 | namestr: Name string to use for wrapped function. 55 | docstr: Docstring to use for wrapped function. 56 | **kwargs: additional string format values. 57 | 58 | Return: 59 | Wrapped function. 60 | """ 61 | def _wraps(f): 62 | try: 63 | f.__name__ = namestr.format(fun=get_name(fun), **kwargs) 64 | f.__doc__ = docstr.format(fun=get_name(fun), doc=get_doc(fun), **kwargs) 65 | finally: 66 | return f 67 | return _wraps 68 | 69 | def wrap_nary_f(fun, op, argnum): 70 | namestr = "{op}_of_{fun}_wrt_argnum_{argnum}" 71 | docstr = """\ 72 | {op} of function {fun} with respect to argument number {argnum}. Takes the 73 | same arguments as {fun} but returns the {op}. 74 | """ 75 | return wraps(fun, namestr, docstr, op=get_name(op), argnum=argnum) 76 | 77 | get_name = lambda f: getattr(f, '__name__', '[unknown name]') 78 | get_doc = lambda f: getattr(f, '__doc__' , '') 79 | -------------------------------------------------------------------------------- /autograd/numpy/numpy_wrapper.py: -------------------------------------------------------------------------------- 1 | """Wrapped numpy functions. 2 | 3 | This library contains all functions and methods in numpy. Unless specified in 4 | 'nograd_functions', the function is assumed to have a registered vector-Jacobian 5 | product. 6 | 7 | Uses Python namespace-is-a-dict magic. 8 | """ 9 | from __future__ import absolute_import 10 | import types 11 | from autograd.tracer import primitive, notrace_primitive 12 | import numpy as _np 13 | 14 | # ----- Non-differentiable functions ----- 15 | 16 | nograd_functions = [ 17 | _np.all, 18 | _np.allclose, 19 | _np.any, 20 | _np.argmax, 21 | _np.argmin, 22 | _np.argpartition, 23 | _np.argsort, 24 | _np.argwhere, 25 | _np.around, 26 | _np.array_equal, 27 | _np.array_equiv, 28 | _np.ceil, 29 | _np.count_nonzero, 30 | _np.equal, 31 | _np.fix, 32 | _np.flatnonzero, 33 | _np.floor, 34 | _np.floor_divide, 35 | _np.greater, 36 | _np.greater_equal, 37 | _np.isclose, 38 | _np.iscomplex, 39 | _np.iscomplexobj, 40 | _np.isfinite, 41 | _np.isinf, 42 | _np.isnan, 43 | _np.isneginf, 44 | _np.isposinf, 45 | _np.isreal, 46 | _np.isscalar, 47 | _np.less, 48 | _np.less_equal, 49 | _np.logical_and, 50 | _np.logical_not, 51 | _np.logical_or, 52 | _np.logical_xor, 53 | _np.ndim, 54 | _np.nonzero, 55 | _np.not_equal, 56 | _np.ones_like, 57 | _np.result_type, 58 | _np.rint, 59 | _np.round, 60 | _np.searchsorted, 61 | _np.shape, 62 | _np.sign, 63 | _np.size, 64 | _np.trunc, 65 | _np.zeros_like, 66 | ] 67 | 68 | def wrap_intdtype(cls): 69 | class IntdtypeSubclass(cls): 70 | __new__ = notrace_primitive(cls.__new__) 71 | return IntdtypeSubclass 72 | 73 | def wrap_namespace(old, new): 74 | """Copy all functions in 'old' namespace to 'new' namespace. 75 | 76 | Args: 77 | old: __dict__ of module to copy from. 78 | new: __dict__ of module to copy to. 79 | """ 80 | unchanged_types = {float, int, type(None), type} 81 | int_types = {_np.int, _np.int8, _np.int16, _np.int32, _np.int64, _np.integer} 82 | function_types = {_np.ufunc, types.FunctionType, types.BuiltinFunctionType} 83 | for name, obj in old.items(): 84 | if obj in nograd_functions: 85 | # Functions without gradients. We don't bother to trace values that 86 | # enter here. 87 | new[name] = notrace_primitive(obj) 88 | elif type(obj) in function_types: 89 | # Functions with gradients. We trace values. 90 | new[name] = primitive(obj) 91 | elif type(obj) is type and obj in int_types: 92 | # Wrap int types with something identical except that calls to __new__ 93 | # immediately strip argument of boxes. 94 | # 95 | # TODO(duckworthd): Why do numpy int types need to be boxed, but not 96 | # Python's int()? 97 | new[name] = wrap_intdtype(obj) 98 | elif type(obj) in unchanged_types: 99 | new[name] = obj 100 | 101 | # Set autograd.numpy. = wrap(numpy.) 102 | wrap_namespace(_np.__dict__, globals()) 103 | -------------------------------------------------------------------------------- /autograd/core.py: -------------------------------------------------------------------------------- 1 | """Vector-Jacobian Products, Backpropagation. 2 | 3 | Construct vector-Jacobian product of a DAG of computation. With this library, 4 | one can, 5 | 6 | - Construct vector-Jacobian product for any single-input single-output function. 7 | (make_vjp) 8 | - Register vector-Jacobian product functions for any primitive function and 9 | argument index. 10 | """ 11 | from collections import defaultdict 12 | from itertools import count 13 | import numpy as np 14 | 15 | from .tracer import trace, Node 16 | from .util import toposort 17 | 18 | def make_vjp(fun, x): 19 | """Make function for vector-Jacobian product. 20 | 21 | Args: 22 | fun: single-arg function. Jacobian derived from this. 23 | x: ndarray. Point to differentiate about. 24 | 25 | Returns: 26 | vjp: single-arg function. vector -> vector-Jacobian[fun, x] product. 27 | end_value: end_value = fun(start_node) 28 | 29 | """ 30 | start_node = Node.new_root() 31 | end_value, end_node = trace(start_node, fun, x) 32 | if end_node is None: 33 | def vjp(g): return np.zeros_like(x) 34 | else: 35 | def vjp(g): return backward_pass(g, end_node) 36 | return vjp, end_value 37 | 38 | def backward_pass(g, end_node): 39 | """Backpropagation. 40 | 41 | Traverse computation graph backwards in topological order from the end node. 42 | For each node, compute local gradient contribution and accumulate. 43 | """ 44 | outgrads = {end_node: g} 45 | for node in toposort(end_node): 46 | outgrad = outgrads.pop(node) 47 | fun, value, args, kwargs, argnums = node.recipe 48 | for argnum, parent in zip(argnums, node.parents): 49 | # Lookup vector-Jacobian product (gradient) function for this 50 | # function/argument. 51 | vjp = primitive_vjps[fun][argnum] 52 | 53 | # Compute vector-Jacobian product (gradient) contribution due to 54 | # parent node's use in this function. 55 | parent_grad = vjp(outgrad, value, *args, **kwargs) 56 | 57 | # Save vector-Jacobian product (gradient) for upstream nodes. 58 | # Sum contributions with all others also using parent's output. 59 | outgrads[parent] = add_outgrads(outgrads.get(parent), parent_grad) 60 | return outgrad 61 | 62 | def add_outgrads(prev_g, g): 63 | """Add gradient contributions together.""" 64 | if prev_g is None: 65 | return g 66 | return prev_g + g 67 | 68 | primitive_vjps = defaultdict(dict) 69 | def defvjp(fun, *vjps, **kwargs): 70 | """Register vector-Jacobian product functions. 71 | 72 | Let fun(x, y, ...) = ans be a function. We wish to register a 73 | vector-Jacobian product for each of fun's arguments. That is, functions 74 | 75 | vjp_x(g, ans, x, y, ...) = g df/dx 76 | vjp_y(g, ans, x, y, ...) = g df/dy 77 | ... 78 | 79 | This function registers said callbacks. 80 | 81 | Args: 82 | fun: function for which one wants to define vjps for. 83 | *vjps: functions. vector-Jacobian products. One per argument to fun(). 84 | **kwargs: additional keyword arugments. Only 'argnums' is used. 85 | """ 86 | argnums = kwargs.get('argnums', count()) 87 | for argnum, vjp in zip(argnums, vjps): 88 | primitive_vjps[fun][argnum] = vjp 89 | -------------------------------------------------------------------------------- /autograd/numpy/numpy_vjps.py: -------------------------------------------------------------------------------- 1 | """Vector-Jacobian products for NumPy functions. 2 | 3 | This library consists of implementations of vector-Jacobian products (vjps, gradients) 4 | for functions implemented in numpy. Each function-argument index pair is 5 | provided a gradient function registered with defvjp(). 6 | 7 | Notice that vjps are implemented with (autograd-wrapped) functions as well. This 8 | is the magic that allows one to compute gradients-of-gradients! 9 | """ 10 | from __future__ import absolute_import 11 | import numpy as onp 12 | from . import numpy_wrapper as anp 13 | from .numpy_boxes import ArrayBox 14 | from autograd.tracer import primitive 15 | from autograd.core import defvjp 16 | 17 | # ----- Binary ufuncs ----- 18 | 19 | defvjp(anp.add, lambda g, ans, x, y : unbroadcast(x, g), 20 | lambda g, ans, x, y : unbroadcast(y, g)) 21 | defvjp(anp.multiply, lambda g, ans, x, y : unbroadcast(x, y * g), 22 | lambda g, ans, x, y : unbroadcast(y, x * g)) 23 | defvjp(anp.subtract, lambda g, ans, x, y : unbroadcast(x, g), 24 | lambda g, ans, x, y : unbroadcast(y, -g)) 25 | defvjp(anp.divide, lambda g, ans, x, y : unbroadcast(x, g / y), 26 | lambda g, ans, x, y : unbroadcast(y, - g * x / y**2)) 27 | defvjp(anp.true_divide, lambda g, ans, x, y : unbroadcast(x, g / y), 28 | lambda g, ans, x, y : unbroadcast(y, - g * x / y**2)) 29 | defvjp(anp.power, 30 | lambda g, ans, x, y: unbroadcast(x, g * y * x ** anp.where(y, y - 1, 1.)), 31 | lambda g, ans, x, y: unbroadcast(y, g * anp.log(replace_zero(x, 1.)) * x ** y)) 32 | 33 | def replace_zero(x, val): 34 | """Replace all zeros in 'x' with 'val'.""" 35 | return anp.where(x, x, val) 36 | 37 | def unbroadcast(target, g, broadcast_idx=0): 38 | """Remove broadcasted dimensions by summing along them. 39 | 40 | When computing gradients of a broadcasted value, this is the right thing to 41 | do when computing the total derivative and accounting for cloning. 42 | """ 43 | while anp.ndim(g) > anp.ndim(target): 44 | g = anp.sum(g, axis=broadcast_idx) 45 | for axis, size in enumerate(anp.shape(target)): 46 | if size == 1: 47 | g = anp.sum(g, axis=axis, keepdims=True) 48 | if anp.iscomplexobj(g) and not anp.iscomplex(target): 49 | g = anp.real(g) 50 | return g 51 | 52 | # ----- Simple grads ----- 53 | 54 | defvjp(anp.negative, lambda g, ans, x: -g) 55 | defvjp(anp.exp, lambda g, ans, x: ans * g) 56 | defvjp(anp.log, lambda g, ans, x: g / x) 57 | defvjp(anp.tanh, lambda g, ans, x: g / anp.cosh(x) **2) 58 | defvjp(anp.sinh, lambda g, ans, x: g * anp.cosh(x)) 59 | defvjp(anp.cosh, lambda g, ans, x: g * anp.sinh(x)) 60 | 61 | defvjp(anp.where, None, 62 | lambda g, ans, c, x=None, y=None: anp.where(c, g, anp.zeros(g.shape)), 63 | lambda g, ans, c, x=None, y=None: anp.where(c, anp.zeros(g.shape), g)) 64 | 65 | defvjp(anp.reshape, lambda g, ans, x, shape, order=None: 66 | anp.reshape(g, anp.shape(x), order=order)) 67 | 68 | # ----- Dot grads ----- 69 | 70 | def _dot_vjp_0(g, ans, lhs, rhs): 71 | if max(anp.ndim(lhs), anp.ndim(rhs)) > 2: 72 | raise NotImplementedError("Current dot vjps only support ndim <= 2.") 73 | 74 | if anp.ndim(lhs) == 0: 75 | return anp.sum(rhs * g) 76 | if anp.ndim(lhs) == 1 and anp.ndim(rhs) == 1: 77 | return g * rhs 78 | if anp.ndim(lhs) == 2 and anp.ndim(rhs) == 1: 79 | return g[:, None] * rhs 80 | if anp.ndim(lhs) == 1 and anp.ndim(rhs) == 2: 81 | return anp.dot(rhs, g) 82 | return anp.dot(g, rhs.T) 83 | 84 | def _dot_vjp_1(g, ans, lhs, rhs): 85 | if max(anp.ndim(lhs), anp.ndim(rhs)) > 2: 86 | raise NotImplementedError("Current dot vjps only support ndim <= 2.") 87 | 88 | if anp.ndim(rhs) == 0: 89 | return anp.sum(lhs * g) 90 | if anp.ndim(lhs) == 1 and anp.ndim(rhs) == 1: 91 | return g * lhs 92 | if anp.ndim(lhs) == 2 and anp.ndim(rhs) == 1: 93 | return anp.dot(g, lhs) 94 | if anp.ndim(lhs) == 1 and anp.ndim(rhs) == 2: 95 | return lhs[:, None] * g 96 | return anp.dot(lhs.T, g) 97 | 98 | defvjp(anp.dot, _dot_vjp_0, _dot_vjp_1) 99 | -------------------------------------------------------------------------------- /autograd/numpy/numpy_boxes.py: -------------------------------------------------------------------------------- 1 | """Box type for np.ndarray. 2 | 3 | This library contains a wrapper for np.ndarray with methods that uses 4 | autograd-wrapped numpy functions rather than their originals. This is necessary 5 | to ensure all transformations of an np.ndarray are traced. 6 | """ 7 | from __future__ import absolute_import 8 | import numpy as np 9 | from autograd.tracer import Box, primitive 10 | from . import numpy_wrapper as anp 11 | 12 | Box.__array_priority__ = 90.0 13 | 14 | class ArrayBox(Box): 15 | """Box for np.ndarray. 16 | 17 | Anything you can do with an np.ndarray, you can do with an ArrayBox. 18 | """ 19 | 20 | # This class has no attributes. 21 | __slots__ = [] 22 | 23 | # Used by NumPy to determine which type gets returned when there are 24 | # multiple possibilities. Larger numbers == higher priority. 25 | __array_priority__ = 100.0 26 | 27 | @primitive 28 | def __getitem__(A, idx): return A[idx] 29 | 30 | # Constants w.r.t float data just pass though 31 | shape = property(lambda self: self._value.shape) 32 | ndim = property(lambda self: self._value.ndim) 33 | size = property(lambda self: self._value.size) 34 | dtype = property(lambda self: self._value.dtype) 35 | T = property(lambda self: anp.transpose(self)) 36 | def __len__(self): return len(self._value) 37 | def astype(self, *args, **kwargs): return anp._astype(self, *args, **kwargs) 38 | 39 | def __neg__(self): return anp.negative(self) 40 | def __add__(self, other): return anp.add( self, other) 41 | def __sub__(self, other): return anp.subtract(self, other) 42 | def __mul__(self, other): return anp.multiply(self, other) 43 | def __pow__(self, other): return anp.power (self, other) 44 | def __div__(self, other): return anp.divide( self, other) 45 | def __mod__(self, other): return anp.mod( self, other) 46 | def __truediv__(self, other): return anp.true_divide(self, other) 47 | def __matmul__(self, other): return anp.matmul(self, other) 48 | def __radd__(self, other): return anp.add( other, self) 49 | def __rsub__(self, other): return anp.subtract(other, self) 50 | def __rmul__(self, other): return anp.multiply(other, self) 51 | def __rpow__(self, other): return anp.power( other, self) 52 | def __rdiv__(self, other): return anp.divide( other, self) 53 | def __rmod__(self, other): return anp.mod( other, self) 54 | def __rtruediv__(self, other): return anp.true_divide(other, self) 55 | def __rmatmul__(self, other): return anp.matmul(other, self) 56 | def __eq__(self, other): return anp.equal(self, other) 57 | def __ne__(self, other): return anp.not_equal(self, other) 58 | def __gt__(self, other): return anp.greater(self, other) 59 | def __ge__(self, other): return anp.greater_equal(self, other) 60 | def __lt__(self, other): return anp.less(self, other) 61 | def __le__(self, other): return anp.less_equal(self, other) 62 | def __abs__(self): return anp.abs(self) 63 | def __hash__(self): return id(self) 64 | 65 | # Register ArrayBox as the type to use when boxing np.ndarray and scalar values. 66 | ArrayBox.register(np.ndarray) 67 | for type_ in [float, np.float64, np.float32, np.float16, 68 | complex, np.complex64, np.complex128]: 69 | ArrayBox.register(type_) 70 | 71 | # Set ArrayBox. = autograd.numpy. 72 | nondiff_methods = [ 73 | 'all', 74 | 'any', 75 | 'argmax', 76 | 'argmin', 77 | 'argpartition', 78 | 'argsort', 79 | 'nonzero', 80 | 'searchsorted', 81 | 'round'] 82 | 83 | diff_methods = [ 84 | 'clip', 85 | 'compress', 86 | 'cumprod', 87 | 'cumsum', 88 | 'diagonal', 89 | 'max', 90 | 'mean', 91 | 'min', 92 | 'prod', 93 | 'ptp', 94 | 'ravel', 95 | 'repeat', 96 | 'reshape', 97 | 'squeeze', 98 | 'std', 99 | 'sum', 100 | 'swapaxes', 101 | 'take', 102 | 'trace', 103 | 'transpose', 104 | 'var'] 105 | for method_name in nondiff_methods + diff_methods: 106 | setattr(ArrayBox, method_name, anp.__dict__[method_name]) 107 | 108 | # Flatten has no function, only a method. 109 | setattr(ArrayBox, 'flatten', anp.__dict__['ravel']) 110 | 111 | -------------------------------------------------------------------------------- /autograd/tracer.py: -------------------------------------------------------------------------------- 1 | """Tracing utilities. 2 | 3 | 4 | This library provides functions for constructing a computation graph. With this 5 | library, one can, 6 | 7 | - Build a computation graph. (trace) 8 | - Register wrapper types for unwrapped values based on type(). (Box.register) 9 | - Build functions that can deal with wrapped values. (primitive, 10 | notrace_primitive) 11 | - Box values. (new_box) 12 | """ 13 | from collections import defaultdict 14 | from contextlib import contextmanager 15 | 16 | from .util import subvals, wraps 17 | 18 | def trace(start_node, fun, x): 19 | with trace_stack.new_trace() as trace_id: 20 | # Wrap 'x' in a box. 21 | start_box = new_box(x, trace_id, start_node) 22 | 23 | # Apply fun() to boxed value. This will carry the value throughout the 24 | # comutation as well as the box. 25 | end_box = fun(start_box) 26 | 27 | if isbox(end_box) and end_box._trace_id == start_box._trace_id: 28 | # Extract final value (== fun(x)) and its node in the computation 29 | # graph. 30 | return end_box._value, end_box._node 31 | else: 32 | # Output seems independent of input 33 | return end_box, None 34 | 35 | class Node(object): 36 | """A node in a computation graph.""" 37 | def __init__(self, value, fun, args, kwargs, parent_argnums, parents): 38 | """ 39 | 40 | Args: 41 | value: output of fun(*args, **kwargs) 42 | fun: wrapped numpy that was applied. 43 | args: all (unboxed) positional arguments. 44 | kwargs: dict of additional keyword args. 45 | parent_argnums: integers corresponding to positional indices of boxed 46 | values. 47 | parents: Node instances corresponding to parent_argnums. 48 | """ 49 | self.parents = parents 50 | self.recipe = (fun, value, args, kwargs, parent_argnums) 51 | 52 | def initialize_root(self): 53 | self.parents = [] 54 | self.recipe = (lambda x: x, None, (), {}, []) 55 | 56 | @classmethod 57 | def new_root(cls, *args, **kwargs): 58 | root = cls.__new__(cls) 59 | root.initialize_root(*args, **kwargs) 60 | return root 61 | 62 | def primitive(f_raw): 63 | """Wraps a function so that its gradient (vjp) can be specified and its 64 | invocation can be recorded.""" 65 | @wraps(f_raw) 66 | def f_wrapped(*args, **kwargs): 67 | # Fetch boxed arguments with largest trace_id. This ensures that the 68 | # computational graph being constructed only consists of other nodes 69 | # from the same call to trace(). 70 | boxed_args, trace_id = find_top_boxed_args(args) 71 | if boxed_args: 72 | # Replace some elements of args with corresponding unboxed values. 73 | argvals = subvals(args, [(argnum, box._value) for argnum, box in boxed_args]) 74 | # Get nodes for each boxed argument. 75 | parents = tuple(box._node for _, box in boxed_args) 76 | 77 | # Get argument indices for each boxed argument. 78 | argnums = tuple(argnum for argnum, _ in boxed_args) 79 | 80 | # Calculate result of applying original numpy function. 81 | # 82 | # Note that we use a recursive call here in order to also augment 83 | # outer calls to trace() with lower trace_ids. See TraceStack's 84 | # docstring for details. 85 | ans = f_wrapped(*argvals, **kwargs) 86 | 87 | # Create a new node 88 | node = Node(ans, f_wrapped, argvals, kwargs, argnums, parents) 89 | return new_box(ans, trace_id, node) 90 | else: 91 | return f_raw(*args, **kwargs) 92 | return f_wrapped 93 | 94 | def notrace_primitive(f_raw): 95 | """Wrap a raw numpy function by discarding boxes. 96 | 97 | Results are not boxed. Unboxing is a signal that the f_raw() is 98 | non-differentiable with respect to its arguments. Consider the computation, 99 | 100 | ``` 101 | x = 1.5 102 | y = np.floor(x) + x 103 | ``` 104 | 105 | What is the derivative of y wrt x? Autograd says 1. as np.floor has zero 106 | derivative near x=1.5. 107 | """ 108 | @wraps(f_raw) 109 | def f_wrapped(*args, **kwargs): 110 | # Extract np.ndarray values from boxed values. 111 | argvals = map(getval, args) 112 | 113 | # Call original function. Note that f_raw()'s arguments may still be 114 | # boxed, but with a lower trace_id. 115 | return f_raw(*argvals, **kwargs) 116 | return f_wrapped 117 | 118 | def find_top_boxed_args(args): 119 | """Finds boxed arguments with largest trace_id. 120 | 121 | Equivalent to finding the largest trace_id of any argument, keeping args 122 | with the same, and dropping the remainder. 123 | 124 | Args: 125 | args: Arguments to function wrapped by primitive(). 126 | 127 | Returns: 128 | top_boxes: List of (index, boxed argument). Arguments have same, largest 129 | trace_id. 130 | top_trace_id: trace_id of all elements in top_boxes. 131 | """ 132 | top_trace_id = -1 133 | top_boxes = [] 134 | for argnum, arg in enumerate(args): 135 | if isbox(arg): 136 | if arg._trace_id > top_trace_id: 137 | top_boxes = [(argnum, arg)] 138 | top_trace_id = arg._trace_id 139 | elif arg._trace_id == top_trace_id: 140 | top_boxes.append((argnum, arg)) 141 | return top_boxes, top_trace_id 142 | 143 | class TraceStack(object): 144 | """Tracks number of times trace() has been called. 145 | 146 | This is critical to ensure calling grad() on a function that also calls 147 | grad() resolves correctly. For example, 148 | 149 | ``` 150 | def f(x): 151 | def g(y): 152 | return x * y 153 | return grad(g)(x) 154 | 155 | y = grad(f)(5.) 156 | ``` 157 | 158 | First, grad(f)(5.) wraps 5. in a Box and calls f(Box(5)). Then, grad(g)(x) 159 | wraps Box(5) again and calls g(Box(Box(5)). When computing grad(g), we want 160 | to treat x=Box(5) as fixed -- it's not a direct argument to g(). How does 161 | Autograd know that x is fixed, when all it can see is 162 | np.multipy(Box(5.), Box(Box(5.))? Because the second argument has a larger 163 | trace_id than the former! 164 | """ 165 | def __init__(self): 166 | self.top = -1 167 | 168 | @contextmanager 169 | def new_trace(self): 170 | """Increment trace depth.""" 171 | self.top += 1 172 | yield self.top 173 | self.top -= 1 174 | 175 | trace_stack = TraceStack() 176 | 177 | class Box(object): 178 | """Boxes a value within a computation graph.""" 179 | 180 | # Type -> subclasses of Box. Types may be instances of Box. Subclasses must 181 | # take same arguments for __init__(). 182 | type_mappings = {} 183 | 184 | # Non-Box types that can be boxed. 185 | types = set() 186 | 187 | def __init__(self, value, trace_id, node): 188 | self._value = value 189 | self._node = node 190 | self._trace_id = trace_id 191 | 192 | def __bool__(self): 193 | return bool(self._value) 194 | 195 | __nonzero__ = __bool__ 196 | 197 | def __str__(self): 198 | return "Autograd {0} with value {1}".format( 199 | type(self).__name__, str(self._value)) 200 | 201 | @classmethod 202 | def register(cls, value_type): 203 | """Register a class as a Box for type 'value_type'. 204 | 205 | Should be called immediately after declaration. 206 | 207 | Args: 208 | cls: Inherits from Box. Type to box values of type 'value_type'. 209 | value_type: Type to be boxed. 210 | """ 211 | Box.types.add(cls) 212 | Box.type_mappings[value_type] = cls 213 | 214 | # The Box implementation for a Box type is itself. Why? Imagine a nested 215 | # call to grad(). One doesn't want the inner Box's computation graph to 216 | # interact with the outer Box's. 217 | Box.type_mappings[cls] = cls 218 | 219 | 220 | box_type_mappings = Box.type_mappings 221 | 222 | def new_box(value, trace_id, node): 223 | """Box an unboxed value. 224 | 225 | Args: 226 | value: unboxed value 227 | trace_id: int. Trace stack depth. 228 | node: Node corresponding to this boxed value. 229 | 230 | Returns: 231 | Boxed value. 232 | """ 233 | try: 234 | return box_type_mappings[type(value)](value, trace_id, node) 235 | except KeyError: 236 | raise TypeError("Can't differentiate w.r.t. type {}".format(type(value))) 237 | 238 | box_types = Box.types 239 | 240 | # If True, the value is Box. 241 | isbox = lambda x: type(x) in box_types # almost 3X faster than isinstance(x, Box) 242 | 243 | # Get value from a Box. 244 | getval = lambda x: getval(x._value) if isbox(x) else x 245 | -------------------------------------------------------------------------------- /notebooks/Autograd Exploration.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from __future__ import absolute_import\n", 10 | "\n", 11 | "import sys\n", 12 | "if '../' not in sys.path:\n", 13 | " sys.path.append('../')\n", 14 | "\n", 15 | "import autograd.numpy as np\n", 16 | "from autograd import grad\n", 17 | "import matplotlib.pyplot as plt\n", 18 | "\n", 19 | "%matplotlib inline" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 3, 25 | "metadata": {}, 26 | "outputs": [ 27 | { 28 | "data": { 29 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzsnXd8G+X9x9+n4ZlE2WQoRGFaBREgZSOGKWHYjLJcSsHsGtwCxUCPLkRbihhuyzD4V1rAQNqaQlkSI4AZYpUNoshACgpREkYGSuIp6+73x3MOtuMh7yT+vvPS62zp7rnvxfZ97nm+SzNNE0EQBEFoxzbaBgiCIAibFyIMgiAIQidEGARBEIROiDAIgiAInRBhEARBEDohwiAIgiB0QoRBEARB6IQIgyAIgtAJEQZBEAShEyIMgiAIQidEGARBEIROiDAIgiAInRBhEARBEDohwiAIgiB0QoRBEARB6IQIgyAIgtAJEQZBEAShEyIMgiAIQicco22AIAjClopHDzuAbCDL2nZ9tb/vRN1v7daru6/7+rz965viwaJVw3ldIgyCIIwJPHpYA/KByV1ek4DxwLg+XvnWq+ONf6RXXUzg78CwCoNmmuZwji8IgjCsePRwFjAbmAXM7GY7E5iGEgFnH8M1AQ3Ahh5ejUBLH6/Wbt5LAW1A2npl8vUm38eDRcYA/ov6jQiDIAibPdbNf2egANge2K7Ddls2fXJPASs7vL4E1gJrunmtBdYBDfFgUXq4r2VLQIRBEITNCo8engnsCfg6vAro/LT/FfAp8D9rGweWo0RgBbBmpJ6ut0ZEGARBGDU8etgO7A7sD+xnvTwddvkciFqv94EPgU/jwaL1I2vp2EKEQRCEEcWjh+cChwMLge+hnL+gnvhftV5vANF4sOibUTFyjCPCIAjCsGJFA+0KnAycBHitjxLAYuAZ4KV4sGjZ6FgodEWEQRCEYcGjh3cCTkcJws6AAbwAPAo8BdTHg0VyA9oMEWEQBGHIsKKHvg/8GDgUJQbPA/8CHooHi74cPeuETBFhEARh0FiRRBcB56ByBpYCdwB3xoNFK0fTNqH/SOazIAgDxqOHtweuAM5E3U8eA/4PWCw5AVsuIgyCIPQbjx6eDQSAs1BZuXcBN8aDRUtG0y5haBBhEAQhYzx6eALwC+BiVFG3KiAoy0VbFyIMgiD0iRVy+gOgEpgBLAJ+Ew8WfTaqhgnDggiDIAi94tHDOwB/QUUZvQkcHw8WvT66VgnDiQiDIAjd4tHDNuAC4HpUUboLgDvEqbz1I8IgCMImWM7lu1ElK54Czo0HixKjapQwYogwCEIXEnpkCrAHsNYd9L812vaMNB49fBjwD1RTmh+jZgmS8DSGkAQ3QehAQo94gddRHbtM4Ch30P/U6Fo1MlhLR78AfgvUAyfGg0Wx0bVKGA1Gui2dIGy2JPSIHRWP3wocCfwXWJTQI3NG1bARwKOH84AHgN+hWkfuLaIwdhFhEIRvuRDYB/ipNUs4EcgDfjWqVg0zVjmLF4DjgZ8Bp8eDRRtG1yphNJGlJEEAEnpEAz5CdQbzu4N+03p/EXAUMMMd9LeOoonDgkcP7wI8geqHfGo8WPTYKJskbAbIjEEQFPsCOwJ3touCxSJUI5kjR8WqYcSjhxegZgoOwC+iILQjwiAIilKgCbXO3pGngVXAaSNu0TDi0cMHAHXABpQovDPKJgmbESIMwpgnoUeygRLg3+6gf13Hz9xBfwq4Hzg2oUdyR8O+ocajhw9BdU77AiUK/xtdi4TNDREGQQA/MBH4Zw+fPwnkAHuNmEXDhEcP7wuEgDhwkLTTFLpDhEEQVGP6FKrTWHe8Ym0PGBFrhgmPHt4N5Wj+AviedFMTekKEQRBgIfCKO+jvNkTTHfSvRiV8bbHC4NHDO6L8JQ0oUZAy2UKPiDAIY5qEHpkO7I5ac++Nl4H9E3pki/ub8ejhqaiZgg0lCvHRtUjY3NnifskFYYg5zNo+3cd+L6PCVguG15yhxaOHs4GHADdwbDxYVD/KJglbACIMwljne8Ba4O0+9nvZ2m4xy0lWc507gQOB0niw6NVRNknYQhBhEMY6BwAvuYP+vnoMfAKsBvYefpOGjF8CPwR+FQ8W1Y62McKWgwiDMGaxymvvDPT5JG1lQ0eBXYfbrqHAo4ePQFVJXQT8YZTNEbYwRBiEscy+1vaVXvf6liiwq1VXabPFo4fnoiqkfgCcL70UhP4iwiCMZfYD0qg+xpnwAapPw9xhs2iQePRwDqqshwPVT6FxlE0StkBEGISxzP7Ae+6gvyHD/T+wtpvzclIl8F3gjHiw6JPRNkbYMhFhEMYkCT3iQDmSM11Ggs1cGDx6+BhUT4nKeLDokdG2R9hyEWEQxiq7oHoav5bpAVaBvc/ZDIXBo4dnoEJT30VFIwnCgBFhEMYqC6ztG/087gPAN8S2DAorX+EulP/jh/FgUcsomyRs4YgwCGOVBcB6YEk/j4sCBQk94hx6kwZMOaqRUIX0aRaGAhEGYayyAHjbHfQb/TzuQyALmDf0JvUfjx7eHrgeeBy4fZTNEbYSRBiEMYfleJ4PvDWAw9tnGDsMnUUDw1pCugNVMlzyFYQhwzHaBgjCKPAdVOOdLVoYgPOAQ1GisHy0jRG2HmTGIIxF2h3PAxGGr4F1jLIwePSwG7gR1bf5r6Npi7D1IcIgjEUWABtQhfH6hVUzaQmw41Ab1U9uA+zAebKEJAw1IgzCWGQP4N0BOJ7bWcIozhg8evhY4Bjgqniw6NPRskPYehFhEMYUVge23VCJYANlCeAZjZBVjx7OA24G/gvcNNLnF8YG4nwWxhrzUIlg7w1ijCWov5259D8PYrBcaZ33kHiwKDXC5xbGCCIMwlhjvrUdjDC0+yZ2AJYQcNmBaata8rZpSTuc2+SsX+2wmV8RSGZanC8jPHp4R+AK4L54sOiFoRxbEDoiwiCMNeYDBt8WxOs3ubaXVjYZBzLO/vCVBIqvTxm2nV/62pP1zppZmGhMzW7giJkfMyPgWokSoBeBhwkkB5uVfDPQDFw+yHEEoVdEGISxxnzgY3fQ39SvowIuDTgEKJ/s5LgVLf/CxL5/yrDV3f3pgqnrUjkzp2Y3PJ/vaF25onHCkYviu7sWzvzkv76JX85BdVD7AwHXe8A/gDsJJL/uz+k9evgoVNmLS+PBoi/6Zbsg9BNxPgtjjfn0Zxkp4LIRcJ2GmmHUAYdqGjeD+WlD+uinbv7ogE/XpXJmAieX3vPcoSfd+fIPU6Z9O9Cii1futGdlzH8UMAu4GGgCgkCcgOuPBFyzMjHBo4cdqD4LS4Cq/lysIAwEEQZhzJDQIy7AQ6bCEHAdhurudh/QBpwFuAkkK0zyPmxNp74DlAE3VNSGHmg/rKI29A1wCqqmUk1lzP8FgeTNBJL7obKuHwQuAj4l4AoScOX3Ycl5gBe4PB4sas34ggVhgIgwCGOJ3azt+73uFXDNIOD6N/AMMBk4DdiDQPJuAskmANM045qmzUU5ojfpf1BRG/oY+AWqZMXh346djBFIngHsDNQCPwc+JOD6vrVc1QmPHp4I/BZ4HpDmO8KIIMIgjCV6j0gKuDQCrjNQFVSPRt3YCwgk/04g2SkZbmXTp5OctmzbeOeU6ypqQz2Fjf4F1djnmsqS4s43/UDyfwSSpYAf+Ab4N/AgAdfULmP8ApiCKqktGc7CiCDCIIwldgPWAJsWnAu4JqNuzjVADJhPIHktgWRz110rS4q1ZQ31hwAcMevMHustVdSGWlBP+99FCc2mBJIvoUp0XAEUAVECriMAPHp4W9SS0z3xYNHbmV6kIAwWEQZhLDEfeM+qd/QtAde+wDuoG/PlwEEEkh/1Ms6h61pXzQaw2xzb9nHOe4AvgAt63COQbCOQvAHVg3o18CQBV2U2rQFAA37TxzkEYUgRYRDGBAk9Yke15Px2GUktHV0ERIA0cCCB5I0Ekuk+hvtxQ3rdN9bXnt52tJaZ/gocXVlSPLfXUQPJ94C9UJFHl96bde1Z22vL74oHiz7vwx5BGFJEGISxwg5ALu3CEHA5UDfgm4AQsCeB5Ot9DVJZUrwNcEJLuvEuVPhp7zd7xR2ACZzb556BZBOB5E9+nzrt9V21z1icdcXxBFz7Z3AOQRgyRBiEscK3jueAawJKDC4ArgNOJJD8pscjO3M6KjH0L8BSMhCGitrQ58ATwFmVJcV9/s159PBef00X7f3T1EW32TVzPfAcAdfpGdonCINGhEEYK8wH2qZn/XQD8DJwGHAegaTeNeKoD34AvFlRG6onQ2GwuA+YDRyYwb7XAqueNfbUUX6Hl4B7CLiuIeCSv1lh2JFfMmGsMF+jeWmW7bMIMAc4kkCyX53PKkuKd0BFEP3TeitOHz6GDjwGNAIlve3k0cN+lGhdGw8WrSeQXIsqhfEXVOjq/QRcef2xWxD6iwiDMCbQaN43x/aaB3Vz3o9A8tkBDNN+U7/f2i4Fpib0SF+Zy1TUhhpQy1cnV5YU91aj7CrgS6B64zuBZAqVYX0pcALwYqblNARhIIgwCFs3AZfW9puCgEnOFIf21TJgn0FUOf0B8HJFbWiZ9f1Sa9tXyGo7tcA0VDb0Jnj08AGo2cIN8WBRY6cPA0mTQPJPwLGorOn/EHDttukogjB4RBiErZeAKwu4o82cehVAQ/rIn/S3qmk71jLSrsADHd5OWNvZGQ7zBGrGcnwPn18FfE3H2UJXAskQKltaA14i4Doyw3MLQsaIMAhbJwHXJNSN+Jwm45CnAAwmvDmIEY+xto92eK9dGNyZDFBRG2oCFgPHdi2R4dHD+6FqKt0QDxb13uAnkHwX2Bf4HxAi4PpxJucXhEwRYRC2PgKu7YFXUU/WZzSkj1gJfOkO+r8cxKjHAh9U1IY+7fDeCmubkTBYPGrtv3uX968CVgG3ZTRKIJkADgKeAqoJuG6QiCVhqJBfJGHrIuA6AHgNtZb/PQLJe+lvD4YuVJYUT0aJTKfqpu6gvxm19NMfYQijkt2ObX/Do4f3AY4AbuxzttCRQHI9cBwqUe8y4F8SsSQMBSIMwtZDwPVDVDOdtcC+BJIvJvSIE9iFwfV4Pgqw03kZqZ3lZO5joKI29BVqNnNch7evQtVI6n8TnkCyDfgpcAnwfVQy3Db9HkcQOiDCIGz5qJpHvwEWoW66+xJIfmJ9ujOqYc5gheFrVNOeriTo34wBVNjqHpUlxTM8engva/zKeLBow4CsUxFLN6GEYVdUxNJ3BjSWICDCIGzpBFw5qAqmV1vbhQSSazrs0b6WPyBhsEpYHA4srqgNdZchPRBheMraLkQlra0Fbh2IfZ0IJB8BDgaygVesDnSC0G9EGIQtl4BrJqqz2Y+AXwNnEkh2bX25B6rYXf0Az7IbMB0VTdQdy1FJbjn9GPNd4OsmW87JqNDVW+LBovUDtK8zgeSbwD7AMlT57rOHZFxhTCHCIGyZBFwLgDdQpbRPJJD8PYFkdx3O9kT1YGgb4JmOsLZP9/B5e8hqxpnI1sxjsd1MH45pNgG3DNC27gkkP0fVZKoD/kbA9QeJWBL6Q2+p+YKweRJwlQB3odb9D7Di+jchoUdsKGG4bxBnWwhEK2pDK3v4vGMuw6c97LMJq52TXp+SWnvavMal/3zulvJVVWV1GlAInAHsD7iABpRf41Hg/vLqwpaMrQ4kkwRcxaglqisBLwFXKYHkuozHEMYs8hQhbDkEXA4CrmtRRezeBvbqSRQs5gETrH37TWVJcT7qyfupXnbrV5JbO+FtjvQCHLQ6sqyqrG4eaknsGaAY+ACVYf0aqrrqPcDSqrK6s6vK6jL/m/22xtIlqAS9N8QpLWSCCIOwZaD8Cc8AOqrxzWEEkl/1cdSe1nag/ZIPQkU09eRfgG/7R2csDB49PDnpnHh60jFhzQQjfSSqreh8oByYXV5d+P3y6sILy6sLT0VVb10ILAH+BjxbVVaXeQG9byOWClGzkNcJuE7J+HhhTCJLScLmT8B1KPAPYDxwhpW0lgl7AingvwM88xFAM6ofQre4g/71CT2yjn7kMqAEID9bc72EufwI02iOarac48qrCz/bZMfqQhN4uqqs7hngLJQ/4t2qsrrjyqsLX834jIHkiwRcewL/AmoJuPYBrkw0hzRgO6AFSLiD/q7Oe2EMIjMGYfMl4LITcP0SNVNYC+zdD1EA1TshOoib3ULgRavGUW9kHLLq0cN5wEXbpmwvjc/e51AwaF2/6LruRKEj5dWFZnl14Z3Ad4F1wDNVZXVHZXLOjQSSK1CVXW9NGXMu/br1mq/ATAIfouourU7okQcTeuSghB7Reh1L2KoRYRA2TwKuecBzwO9R/Q/2IpDM+MnfurHtycD9C3MAL737F9pZTuZLSWfnGkw9sSFrJ5tj1hdAg2kkM+nqBkB5dWEMOAD4CHi0qqzutEyPBSCQbE00h2JftlalW43tJ4yzP2bPtz/+NzDOQSUI+oEXgMUJPbJjv8YWtho00+wuwk8QRomASwPOAf4EGKhyD/f2EIoKgK/GNxlV9mIn6zV125aZ0//v018XL5oafu++aeE3UBFMS1Hr+e9ES6Op3syoLCk+G7Wm76uoDX3Q274JPXIncIQ76O91Ocmjhx2YfPKDDVkud9qWp6Ed0Lz2j1ehspW3r6gNZfzHWFVWNwF4GDUDOLe8uvBvmRyX0CMXATcBoQmOu6+c4Hjgz6geEP8Gzk80hxqB81EJgznARcAd7qBfbhRjCJkxCJsPAdcMVGjmHcDrgI9A8p6uouCr8U3z1fhO99X47vDV+GKoOkMvAn9FReAs3K559u4A7+V9PA4V6XM5qs/Bf4A1vhrfw74a36m+Gl9uD9YsBFaSmX8iAcxI6JG+fHYn7pKye+ak7ZM0tJ+XVxe+hZqRzAO2z+A8GymvLlwHHG0df0dVWd2pfRqpRw5DicJDwAkTfv+3D1DXeTkqaulDd07xce6c4ptRs6UXgP8D7kroESnON4YQYRBGH+VLuBCVnfw94GLgcCtRCwBfjc/jq/Fd6avxvYpqfXkPcCIqWudKVL2h7YC8aGl0zs9XnH0XkL7+85/5oqXRmagyEdsBJwP3otbq/w4kfDW+a301vmnt56osKbbzbRmMTJ6UE6i/pRk97eDRw9p4Q7visEanYWK+xLdJbe1LVUf0cGiPlFcXNqNafUaAe6vK6o7rad+EHpkE3I1agvqRO+hXM6ZA0iCQvBHYCzWj+gcQducUZ6GE52pUbsWrCT0yr782ClsmspQkjC4B196oHgQLUE7mcgLJjwF8NT4XcBLqxnSQdcQbqCJ0YdSSUHf1i0jokRAw1x30+7r73Ffjs6HqCpWjbq6NKH9G5ZmPz90dNWP5YUVt6B99XUJCjxRZNu3nDvpf624fjx7e/4hG58u7ttrTNrRdyqsLP2r/rLKk+H+oXg893th7o6qsbjzq/253oKi8uvCZbmy8HTjXsrH7hkUBlx34CXANqkPcr4BbE82hw1Ai2gYc7w76XxmIncKWg8wYhNEh4JpGwHU7KolrFqqf8kLfvG0/9dX4jvbV+P4BfIFaHtoG+CXgiZZG946WRn8bLY2+1ZMoWPTqeI6WRo1oafS5aGn0JJR/YjFwLfDOqgkt51q7bXKD7YE+k9w8Kdtvfa120nBrR1GwWAwUVpYUZ2V4vk6UVxeuR82YPgIeriqr27eTcXpke5Qo/F+PogAQSKatnIddUAl3fwTed+cUO6FtXyAJPJfQI/1zeAtbHCIMwsgScE0g4AqgykecD9yUtNkKfPO2/dg3b9tK1E02jFrK+RuqIJw3Whr9Q7Q0ujSTUyT0yExgJhlGJEVLo7FoafQEVPOc8W0O8/wNOW1f3n300rUZXlWvvZ89enjePi2Ow1LQ6ET7TTe7PAWMA/bL8HybUF5duAa1HPUl8HhVWd2uHT4OoPI5rslosEByKcovcxzqHvGoO+f46inOwPmosub3JfTI1RLSuvUiwiCMDAFXDgHXJah4+auApxZNGHeob962Kw6c634ZdRMvB15GVRydFS2N/iRaGn09Whrt73rnHta2X6Gq0dLoY3vFJu07fU228emshm2Ap301vkya3qxBJcJ1O2PYpcV+3bZtdppsZtByGnflOSCNcgQPmPLqwpUoH00zsLiqrG47yy9wGnCrO+jvqd7TpqiM6UdREVM/AXy59jfrZmWfsNLG2oeB3wB/72dVWWELQYRBGF4CLhcB1+UoJ/GfkjZb9LwZ06/yzdt2QnDK5OeB64H1wAXAzGhp9MRoafSRaGl0MBm4+6BCXd/p74G7fDZhLxuarc1h/MEa5x1fjW+f3o6xQjm7zWXw6OEJvlb7Cc2a2eQybDd2d3xFbSiJehLvtwO6K1ai3OEoZ/vTLYZ5Ker/4uYBDRhIpggkq4AdgKBNaz1mZvbpx4+zP/Jf1PLfcwk9Ih3jtjJEGIThIeCaQ8B1I7AsBdc/Oi5/9UL3rOcPnOve97XcnKuBHYHfATtFS6P7R0uj1dHS6JreB82YfVEZzwPpiLYQaNjz40m/tcZpAup8Nb6+btrdZj/v2WK/dk7abl9jM28try7sLYP6KWDPypLiab3skxHl1YX/BY62wTYaXJA2zSfdQX+izwN7I5BMEkheCczVNH470XnH7MnOPwCte2u0vL9cf6ZbJ7+wZSLCIAwdKuz0SAKuf6Xh0zdzsi+5aPrUL/fyzEn+ctqU3VY6HT5UyOSBwHbR0uhV0dLoJ70P2j+sUtv7oJ7AB8JC4PmK2lBLtDT6PqoE9sdAyFfj6y1XIEEXH8M+lz3u8LY6zmnUzNZZaVt3voWOLEZFAn1vgHZ3ory68D8L8uw3Z9k0+1uN6e2qyurGDcW4BJKrCSSvAubm2V/5+bSsK9dqNEwH4731vzrrbwRc04fkPMKoIsIgDJ6AazsCrt81adrSF3Jznvjl1MnF+891p86auY39ufy8WWlNexzlzJwZLY1eGC2NvjwAv0GmFKCqiHYbNtoblSXF81AzmY3VVKOl0S+BQ1C+j7/7anxlPRy+HJhtCRMAO6bsv56VtmWvsBs1Vs5Bb7yF8lUMejmpnVlZtgVtpvnlypS5M8ohPX6oxiaQXEcgeX227aNZk51/KrdrqxuSbaVnr2s7aYV51aRaAq7jCLiyh+x8wogi1VWFgRFw7QicuMJuP/XNcfm7PZeXSyQ3J91is4FptqJpYeChGWvMJ2/+v/RkVCbtT2LXendAtcqcDkwBclGlF5wop2kjyufwOSpy6VNUTsG73vpYOgPL2iN7BjJjaHf+dqqPFC2NJn01viNRlUlv99X42qKl0b92OTaBKtE9FfgKYF6b/ZINmpl2qGzsXqmoDaUrS4qfARZWlhRr/SmP0R0JPTIVOMyhadej+l0vQjmkjyp8vjyJmt3sbr28fPszmQTYUbMXE/Wz+MZ6LUe1DG1/fQazPvPWP39bmx65x8a6f61rO/PI5vSC70/JuvEUu7Y6ScD1b1T/jDoCyYF20RNGGBEGITMCrixg31V2W9GbOTmnvDd5oue13ByWZKnQ+6yU8eWOn/Pcoe+nP9kvZqadBjuhMpLvBjqWU/gGVWriK1TcfSNKEFIokchHNdfxAkUoJyrAuliB92VUTZ9/eetjyR4s3RdViXUgS1RHoQTp464fREujzb4a30mo+kR/8dX4WqOl0Xs67NIxl+GrMy5+6sx90s4JMWfbw7fesrAxw/M/BZyCigSKDsD+jnwfdYO/v7y68N3bzl/cZmr2f+a0rKlvdY7bkJXa0LEEx1LUzySOiuRKo0TBhip17kKJ+G6o7O6OYaqtsQLvJ0A9aG/mLDhrmene+/SVzX9tdjnvfX+c/d8naRpnAUkCrsXA48CTBJJfDPL6hGFEhEHonoDLAez6bnbWMfVZWcd+Pnni/Gh2tnMJTrZZo7HtSiN9+HJzRdnytnUzvmGcsw23pqJU2vkciKHq7cRQ5S5i3vrY15maECvw2oA5qHX+g1DF3u4AbokVeB8C/uytj73e5bD9gNf6W/StsqQ4G7W+f09PT+vR0miLr8Z3AvAYcJevxtccLY3eb33c3rBnNvD2jLTt6hZMNtjMC/thRvsS1hEMXhhOAT5ZH/7Z/2IPN1x8KFy0avIujg92PW+bNxbo4wo+WvS7KWtjTwJRb31sfaaDxgq8WahrnIOq71RgvXYD8/jmt+60ax+FyV1wdk5y0tn+NV8fusr8rOa1ca767LxtWgpzJradbHOYEHC9jUqiexF4iUBy9SCvVxhCpCSGAAGX1qKx7TvZ2Qd/5nQu/MJu32dVi3PemoYse943NmavNpmzyjTnrCLtauj0MJFCPZlvvPFbr4+89bGGoTYzVuDVUDV9SoEfAhNRN+nfeOtj71rLJ18Dv3QH/X/oz9iVJcULUU/sRRW1ocd729dX48sHnkDNTo6MlkbrEnrEjVpeKftFU2N0rxbHy/9zGG/8+dbD9+6nHf8FllfUhgac05DQI1NN0/yybeW7Lze/fvt81AzsFeCWV/cJrGnKnfZ3a9fjy6sLe2xC1F9iBd5slFh8B82+S1bBMcdkbXfoHtgcttSnz9H6yZOYrRuwOY212RNTZu7klCvblbJnu9rImtAWszvNl1C+lreAKIFk5j2uhSFFhGEsEXBpq222aa/l5uz3tWY/sKXRvl9Ts2Pntg32Say32yev1Zi5xmTGWshJgaHZaHPk0uIc19iaNXFpc+7UFY1501c35U5b3ZQz9avGvG1WmzZH+y9QE2o9ej0q+zYBrCqvLuytbMWAiRV4x6NKQl+OWuq4M3/htXW2vCn3AQe6g/6X+zNeZUnxzaiyEVMyaMyDr8Y3EVW8bi5w8BOx26KoLmjXVG5oPnpem23Bm9lt+9xz0xFdZzR92fFH4EJgckVtKNMlqI3ECrxazp5n/dG57X6XNDx/DcY3Sx8CrvPWx/7Tvk9VWd0OqOxyDyqp8G9Wp7ghJ6FHZpimcR1oP8I0WtNfffBG8/v/XGU2rt4ezALQNpYBceSmjeyJKVvWuDac+WnDkWMsdeSlP8we3/aWI9d4D/XwsYRAUrrMDTMiDFsTAZf2Zk72Nh/Zs3ZvXW//rtFk89Fq24Fm+yyaxk+0N4/LzmqkMl6HAAAgAElEQVTO18Y155HXmo/hyCflyCPlzCflzKM5e3xba9aEljbH+HTanus0bY6eSlJnSitqvf494F3UU+vr5dWFQ+aEjBV4J6F8GRXZ809rdHr8Tk2zTehP17bKkmINlZH9YUVtqDjT43w1vtkoJ3cWsN8TsdtebjLNF0LJ1A9WOoyl1996uKd/VwOVJcVHomYjR1bUhjJpErSRWIF3MnBnzoKzj3Ns40tvePLyhd4Po3Xd7VtVVjcZ1QDpMOAB4Pzy6sJMS4D0m4QeKQB+DZyKKsb3oNnWfMeGx3+WwEh/B9gFzO9oNnY3TTyYWqcy3/acNFn5aZz5adOWZay1O40v7VlmwpZlLHHkpGM5k1LvOfOMT4CvCCQzCVIQekGEYTPmqRtm2hpabNOa12ftlm7J39VM5ReY6fy5Zjp3BuncyaaRO4627ByMXCdGjt1m5miamY2pZZG2Z5GyZ5G2O2mzO8BsA9JgtmHSBmYalRCrmaA1ARtAW4fmSGpa9mq0nK80W95KzTZhuaY5V6FCKVdb2wbrYNN6gXIcj0ctW2yDcsJuC3wH1ei+PfkrCTyLciL/u4+kr4yJFXj3yjvs6hfNpm9yml75023AJd76WK/NeNqpLCn2otpbXlBRG6ruz3l9NT4vKpR11WOxmxu+SttmvbE+Pf0/2akz7rnpiP60IW23JQ/1f1xdURvqM5qpnViBd3/gn6DNGFd8Syt2xyNzrju412J3VWV1duAyVFXZlcA55dWFT/fX5v6Q0CM7o7LcS4GJpmnGW4zGZxINH7/57pq6ZWmzLQfTzMtuS0/NaW1zZ7elp+ak26Zn0zbbkTam29LmBNJars0wNZthYjdNbIaJ3TBwONI4nWmcDiPlsKebHTZjvdNhrHXa0qvsdmOl3W4utznN5TaH+YXNYaxw5BgrcqakVjqyjYbeGkGNRUQYeqHmZ0fYbQ3mhLQta7KDrIkatsk2siZopm08mjZeM+35wEQwXaZpm2yadvUZtjyw54ItB9OWpWF3gs2pmXY7ms2Oqdk0bHYNmw3smg2bhmbXNGxomgZoHbagbdyaqHdN63vANDBIY5hpDNPAMNOYpEmbbbQaLaSMZlrTatucbsSg3w9TJmrt/FNUWYv3UGvA7/VnqaOqrG4qqtvYEcCRKAdmEhVGeXt5dWGvXdL6IqFHJpmmubpt2WuvNr991/4ox+bJ3vrYqr6OrSwpvgy4AZhbURv6vK/9u+Kr8e0PPHtt/OLmuRt2nFjbmEr+vup7E/s7Tgd7wsDOwI59ha1afpdLUPYvzd71lKuzdvheDarnwqJMzldVVtfem2JHVNTVZeXVhf8bqP1dsWZk81CJh7sA33FozoI5+QXbuvN3yt8m14Ndc9CabmZVS4KvmxOsblnON61fkzKGyM1gmtgNE9vGrfHt96aBAxO7aRg2zTTsGGm7ZqbtGG02zUzZTTNlw2i1aWazHbPZhtmkYTYCjTbNbNI0GjXNbNQ0s0HTaNQwGzU2bhs009ygmWywmeY6zTDXZ6WNDflm64Ysw2hx5qdbpz2Y2OxuwmNKGCLldxhTc2ZomnXj3fhPs3X4WkPD1vlrbfMsImmqH57WH/sM01hlYiQwzYSJGW81Wj5Ktq76JJZ8Nf518zIT9dQ/pcNrOt92GNsRFacPKqTxdeBJlNP2jYraUEb+hKqyuvZeCOegmu1ko5Y1AuXVhfUZX0wHEnrkWOAR4OD1D5+/Lapc90rgWG99rNcIn8qS4ueBSRW1ofkDOTeAr8Z37C8+/ekj+zQV8NvWDcG7/3z0lQMdq7Kk+MeobnO7VNSGPuxpPytq64+oxkYPAWeNP/4vF6OqqU53B/19imI7VWV1OcDPUOXNncDtwJ/LqwvjA7yGKaiucEejMt1nWh+lUQEL9ahIri/yHBPW+Cb550zNdhfk2vPn223OjQ2BTNNYYWAsMUwjYZjpFW1m6/LWdPOKhrbk+mTr1y2rW1a2rm5Znm41mrNQ+TA5fJsbk4Np5joMY2KO2TYjm/QMm2lORs1q803INdFyDDSn9bIbmmZLo9nSmk0zNI20psHGe4Vt4z3B1n7P0GybfG/rsF9332vqnQ7Hddh2OMbW9RjNjoZGU9vbvz72zn/+fiA/l0wZU8LwdNmfWsY5J2epO6qBiYlpmlj/TDAxTWPju2o/62uM9q+tDwzD+towMdNoZkrDTIHZamI2axpNGlqDhrle0+zrNGxr7ZpztVPL+yrXNnFljiPvm2xbdpPTltNs02xp1Lqrtd7T69cbt+0hmQk9Yketc2eh/qizULkDLlTkzkRU4tIsVJjhttZ2HuoPCGvct1EO1ReAZ91Bf6fIIuvJbzaqqc7eqGJt30VNXpaiOqPdU1EbyjiHwFrrrkDd3HKBu4ArrDLSGZPQI7cCZwGT3UF/S6zAuxfq6TcPOLKj87XLNU0EVgHXV9SGftGfc3a5Ds2dl2pdkJXnOHGHivsanU1nDDS7u7KkeDbKeX9lRW0o2N0+sQJvDqqL3cnAn4EKb33MSOiRF4B8d9D/3QFexyxUee7TUT/XB1CNlF4qry7sdbpphfyegmqsdCgqj2I5avb2EsrHVF9RG+rV/5PQI9NR/TR81msn1O/rTDrnUHSkEeXTan+lgFbTNFOmssNpgiON4TQ1nCamw3rfboJdM7FrYLPm6raOj4S2zezB8OWG0E0lt1yX8TLjQBhTwiB0xhKU7VB/fHsCftR0PxurdDPKF/BgTwXpKkuKp6KWhn6EEgqbddw1FbWhFzO1paqsbhqgowRiFVBeXl34YIbXoaGWuT50B/3HtL8fK/DOBepQs54ib31sE3sqS4pLUJm5B1bUhvoVydSRsosW/7gIZ/UeeQ7O2v7XfJG1+vfR0uivBzpeZUnxG0Cqoja0f9fPYgXePFSYbiFwmbc+VgmQ0CO5qATCm91B/+UDPTdAVVndHOCnwI9RT9hfAA9a5321Y/nwypLiGajS3OcD01A/i3+hfnfeGmwWdzsv/eyFXBt8x67hNZVQzNAwp6e1thlpzZhmasZ4E/I1U8uxYXfaTZvDbtpsavFV0dEp1v49JqQ1g7TWRpuWxsAwDM0wDEzT0NJpQzMNAyNtaGabgdFmakbKwEyZmpEyMVtMzFag1dDMlKkeDNu3LaZmtphgbc1m1P5NaDSZmE0aNIHWZGK2mCataDQZGC1pzWg2TVpMzCZDM5pMaFqvNTebmM1XBfRhd66LMAidSOiRbOAAVE+E41F/gOuB+1AdwN7r6djKkuJZwJmom/t01FPirypqQy9kev6qsrrdUQ169kTdXM7toYdBR5t3QmVRl7uD/ts6fhYr8M5GdWKbCxznrY91cq5WlhTfh8p4nl5RGxrwH9wvy59JFNhssw7Nd2rXzv5b6MUJbxUDF0RLo/1yZnew69eofsuzK2pDG/soWDOFR1HJeGd662Mbs68TeuQQVG+HY9xBf2ig19IRq/heEWpmcjRqVmcCUdNoeDfV+LTbSH12AJhZ4HwazXEzZtPjPYmBtYyYg8pwn2S9Jnd4tX+/cTnTxJhqwjQbth5rPbVpKVocDTQ5G2hxNJqt9qbWlL2lMWVvWZeyt6xNa21rDC292tCMr9K2tq/SttSqlK3ly2Znw5ff5Hy1cl3uqjXAumhpVCKaEGEQesF6Et8P9SRYgvqDfhK4yh309xifb0XWnA38HBWN9A/gsora0IpMzltVVudA5Sf8DvX0eUJ5dWGPa+0JPXIxajllO3fQ/1nXz2MF3unA0ygfyUJvfewly84cVGmOBypqQ2dnYlt3/PiixQfv1up4fm12+t0zc3N2b9VSpx1XcPGpqBvpidHS6MP9HbOypHgX4APgJxW1oSrrOrJRy2NHAGd762N3dzwmoUeuQjVBmuwO+r8Z6PX0RFVZXT6wn2kaB6abXz+lreXtAsxmzebcGUfu/tjsk9p3TVmvtg4vJ9+u/fdKWmtrStlbUk2ODVpj1jpnY1Yyp9nRQLOjkWZHAylH89qUrWVlq7050eJoWtqQ9c0nzc4Nn5qa2V7D6Su5wQ8OEQYhIxJ6ZDJKIC5DPcmFgZ+7g/7/9nSMJRA6cAXqRnElUJXp0kJVWd3BKKd0HnBmT0tLCT3yJOBxB/0FPY1liUMEFUp7iLc+9m5lSXG7w/qoitrQk5nY1B2X/+Tpd9xttt1X5hmH/CQr53ngsqO8F1ajwnLnA9+Llkb7vUxVWVL8IfBVRW3okFiB14Fa7z8OOM9bH+taxI+EHnkOcLmD/j0Hei0Z2LQ9akZ3MPCcZp8eyJ7wowbUjGwKKjjBhSq348Ba30f9/JvaXwbppi/Gf5a33PXJ7NX5iXnrstds35i1brtWe6PTsBlY+32AclJ/3OG1JFoaHUifDaEfiDAI/SKhR8aj1pMvR0UwVQK/6+qo7oh1M6lCPemGgbMqakMZ1UyqKqubjboh7gtcXF5d2KkTmWXP18Dt7qD/Z72NFSvwbota3soBDnx8/va/Ri2TbFNRG8oo56Er5T9dPHenlD2+1GF8dum4nO1ROR63u4P+Cl+Nbyoqx2Ea4I+WRnsU0e6oLCkOAL+xGcasI6Of/QYV//9Tb33s1q77WkuA31jnvnQg19KHLZp1/htRzt1LgbsyFXlfjS8XVfPqAOu1L8p3AWrW9jYqCbI9GfITeeofPaQfg9Av3EH/enfQfy0qzv5e1HLRhwk90mNtn4ra0P9Q6/gXodbG368sKT4sk/OVVxcuR0W4PATcVFVW97uqsrqOYSLHopzlD/Q1lrc+9jnKQU5a057GNI8HHhqoKAA40aocaKxwGL/o0OJzNkC0NLoK5ZhvBp711fh27ufw/wK0WWs33I66KV/fnShYfBcleBk7/DOlsqQ4H+VjqkJFrO1SURu6szdR8NX4NF+Nb1dfja/CV+N7CpW09wwqlHYmannxDFQY9IxoafSoaGn0ymhp9J/R0mi9iMLoIjMGYVAk9MiBwF9QZbIrUQXsesxKqiwp3g0VBbQTcHH7+nlfWFm6twPnAf8HXFheXWgk9MgjqPDZbd1Bf0Z5FLEC74IVrvyX3vXMyMltSX3/woef6rcPAODynzydNz1tW7fGZqy/turwSQAJPfI8YHcH/f72/Xw1vgLUDbUNODhaGl2S6TluOvGoZeNaUu79lyz/J3Catz7W7TUm9EgF6ml+hjvo/3Ig19MdlSXFO6Kii3ZBlbS4tqd8FWtWcARKrBfybUe7GCrX5WnglWhpdMj9H8LQIsIgDBorTPJGVPG3d4AfuIP+TXoatFNZUjwelWlbjHoKvaSiNtRn/SRrpnANyldxx8IJjitybdqXwG19LSN15dYTjnwNk30OjS2ts8HR3vpYv1NsL/rp07ftnLJf8Fp26up7bzoiAJDQI4uA/dxB/3Yd9/XV+HZFxfM3AodES6Of9jV+rMB78P+mTXz2o1lT7LktKd+FDz/VY3Z4Qo/8C1jQ9byDobKkeAHqhq4Bp1bUhhZ33cdX4xuP5WS3tvmoJa1nrGMXR0uj/c4mF0YXWUoSBo076G9yB/3lKMfotsDrCT1yVE/7V9SG1qNCYW9AVfd8zFqu6BWrAugvgT8A5y1tMR5AJfPd3+uBXagsKZ7Z4nTsld+aesymcgHusrKIM6aqrC5rdpvt7C/tRkvcYfyuw0crgFlWRNdGoqXRD1DLaPnAy74a3269jR8r8H4HeHjWN+v/h2mmm7KdvdY9wupD0Z9r6I3KkuIDUTkgG4B9OoqCr8aX56vx/dBX43sE5d/5J8oZfS9qqW56tDR6crQ0+lcRhS0TEQZhyHAH/Y+ilnU+A8IJPaJ3vUG2U1EbSlfUhq5ARTotBBZXlhRP6m7fjlji8CvghmlO7bAWw0wubzW6zWruhR8BtjXjci9DzT5OBa7vzwCf29NXjzO17I+d6bsilUd1XA9fjvJ5TO56TLQ0+i4qiTANvOir8fm77gMQK/DOQlVYbc5NpReiaU8AZ1SWFNu729/qBTGbIRKGypLiw1FJil8A/ora0BJfjc/mq/Ed6qvx3YUqq74IlWvyfyhRmBUtjV4QLY0+Ey2NDthnI2weiDAIQ4o76F+Kijq5H7gWuC+hR3qMXa+oDd2BypHYC3i+sqR4m77OUV5daB7jctwzxWHjkxbD9WZj+upM7assKbahxOjlitrQx8B1wK1ARazAm9FyVFVZXdZUw3bRSruRfjer7YouH3fs5LYJ0dLoh6jonC+Axb4a38kdP7f6TIRRoZ9F3vrYUlSZkFmo2kPdMZg+152oLCk+HgihQkMPuvvopeN8Nb4/oNp+1qGWjO5HBQTMjZZGL46WRl8UZ/HWhQiDMOS4g/5G1FP4L1Gd1p5J6JFpPe1fURt6ABU2ugPwUmVJ8bZ9ncOmaWWmabasaDXuA35VVVaXaYhm+3luBvDWx0xUddIHgT/GCrwlfQ2w2mZclmdqeR9ktf27/vqirm0x25P4ZvV0vLW8ciDKH3O/r8ZX6avxOWMFXicqEsmHqgz7tnXIo6hWqRf3MOS+qMinHrPSM6GypPg04AFDM6MPHrx80d1HL30MVY78ClROwamoCKJzoqXR56Ol0WFpwiSMPiIMwrDgDvpNq71mCSqU8jWrWUu3VNSGnsZanwZetHIfusVKtjtD07T7m0zORIWqVlaV1WWSvXwJqkDdQ+1veOtjadTyUgS4J1bgPbSng6vK6rLzTE1fYTfMD7LS3YlRrzOGdqxQ1kNQs5VLMc1nG7K5FxXVc4G3PvZE+76WY/5W4JDKkuLduxlub+Cd/jQn6sr1pxZdYGLeu3pC65q/H75st/X5bdejEtMuBdzR0ujRVihpv7vKCVseIgzCsOIO+u9H3QDHA68m9EiPN92K2tArKGfweJQ49CQklwPjgBusip8/Qq2J31FVVndST+NXlhTvYY1/a9fcBW99rBnlPF8CPBwr8HbrHG7SzItyTW38u9ltzy25rijRzS7tM4ZehQEgWhptjZZGfwr86IfPG/vmt1Dy9vbaE6dc6bizm93/iopo6iRGViHEPYE3+jpfd/hqfPPPv+zgF+2GdltiWpP2+H5fmG0O81Zg92hpdI9oafRP0dLoFwMZW9hyEWEQhh130P8aqmrrSmBxQo+c1dO+FbWht1DOTDtKHDrdoBN6ZAZqSeUf7qA/ClBeXdgCnIByvv69qqyup2S736JCKf/S3Yfe+thaVELaeuAJK1N6I1VldVOcJlcvdaSJOdNdfQvt19qKitTpUxjauf/atsnHv2Y6X9tZSwRPth0FvOqr8e3TcZ+K2tBaVB7HaZUlxd/p8FEBqmTIm5mez1fj29lX4/uV727fe7t/7Hp352Xj/SumNCVe3H3V99N20x0tjV4aLY0OallK2LIRYRBGBKu4XXtntTsTeuQPCT3S7e9fRW3oA5Q4tKIc0nt1+Pg6VIjqVR2PKa8ubEDlRcSAh6rK6jqVq64sKd7f+vw66ybbLd762DJUlnY+8KTVRxkAA/P3Nsh9MSf1xqfXFb3Vy+WuoBcfQ0diBd4fAjcBD0//xpyHpv0QFfL7mq/G96Svxnekr8bX/v8URM0aftthiAXWtkd7rIiiBb4a31W+Gl8UqMfkdwe8P2Xq7ksmkrIbi2atzvW8fe77D0tEkQCS4CaMMAk94kQltZ2Him45u6c6S5UlxfNQkTBTgKNL5v18Bso5+3t30N9tr4OqsrptUPWQpgIHl1cXvm+FeUZQvSe2r6gN9VjXqZ1YgfcQrM50wOF1h1TtaWJG3s5Ka3V5qePiwaJHe7nGMDCzr2J2sQLvj4Aay7ajvPWxJgBfjW8cKr/jZ6iifwmUH2XxqYvdh2S32a9AhZG+lNAjN6Mq2brcQX/aOn48sDtqiekA4DBU+KwJRGxpHix5ds4+2W22dlG6NNPue8LYQIRBGHGs3IbLUE//HwGnuoP+d7vbt7Kk2A0868qaNmfhrNI2m2b/CNjfHfT3+GRbVVY3FyUOTsDfvPaPR6FugKUVtaF7ejquK7EC78lAbZs9+4UXD7xxzgabNudvE5o/b9XYOR4s6vFGmtAjfwGOcwf9PYbexgq8pagw1OeBY7z1sU3Eylfjy0ItkZ2KWuLKcrRpHP/iLANoffigFYurlv7qAEMzjAu2u+Y11CxlJp07nSVQWcjPAM+c+fjcDai6R8ejyppfNVSNdIStBxEGYdRI6JFCVKLUVFTv4mu76yPw9sWLjpyYNS2UNtvsb69+9sKSuypv72vsqrK6AiBipFcZrevuGY9qYFPc35tgrMB7ev1OP7h7xSy/Lexcw4f5uRfEg0W9Nt+x+iIEgOzuIoViBd4yVLvMZ1HNg/qM9PHV+PJQS3Hzd0jkH3rg+1OLlsxpWHWZ89dTnnW99s0tM/+xDLWEtQKVYPg28E60NLqxyY/VSOlRYA/gZxW1oZs3PZMgiDAIo0xCj0xBZR2fDaxD9TF+EnWDm4l6Yj7bNI0VTy2/6+tkatXuqCfd3/ZVX+nms245KNX8Sh1mm82Zd+iJF9154UO97d8dVWV15wB/nb3sWXPSijqz1e787uFvvvhOH9d0HsrBPdcd9G8sCWH1VLgetUQURuUqNPXXJoDKkuJbXM6pPznSfQ7A6e6g/74+9l+AavIzCfhBRW1oSDq8CVsn4nwWRhV30L/aHfSfg1oPfwI4F5V5+zbq5nk6cIum2XZNplbtB9yNqvL5YmVJsbencStLirdLNT51K8b6VFb+McvsWQV/ryqrO74/tlWV1RUDtzdr5mu3ztlWG9/a2Oje8PVLsQLvWbECb28d4jfJZbD6T7+AEoVbgOMHKgoWP9smd957AC99+dBOVr+ETagsKc6uLCn+JSpiS0P5JkQUhF6RGYOwWWFVat0b9WSbBP5jZVJvpLKk+FTUUswElPP2b8B/KmpDbZUlxVOBs4BfoJytJTmTLn0XJTJ7YZUGt0Jce6SqrO5k1Fr8+1UTmpY22lh4+ZuL9i5MvHMbqhzEs8Cl3vrY+91cw26oLORT1j98/uPAT1FilgZ+7K2P/WMg/zddiV9Rd7NNs5U/EK+0mZiPoKKW3gAMYAaqT/OlqO5q9wMXVtSGVg/FuYWtGxEGYYuksqR4GqqY3nmoXsJpVP7BRGuXZ4HzKmpDnwFUldXloEThQlQS2xXAo1aC3EaqyuqmofwDFwKvPZbXekF9Vvot4I/xYNHlsQKvHVVr6Q/WuZ5ENZ15Dkh462PmZ6fd4HbO2XdZ62cvPNfy3qLdUFFVDwOXWLWPhoSEHnnGNE3X/fHrH0QVA5yACvFt5tvuaG8Av6ioDT0zVOcVtn5EGIQtGqu3QzGqkcwkYCmwuKI21G2Uk5X8dguqUVAcdWP/CNWfeAGqyUwu8GdAv2Fi05+Bc4B58WBRe1YzVn7DhUAZ3y4ZNaOa8YwbV3wLqXjEaPng/seAG7z1sX73fO4NK7LrK+ARd9B/rvX/cAKqYVKedW1PVdSG+tVOVBBAhEEYg1SV1TlR4ZpnoyJ92p+uV6CE4sby6sKYRw9vg7rBLooHi87tbiyrj8MCVD2o7VAZ26vHFd10ITbH63NuKPz+cFxDQo/MQvkyLnIH/bcMxzmEsYtjtA0QhJGmvLowhUqU+5fVMtSF8kd8Y/V7aOciVG+FG3oay2q1+QZdahVZNaFmDLHpHZlvbaV0hTDkiDAIYxrLx7Cm6/sePTwBtVT0UDxY9NEAhk6gMo6Hi/YqqyIMwpAj4aqC0D0/RjmXrxvg8ctQLT6H6+FrPhB3B/3JYRpfGMOIMAhCFzx6OA9VsuPZeLDo9QEOk0D9fQ3XctJ8ZLYgDBMiDIKwKWWohkEZtwzthmXWds7gzelMQo/koaKqRBiEYUGEQRA6YM0WrgCeiweLIoMYqr2Jj3vwVm3Crqi/3W5DcgVhsIjzWRA6cz6q1HWfvZ/7YNhmDEhEkjDMyIxBECw8ejgX+DnwQjxY9MIgh/sG1VRnOGYM81FZ3vFhGFsQRBgEoQPnopzFg/EtAOAO+k3UrGG4hOF9d9AvzXWEYUGEQRAAjx7OAXRUN7Xnh2jYZag2nUOG1Q51PuJfEIYREQZBUJyD6oB2dTxYNFR1Yj5niIUB8ADjEf+CMIyIMAhjHo8ezkbNFl5G9ZgeKpYCMxN6JHsIxxTHszDsiDAIgspbcAOBIZwtwLfO4aGcNcxH9Vv4YAjHFIROiDAIYxqPHh6P6uvwbDxYNNQ9C9p7L8wdwjHnA590bV4kCEOJCIMw1rkUmIpqdDPUxK3tUArD7ojjWRhmRBiEMYtHD08DKoB/x4NFb/S1/wBYjuos5xmKwRJ6xGWNJf4FYVgRYRDGMlcC+ailpCHHHfS3ocRhqGYMu1lbEQZhWBFhEMYkHj08FygH7o4Hi2LDeKo4QzRjQCKShBFChEEYq1yLiu4JDPN5ljJ0M4bdgVWoFqSCMGyIMAhjDo8e3hc4FbgxHixa1tf+g2QpMHuIGvbMB96zym0IwrAhwiCMKTx6WAP+CHzBwLuz9Yc4YGeQNZMSesQJ+JCIJGEEEGEQxhqnAPsBv4oHizaMwPmGKpehAMgG3h7kOILQJyIMwpjBKpR3Hcp5e/cInfYza7vdIMfZw9q+M8hxBKFPpFGPMJa4AvXkfnY8WJQeoXMuBdqA7Qc5zp6o/g4fD9oiQegDmTEIYwKPHt4OlbdwfzxYNJSF8nrFymWIAzsOcqg9UD0YRkrQhDGMCIMwVrgJlYVcMQrnXgLsMNCDrR4MuyP+BWGEEGEQtno8evhYoBhVPTUxCiZ8AuyQ0CPaAI+fB0xA/AvCCCHCIGzVePRwHnAz8F/UrGE0WIK6sU8d4PF7WlsRBmFEEOezsLVzNcrhfEg8WJQaJRuWWNsdgK8HcPweKAe29GAQRgSZMQhbLR49vBeqrPYd8WDRC6NoSrswDNQBvQfwX3fQ3zJE9ghCr4gwCFslHj2cBdwJrAQuH2Vz4qi6TP12QFt+iT2RZSRhBK4KaD4AAAbiSURBVBFhELZWfgHsCpTFg0XJ0TTEHfS3ovIZBhKZNBOYjgiDMIKIMAhbHR49vBtKGBbFg0Wh0bbH4hMGtpQkGc/CiCPCIGxVWGUvFgFrgEtG2ZyOfAQUDCBktV0YpAeDMGKIMAhbG0HUEtKZ8WDRqtE2pgMfAOPofzG9PYBP3EH/uqE3SRC6R4RB2Grw6OEjgYuBm+PBoidH254utIea7trP4xYgy0jCCCPCIGwVePTwNFTF1P8C+uha0y39FoaEHpmBmmH8Z1gsEoQeEGEQtng8etiO8itMAn4YDxY1jbJJm2AtBX2OaraTKftZ21eH3iJB6BnJfBa2BgLA4cD58WDR+6NsS298QP+WkvYHWpHiecIIIzMGYYvGo4eLgV8BdwF/HWVz+iKKikxyZrj/fsBbkvEsjDQiDMIWi0cPbw/ci3LOlseDReYom9QXHwBZZJDPkNAjWcB3kWUkYRQQYRC2SDx6eCIQAkzgpM3Rr9AN7Q7oTPwMe6B6PIswCCOOCIOwxeHRw07gAVS7zBPiwaJPR9mkTPkQaAb2yWDfA63tK8NnjiB0jwiDsEXh0cMacBtwGHBePFj0/OhalDlWzaTXgQMy2P1woN4d9K8YXqsEYVNEGIQtjSuBc4Fr4sGimtE2ZgC8DOyZ0CN5Pe2Q0CM5wEHA0yNmlSB0QIRB2GLw6OFy4BpUzsJvRtmcgfIyKkx87172OQDIBRaPiEWC0AURBmGLwKOHS4FbgUeAs+LBImOUTRoo7c7k3paTDkd1bBvN5kLCGEaEQdjs8ejhk1BNd54BfjCKLToHjTvoX4NyQvclDK+6g/71I2OVIHRGhEHYrPHo4dOBWtST9vHxYFHzKJs0FESAAy1fQicSemQHVMe28IhbJQgWIgzCZotHD5cB9wDPA0fGg0UNo2vRkPEgMB4o6uazM1C5GYtG1CJB6IDUShI2O6yQ1J8D16KenE/aSmYK7dQBXwKnoUQCgIQesQGnA8+4g/7EKNkmCDJjEDYvrOS1v6BE4Z+oBLatSRRwB/1p1LUVJfTIxA4f+QEPsCWG4QpbESIMwmaDVebiCaw8BeC0eLCodXStGjYWoeomnQeQ0CMO4HpgFfDQKNolCLKUJGweePTwbqgyFx5UOOrdo2rQ8PMmqtbTNQk98jpwMCq34QfuoL9xVC0TxjwyYxBGHY8ePgvVpWwccNgYEAXcQb8JlALLUc71q4F/A/ePolmCAIBmmpt7pWJha8Wjh13ATagbZB2q+9qXo2vVyJLQI9sDJwGfAo+5g/6typ8ibJmIMAijgkcP/3979w4aRRDAYfzzlQjGQqKIqLDgCwKSVgOCggRk7cReLEQ7ywUrG1lsBG0UBLFOvY1iJYhgkwgWKuoKYvBFAgoaScBi9nBHLonBXC7nfj8YZu42gTk47r+7szNznDBpbSdhPOFymadz3e2VJHCMQSssyYotwBXgPPAcGCnz1M3upVXEYNCKSLJiLWHy1lVgELgGXOqRDXakRjEY1HFJVhwhBMIhwtIWo2Wejne3V5LmYzCoY5KsGCaMH6TAe+AscLeHV0aVGsFg0LJLsmKEsKHOSWAayIAbZZ76fL7UA3wqScsiyYp+4BRwgbBf8RfgOiEQprrZN0lL4xWD/kmSFfuBc8AZwqDya+AicPs/Wg1VahSDQUuWZMUuwtXBacKGM7OEndVuAQ8cQ5B6m7eStKhqGewDwAnCLN2R6tBTwiY6d8o8nexS9yQtM4NBbSVZsQ04BoxWZXd1aAIYA8bKPH3Rpe5J6iCDQSRZ0QcMAYdrZW91eBp4ANwD7pd5+qYrnZS0YgyGBkmyYh3hzH8fcBAYrsoQsKH6sw/AI8JEtIfAE9cwkprFYPiPVMtObCMsTLcT2AXsIQTBvqrdV/uXScI4wQQwDjwGyjJP/VJIDWYwrGLVoO8mwmOgg8DWWrv1eju/Q2AHv8/8W34Ar4CXf5RnZZ5+6vynkNRrDIZFVD/O64F1tbre7gM2Av1V+dv2QFU2tykDtXqhzZSmgY/AO8KGL+3KpI+PSlqKRgVDkhU3gaPEP+zz1a32mg515zvwtSrfau12rz8TZhK3ymdgqszT2Q71TVKDNW2C21vC/fRZYK5N3e69hY7NAT8Jt2tmavXMIu/NeBYvabVq1BWDJGlxC92/liQ1kMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYr8Ap2dgaeqGU73AAAAAElFTkSuQmCC\n", 30 | "text/plain": [ 31 | "
" 32 | ] 33 | }, 34 | "metadata": { 35 | "needs_background": "light" 36 | }, 37 | "output_type": "display_data" 38 | } 39 | ], 40 | "source": [ 41 | "# We could use np.tanh, but let's write our own as an example.\n", 42 | "\n", 43 | "def tanh(x):\n", 44 | " return (1.0 - np.exp(-x)) / (1.0 + np.exp(-x))\n", 45 | "\n", 46 | "x = np.linspace(-7, 7, 200)\n", 47 | "plt.plot(x, tanh(x),\n", 48 | " x, grad(tanh)(x), # first derivative\n", 49 | " x, grad(grad(tanh))(x), # second derivative\n", 50 | " x, grad(grad(grad(tanh)))(x), # third derivative\n", 51 | " x, grad(grad(grad(grad(tanh))))(x), # fourth derivative\n", 52 | " x, grad(grad(grad(grad(grad(tanh)))))(x), # fifth derivative\n", 53 | " x, grad(grad(grad(grad(grad(grad(tanh))))))(x)) # sixth derivative\n", 54 | "\n", 55 | "plt.axis('off')\n", 56 | "plt.savefig(\"tanh.png\")\n", 57 | "plt.show()" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 4, 63 | "metadata": {}, 64 | "outputs": [ 65 | { 66 | "name": "stdout", 67 | "output_type": "stream", 68 | "text": [ 69 | "3.0\n", 70 | "\n", 71 | "[[3.]\n", 72 | " [3.]]\n" 73 | ] 74 | } 75 | ], 76 | "source": [ 77 | "# How does Autograd deal with broadcasting?\n", 78 | "\n", 79 | "from autograd.numpy.numpy_vjps import unbroadcast\n", 80 | "\n", 81 | "x = 2.0\n", 82 | "\n", 83 | "# Let f(x) = x + [1, 1, 1].\n", 84 | "#\n", 85 | "# If x[], then it is numpy first prepends as many length-1 dims as necessary (in this case, 1) to get\n", 86 | "# x[1], then copies along those length-1 dims to match [1, 1, 1] (in this case, 3x).\n", 87 | "#\n", 88 | "# We can look at f(x) as f(x_1=x, x_2=x, x_3=x). Then the total derivative of f() wrt x is,\n", 89 | "# df/dx = \\sum_{i} (df/dx_i) (dx_i/dx)\n", 90 | "# = \\sum_{i} 1 * 1\n", 91 | "# = 3 = number of times x was copied.\n", 92 | "\n", 93 | "def f(x):\n", 94 | " return x + np.ones((3,))\n", 95 | "\n", 96 | "x = 2.0\n", 97 | "y = f(x)\n", 98 | "g = np.ones(np.shape(y))\n", 99 | "print unbroadcast(x, g)\n", 100 | "\n", 101 | "# Any time numpy needs to broadcast, it clones the broadcasted value along leading dimensions.\n", 102 | "# During backpropagation, we sum over these dimensions as total derivative requires.\n", 103 | "print ''\n", 104 | "\n", 105 | "x = np.arange(2).reshape((2, 1))\n", 106 | "y = f(x)\n", 107 | "g = np.ones(np.shape(y))\n", 108 | "print unbroadcast(x, g)" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 5, 114 | "metadata": {}, 115 | "outputs": [ 116 | { 117 | "data": { 118 | "text/plain": [ 119 | "array(1.)" 120 | ] 121 | }, 122 | "execution_count": 5, 123 | "metadata": {}, 124 | "output_type": "execute_result" 125 | } 126 | ], 127 | "source": [ 128 | "# What does Autograd do when a function has non-differentiable functions?\n", 129 | "#\n", 130 | "# It assumes their derivative is zero wrt their inputs.\n", 131 | "\n", 132 | "def foo(x):\n", 133 | " return np.floor(x) + x\n", 134 | "\n", 135 | "grad(foo)(1.5)" 136 | ] 137 | } 138 | ], 139 | "metadata": { 140 | "kernelspec": { 141 | "display_name": "Python 2", 142 | "language": "python", 143 | "name": "python2" 144 | }, 145 | "language_info": { 146 | "codemirror_mode": { 147 | "name": "ipython", 148 | "version": 2 149 | }, 150 | "file_extension": ".py", 151 | "mimetype": "text/x-python", 152 | "name": "python", 153 | "nbconvert_exporter": "python", 154 | "pygments_lexer": "ipython2", 155 | "version": "2.7.13" 156 | } 157 | }, 158 | "nbformat": 4, 159 | "nbformat_minor": 2 160 | } 161 | -------------------------------------------------------------------------------- /notebooks/.ipynb_checkpoints/Autograd Exploration-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from __future__ import absolute_import\n", 10 | "\n", 11 | "import sys\n", 12 | "if '../' not in sys.path:\n", 13 | " sys.path.append('../')\n", 14 | "\n", 15 | "import autograd.numpy as np\n", 16 | "from autograd import grad\n", 17 | "import matplotlib.pyplot as plt\n", 18 | "\n", 19 | "%matplotlib inline" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 3, 25 | "metadata": {}, 26 | "outputs": [ 27 | { 28 | "data": { 29 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzsnXd8G+X9x9+n4ZlE2WQoRGFaBREgZSOGKWHYjLJcSsHsGtwCxUCPLkRbihhuyzD4V1rAQNqaQlkSI4AZYpUNoshACgpREkYGSuIp6+73x3MOtuMh7yT+vvPS62zp7rnvxfZ97nm+SzNNE0EQBEFoxzbaBgiCIAibFyIMgiAIQidEGARBEIROiDAIgiAInRBhEARBEDohwiAIgiB0QoRBEARB6IQIgyAIgtAJEQZBEAShEyIMgiAIQidEGARBEIROiDAIgiAInRBhEARBEDohwiAIgiB0QoRBEARB6IQIgyAIgtAJEQZBEAShEyIMgiAIQicco22AIAjClopHDzuAbCDL2nZ9tb/vRN1v7daru6/7+rz965viwaJVw3ldIgyCIIwJPHpYA/KByV1ek4DxwLg+XvnWq+ONf6RXXUzg78CwCoNmmuZwji8IgjCsePRwFjAbmAXM7GY7E5iGEgFnH8M1AQ3Ahh5ejUBLH6/Wbt5LAW1A2npl8vUm38eDRcYA/ov6jQiDIAibPdbNf2egANge2K7Ddls2fXJPASs7vL4E1gJrunmtBdYBDfFgUXq4r2VLQIRBEITNCo8engnsCfg6vAro/LT/FfAp8D9rGweWo0RgBbBmpJ6ut0ZEGARBGDU8etgO7A7sD+xnvTwddvkciFqv94EPgU/jwaL1I2vp2EKEQRCEEcWjh+cChwMLge+hnL+gnvhftV5vANF4sOibUTFyjCPCIAjCsGJFA+0KnAycBHitjxLAYuAZ4KV4sGjZ6FgodEWEQRCEYcGjh3cCTkcJws6AAbwAPAo8BdTHg0VyA9oMEWEQBGHIsKKHvg/8GDgUJQbPA/8CHooHi74cPeuETBFhEARh0FiRRBcB56ByBpYCdwB3xoNFK0fTNqH/SOazIAgDxqOHtweuAM5E3U8eA/4PWCw5AVsuIgyCIPQbjx6eDQSAs1BZuXcBN8aDRUtG0y5haBBhEAQhYzx6eALwC+BiVFG3KiAoy0VbFyIMgiD0iRVy+gOgEpgBLAJ+Ew8WfTaqhgnDggiDIAi94tHDOwB/QUUZvQkcHw8WvT66VgnDiQiDIAjd4tHDNuAC4HpUUboLgDvEqbz1I8IgCMImWM7lu1ElK54Czo0HixKjapQwYogwCEIXEnpkCrAHsNYd9L812vaMNB49fBjwD1RTmh+jZgmS8DSGkAQ3QehAQo94gddRHbtM4Ch30P/U6Fo1MlhLR78AfgvUAyfGg0Wx0bVKGA1Gui2dIGy2JPSIHRWP3wocCfwXWJTQI3NG1bARwKOH84AHgN+hWkfuLaIwdhFhEIRvuRDYB/ipNUs4EcgDfjWqVg0zVjmLF4DjgZ8Bp8eDRRtG1yphNJGlJEEAEnpEAz5CdQbzu4N+03p/EXAUMMMd9LeOoonDgkcP7wI8geqHfGo8WPTYKJskbAbIjEEQFPsCOwJ3touCxSJUI5kjR8WqYcSjhxegZgoOwC+iILQjwiAIilKgCbXO3pGngVXAaSNu0TDi0cMHAHXABpQovDPKJgmbESIMwpgnoUeygRLg3+6gf13Hz9xBfwq4Hzg2oUdyR8O+ocajhw9BdU77AiUK/xtdi4TNDREGQQA/MBH4Zw+fPwnkAHuNmEXDhEcP7wuEgDhwkLTTFLpDhEEQVGP6FKrTWHe8Ym0PGBFrhgmPHt4N5Wj+AviedFMTekKEQRBgIfCKO+jvNkTTHfSvRiV8bbHC4NHDO6L8JQ0oUZAy2UKPiDAIY5qEHpkO7I5ac++Nl4H9E3pki/ub8ejhqaiZgg0lCvHRtUjY3NnifskFYYg5zNo+3cd+L6PCVguG15yhxaOHs4GHADdwbDxYVD/KJglbACIMwljne8Ba4O0+9nvZ2m4xy0lWc507gQOB0niw6NVRNknYQhBhEMY6BwAvuYP+vnoMfAKsBvYefpOGjF8CPwR+FQ8W1Y62McKWgwiDMGaxymvvDPT5JG1lQ0eBXYfbrqHAo4ePQFVJXQT8YZTNEbYwRBiEscy+1vaVXvf6liiwq1VXabPFo4fnoiqkfgCcL70UhP4iwiCMZfYD0qg+xpnwAapPw9xhs2iQePRwDqqshwPVT6FxlE0StkBEGISxzP7Ae+6gvyHD/T+wtpvzclIl8F3gjHiw6JPRNkbYMhFhEMYkCT3iQDmSM11Ggs1cGDx6+BhUT4nKeLDokdG2R9hyEWEQxiq7oHoav5bpAVaBvc/ZDIXBo4dnoEJT30VFIwnCgBFhEMYqC6ztG/087gPAN8S2DAorX+EulP/jh/FgUcsomyRs4YgwCGOVBcB6YEk/j4sCBQk94hx6kwZMOaqRUIX0aRaGAhEGYayyAHjbHfQb/TzuQyALmDf0JvUfjx7eHrgeeBy4fZTNEbYSRBiEMYfleJ4PvDWAw9tnGDsMnUUDw1pCugNVMlzyFYQhwzHaBgjCKPAdVOOdLVoYgPOAQ1GisHy0jRG2HmTGIIxF2h3PAxGGr4F1jLIwePSwG7gR1bf5r6Npi7D1IcIgjEUWABtQhfH6hVUzaQmw41Ab1U9uA+zAebKEJAw1IgzCWGQP4N0BOJ7bWcIozhg8evhY4Bjgqniw6NPRskPYehFhEMYUVge23VCJYANlCeAZjZBVjx7OA24G/gvcNNLnF8YG4nwWxhrzUIlg7w1ijCWov5259D8PYrBcaZ33kHiwKDXC5xbGCCIMwlhjvrUdjDC0+yZ2AJYQcNmBaata8rZpSTuc2+SsX+2wmV8RSGZanC8jPHp4R+AK4L54sOiFoRxbEDoiwiCMNeYDBt8WxOs3ubaXVjYZBzLO/vCVBIqvTxm2nV/62pP1zppZmGhMzW7giJkfMyPgWokSoBeBhwkkB5uVfDPQDFw+yHEEoVdEGISxxnzgY3fQ39SvowIuDTgEKJ/s5LgVLf/CxL5/yrDV3f3pgqnrUjkzp2Y3PJ/vaF25onHCkYviu7sWzvzkv76JX85BdVD7AwHXe8A/gDsJJL/uz+k9evgoVNmLS+PBoi/6Zbsg9BNxPgtjjfn0Zxkp4LIRcJ2GmmHUAYdqGjeD+WlD+uinbv7ogE/XpXJmAieX3vPcoSfd+fIPU6Z9O9Cii1futGdlzH8UMAu4GGgCgkCcgOuPBFyzMjHBo4cdqD4LS4Cq/lysIAwEEQZhzJDQIy7AQ6bCEHAdhurudh/QBpwFuAkkK0zyPmxNp74DlAE3VNSGHmg/rKI29A1wCqqmUk1lzP8FgeTNBJL7obKuHwQuAj4l4AoScOX3Ycl5gBe4PB4sas34ggVhgIgwCGOJ3azt+73uFXDNIOD6N/AMMBk4DdiDQPJuAskmANM045qmzUU5ojfpf1BRG/oY+AWqZMXh346djBFIngHsDNQCPwc+JOD6vrVc1QmPHp4I/BZ4HpDmO8KIIMIgjCV6j0gKuDQCrjNQFVSPRt3YCwgk/04g2SkZbmXTp5OctmzbeOeU6ypqQz2Fjf4F1djnmsqS4s43/UDyfwSSpYAf+Ab4N/AgAdfULmP8ApiCKqktGc7CiCDCIIwldgPWAJsWnAu4JqNuzjVADJhPIHktgWRz110rS4q1ZQ31hwAcMevMHustVdSGWlBP+99FCc2mBJIvoUp0XAEUAVECriMAPHp4W9SS0z3xYNHbmV6kIAwWEQZhLDEfeM+qd/QtAde+wDuoG/PlwEEEkh/1Ms6h61pXzQaw2xzb9nHOe4AvgAt63COQbCOQvAHVg3o18CQBV2U2rQFAA37TxzkEYUgRYRDGBAk9Yke15Px2GUktHV0ERIA0cCCB5I0Ekuk+hvtxQ3rdN9bXnt52tJaZ/gocXVlSPLfXUQPJ94C9UJFHl96bde1Z22vL74oHiz7vwx5BGFJEGISxwg5ALu3CEHA5UDfgm4AQsCeB5Ot9DVJZUrwNcEJLuvEuVPhp7zd7xR2ACZzb556BZBOB5E9+nzrt9V21z1icdcXxBFz7Z3AOQRgyRBiEscK3jueAawJKDC4ArgNOJJD8pscjO3M6KjH0L8BSMhCGitrQ58ATwFmVJcV9/s159PBef00X7f3T1EW32TVzPfAcAdfpGdonCINGhEEYK8wH2qZn/XQD8DJwGHAegaTeNeKoD34AvFlRG6onQ2GwuA+YDRyYwb7XAqueNfbUUX6Hl4B7CLiuIeCSv1lh2JFfMmGsMF+jeWmW7bMIMAc4kkCyX53PKkuKd0BFEP3TeitOHz6GDjwGNAIlve3k0cN+lGhdGw8WrSeQXIsqhfEXVOjq/QRcef2xWxD6iwiDMCbQaN43x/aaB3Vz3o9A8tkBDNN+U7/f2i4Fpib0SF+Zy1TUhhpQy1cnV5YU91aj7CrgS6B64zuBZAqVYX0pcALwYqblNARhIIgwCFs3AZfW9puCgEnOFIf21TJgn0FUOf0B8HJFbWiZ9f1Sa9tXyGo7tcA0VDb0Jnj08AGo2cIN8WBRY6cPA0mTQPJPwLGorOn/EHDttukogjB4RBiErZeAKwu4o82cehVAQ/rIn/S3qmk71jLSrsADHd5OWNvZGQ7zBGrGcnwPn18FfE3H2UJXAskQKltaA14i4Doyw3MLQsaIMAhbJwHXJNSN+Jwm45CnAAwmvDmIEY+xto92eK9dGNyZDFBRG2oCFgPHdi2R4dHD+6FqKt0QDxb13uAnkHwX2Bf4HxAi4PpxJucXhEwRYRC2PgKu7YFXUU/WZzSkj1gJfOkO+r8cxKjHAh9U1IY+7fDeCmubkTBYPGrtv3uX968CVgG3ZTRKIJkADgKeAqoJuG6QiCVhqJBfJGHrIuA6AHgNtZb/PQLJe+lvD4YuVJYUT0aJTKfqpu6gvxm19NMfYQijkt2ObX/Do4f3AY4AbuxzttCRQHI9cBwqUe8y4F8SsSQMBSIMwtZDwPVDVDOdtcC+BJIvJvSIE9iFwfV4Pgqw03kZqZ3lZO5joKI29BVqNnNch7evQtVI6n8TnkCyDfgpcAnwfVQy3Db9HkcQOiDCIGz5qJpHvwEWoW66+xJIfmJ9ujOqYc5gheFrVNOeriTo34wBVNjqHpUlxTM8engva/zKeLBow4CsUxFLN6GEYVdUxNJ3BjSWICDCIGzpBFw5qAqmV1vbhQSSazrs0b6WPyBhsEpYHA4srqgNdZchPRBheMraLkQlra0Fbh2IfZ0IJB8BDgaygVesDnSC0G9EGIQtl4BrJqqz2Y+AXwNnEkh2bX25B6rYXf0Az7IbMB0VTdQdy1FJbjn9GPNd4OsmW87JqNDVW+LBovUDtK8zgeSbwD7AMlT57rOHZFxhTCHCIGyZBFwLgDdQpbRPJJD8PYFkdx3O9kT1YGgb4JmOsLZP9/B5e8hqxpnI1sxjsd1MH45pNgG3DNC27gkkP0fVZKoD/kbA9QeJWBL6Q2+p+YKweRJwlQB3odb9D7Di+jchoUdsKGG4bxBnWwhEK2pDK3v4vGMuw6c97LMJq52TXp+SWnvavMal/3zulvJVVWV1GlAInAHsD7iABpRf41Hg/vLqwpaMrQ4kkwRcxaglqisBLwFXKYHkuozHEMYs8hQhbDkEXA4CrmtRRezeBvbqSRQs5gETrH37TWVJcT7qyfupXnbrV5JbO+FtjvQCHLQ6sqyqrG4eaknsGaAY+ACVYf0aqrrqPcDSqrK6s6vK6jL/m/22xtIlqAS9N8QpLWSCCIOwZaD8Cc8AOqrxzWEEkl/1cdSe1nag/ZIPQkU09eRfgG/7R2csDB49PDnpnHh60jFhzQQjfSSqreh8oByYXV5d+P3y6sILy6sLT0VVb10ILAH+BjxbVVaXeQG9byOWClGzkNcJuE7J+HhhTCJLScLmT8B1KPAPYDxwhpW0lgl7AingvwM88xFAM6ofQre4g/71CT2yjn7kMqAEID9bc72EufwI02iOarac48qrCz/bZMfqQhN4uqqs7hngLJQ/4t2qsrrjyqsLX834jIHkiwRcewL/AmoJuPYBrkw0hzRgO6AFSLiD/q7Oe2EMIjMGYfMl4LITcP0SNVNYC+zdD1EA1TshOoib3ULgRavGUW9kHLLq0cN5wEXbpmwvjc/e51AwaF2/6LruRKEj5dWFZnl14Z3Ad4F1wDNVZXVHZXLOjQSSK1CVXW9NGXMu/br1mq/ATAIfouourU7okQcTeuSghB7Reh1L2KoRYRA2TwKuecBzwO9R/Q/2IpDM+MnfurHtycD9C3MAL737F9pZTuZLSWfnGkw9sSFrJ5tj1hdAg2kkM+nqBkB5dWEMOAD4CHi0qqzutEyPBSCQbE00h2JftlalW43tJ4yzP2bPtz/+NzDOQSUI+oEXgMUJPbJjv8YWtho00+wuwk8QRomASwPOAf4EGKhyD/f2EIoKgK/GNxlV9mIn6zV125aZ0//v018XL5oafu++aeE3UBFMS1Hr+e9ES6Op3syoLCk+G7Wm76uoDX3Q274JPXIncIQ76O91Ocmjhx2YfPKDDVkud9qWp6Ed0Lz2j1ehspW3r6gNZfzHWFVWNwF4GDUDOLe8uvBvmRyX0CMXATcBoQmOu6+c4Hjgz6geEP8Gzk80hxqB81EJgznARcAd7qBfbhRjCJkxCJsPAdcMVGjmHcDrgI9A8p6uouCr8U3z1fhO99X47vDV+GKoOkMvAn9FReAs3K559u4A7+V9PA4V6XM5qs/Bf4A1vhrfw74a36m+Gl9uD9YsBFaSmX8iAcxI6JG+fHYn7pKye+ak7ZM0tJ+XVxe+hZqRzAO2z+A8GymvLlwHHG0df0dVWd2pfRqpRw5DicJDwAkTfv+3D1DXeTkqaulDd07xce6c4ptRs6UXgP8D7kroESnON4YQYRBGH+VLuBCVnfw94GLgcCtRCwBfjc/jq/Fd6avxvYpqfXkPcCIqWudKVL2h7YC8aGl0zs9XnH0XkL7+85/5oqXRmagyEdsBJwP3otbq/w4kfDW+a301vmnt56osKbbzbRmMTJ6UE6i/pRk97eDRw9p4Q7visEanYWK+xLdJbe1LVUf0cGiPlFcXNqNafUaAe6vK6o7rad+EHpkE3I1agvqRO+hXM6ZA0iCQvBHYCzWj+gcQducUZ6GE52pUbsWrCT0yr782ClsmspQkjC4B196oHgQLUE7mcgLJjwF8NT4XcBLqxnSQdcQbqCJ0YdSSUHf1i0jokRAw1x30+7r73Ffjs6HqCpWjbq6NKH9G5ZmPz90dNWP5YUVt6B99XUJCjxRZNu3nDvpf624fjx7e/4hG58u7ttrTNrRdyqsLP2r/rLKk+H+oXg893th7o6qsbjzq/253oKi8uvCZbmy8HTjXsrH7hkUBlx34CXANqkPcr4BbE82hw1Ai2gYc7w76XxmIncKWg8wYhNEh4JpGwHU7KolrFqqf8kLfvG0/9dX4jvbV+P4BfIFaHtoG+CXgiZZG946WRn8bLY2+1ZMoWPTqeI6WRo1oafS5aGn0JJR/YjFwLfDOqgkt51q7bXKD7YE+k9w8Kdtvfa120nBrR1GwWAwUVpYUZ2V4vk6UVxeuR82YPgIeriqr27eTcXpke5Qo/F+PogAQSKatnIddUAl3fwTed+cUO6FtXyAJPJfQI/1zeAtbHCIMwsgScE0g4AqgykecD9yUtNkKfPO2/dg3b9tK1E02jFrK+RuqIJw3Whr9Q7Q0ujSTUyT0yExgJhlGJEVLo7FoafQEVPOc8W0O8/wNOW1f3n300rUZXlWvvZ89enjePi2Ow1LQ6ET7TTe7PAWMA/bL8HybUF5duAa1HPUl8HhVWd2uHT4OoPI5rslosEByKcovcxzqHvGoO+f46inOwPmosub3JfTI1RLSuvUiwiCMDAFXDgHXJah4+auApxZNGHeob962Kw6c634ZdRMvB15GVRydFS2N/iRaGn09Whrt73rnHta2X6Gq0dLoY3vFJu07fU228emshm2Ap301vkya3qxBJcJ1O2PYpcV+3bZtdppsZtByGnflOSCNcgQPmPLqwpUoH00zsLiqrG47yy9wGnCrO+jvqd7TpqiM6UdREVM/AXy59jfrZmWfsNLG2oeB3wB/72dVWWELQYRBGF4CLhcB1+UoJ/GfkjZb9LwZ06/yzdt2QnDK5OeB64H1wAXAzGhp9MRoafSRaGl0MBm4+6BCXd/p74G7fDZhLxuarc1h/MEa5x1fjW+f3o6xQjm7zWXw6OEJvlb7Cc2a2eQybDd2d3xFbSiJehLvtwO6K1ai3OEoZ/vTLYZ5Ker/4uYBDRhIpggkq4AdgKBNaz1mZvbpx4+zP/Jf1PLfcwk9Ih3jtjJEGIThIeCaQ8B1I7AsBdc/Oi5/9UL3rOcPnOve97XcnKuBHYHfATtFS6P7R0uj1dHS6JreB82YfVEZzwPpiLYQaNjz40m/tcZpAup8Nb6+btrdZj/v2WK/dk7abl9jM28try7sLYP6KWDPypLiab3skxHl1YX/BY62wTYaXJA2zSfdQX+izwN7I5BMEkheCczVNH470XnH7MnOPwCte2u0vL9cf6ZbJ7+wZSLCIAwdKuz0SAKuf6Xh0zdzsi+5aPrUL/fyzEn+ctqU3VY6HT5UyOSBwHbR0uhV0dLoJ70P2j+sUtv7oJ7AB8JC4PmK2lBLtDT6PqoE9sdAyFfj6y1XIEEXH8M+lz3u8LY6zmnUzNZZaVt3voWOLEZFAn1vgHZ3ory68D8L8uw3Z9k0+1uN6e2qyurGDcW4BJKrCSSvAubm2V/5+bSsK9dqNEwH4731vzrrbwRc04fkPMKoIsIgDJ6AazsCrt81adrSF3Jznvjl1MnF+891p86auY39ufy8WWlNexzlzJwZLY1eGC2NvjwAv0GmFKCqiHYbNtoblSXF81AzmY3VVKOl0S+BQ1C+j7/7anxlPRy+HJhtCRMAO6bsv56VtmWvsBs1Vs5Bb7yF8lUMejmpnVlZtgVtpvnlypS5M8ohPX6oxiaQXEcgeX227aNZk51/KrdrqxuSbaVnr2s7aYV51aRaAq7jCLiyh+x8wogi1VWFgRFw7QicuMJuP/XNcfm7PZeXSyQ3J91is4FptqJpYeChGWvMJ2/+v/RkVCbtT2LXendAtcqcDkwBclGlF5wop2kjyufwOSpy6VNUTsG73vpYOgPL2iN7BjJjaHf+dqqPFC2NJn01viNRlUlv99X42qKl0b92OTaBKtE9FfgKYF6b/ZINmpl2qGzsXqmoDaUrS4qfARZWlhRr/SmP0R0JPTIVOMyhadej+l0vQjmkjyp8vjyJmt3sbr28fPszmQTYUbMXE/Wz+MZ6LUe1DG1/fQazPvPWP39bmx65x8a6f61rO/PI5vSC70/JuvEUu7Y6ScD1b1T/jDoCyYF20RNGGBEGITMCrixg31V2W9GbOTmnvDd5oue13ByWZKnQ+6yU8eWOn/Pcoe+nP9kvZqadBjuhMpLvBjqWU/gGVWriK1TcfSNKEFIokchHNdfxAkUoJyrAuliB92VUTZ9/eetjyR4s3RdViXUgS1RHoQTp464fREujzb4a30mo+kR/8dX4WqOl0Xs67NIxl+GrMy5+6sx90s4JMWfbw7fesrAxw/M/BZyCigSKDsD+jnwfdYO/v7y68N3bzl/cZmr2f+a0rKlvdY7bkJXa0LEEx1LUzySOiuRKo0TBhip17kKJ+G6o7O6OYaqtsQLvJ0A9aG/mLDhrmene+/SVzX9tdjnvfX+c/d8naRpnAUkCrsXA48CTBJJfDPL6hGFEhEHonoDLAez6bnbWMfVZWcd+Pnni/Gh2tnMJTrZZo7HtSiN9+HJzRdnytnUzvmGcsw23pqJU2vkciKHq7cRQ5S5i3vrY15maECvw2oA5qHX+g1DF3u4AbokVeB8C/uytj73e5bD9gNf6W/StsqQ4G7W+f09PT+vR0miLr8Z3AvAYcJevxtccLY3eb33c3rBnNvD2jLTt6hZMNtjMC/thRvsS1hEMXhhOAT5ZH/7Z/2IPN1x8KFy0avIujg92PW+bNxbo4wo+WvS7KWtjTwJRb31sfaaDxgq8WahrnIOq71RgvXYD8/jmt+60ax+FyV1wdk5y0tn+NV8fusr8rOa1ca767LxtWgpzJradbHOYEHC9jUqiexF4iUBy9SCvVxhCpCSGAAGX1qKx7TvZ2Qd/5nQu/MJu32dVi3PemoYse943NmavNpmzyjTnrCLtauj0MJFCPZlvvPFbr4+89bGGoTYzVuDVUDV9SoEfAhNRN+nfeOtj71rLJ18Dv3QH/X/oz9iVJcULUU/sRRW1ocd729dX48sHnkDNTo6MlkbrEnrEjVpeKftFU2N0rxbHy/9zGG/8+dbD9+6nHf8FllfUhgac05DQI1NN0/yybeW7Lze/fvt81AzsFeCWV/cJrGnKnfZ3a9fjy6sLe2xC1F9iBd5slFh8B82+S1bBMcdkbXfoHtgcttSnz9H6yZOYrRuwOY212RNTZu7klCvblbJnu9rImtAWszvNl1C+lreAKIFk5j2uhSFFhGEsEXBpq222aa/l5uz3tWY/sKXRvl9Ts2Pntg32Say32yev1Zi5xmTGWshJgaHZaHPk0uIc19iaNXFpc+7UFY1501c35U5b3ZQz9avGvG1WmzZH+y9QE2o9ej0q+zYBrCqvLuytbMWAiRV4x6NKQl+OWuq4M3/htXW2vCn3AQe6g/6X+zNeZUnxzaiyEVMyaMyDr8Y3EVW8bi5w8BOx26KoLmjXVG5oPnpem23Bm9lt+9xz0xFdZzR92fFH4EJgckVtKNMlqI3ECrxazp5n/dG57X6XNDx/DcY3Sx8CrvPWx/7Tvk9VWd0OqOxyDyqp8G9Wp7ghJ6FHZpimcR1oP8I0WtNfffBG8/v/XGU2rt4ezALQNpYBceSmjeyJKVvWuDac+WnDkWMsdeSlP8we3/aWI9d4D/XwsYRAUrrMDTMiDFsTAZf2Zk72Nh/Zs3ZvXW//rtFk89Fq24Fm+yyaxk+0N4/LzmqkMl6HAAAgAElEQVTO18Y155HXmo/hyCflyCPlzCflzKM5e3xba9aEljbH+HTanus0bY6eSlJnSitqvf494F3UU+vr5dWFQ+aEjBV4J6F8GRXZ809rdHr8Tk2zTehP17bKkmINlZH9YUVtqDjT43w1vtkoJ3cWsN8TsdtebjLNF0LJ1A9WOoyl1996uKd/VwOVJcVHomYjR1bUhjJpErSRWIF3MnBnzoKzj3Ns40tvePLyhd4Po3Xd7VtVVjcZ1QDpMOAB4Pzy6sJMS4D0m4QeKQB+DZyKKsb3oNnWfMeGx3+WwEh/B9gFzO9oNnY3TTyYWqcy3/acNFn5aZz5adOWZay1O40v7VlmwpZlLHHkpGM5k1LvOfOMT4CvCCQzCVIQekGEYTPmqRtm2hpabNOa12ftlm7J39VM5ReY6fy5Zjp3BuncyaaRO4627ByMXCdGjt1m5miamY2pZZG2Z5GyZ5G2O2mzO8BsA9JgtmHSBmYalRCrmaA1ARtAW4fmSGpa9mq0nK80W95KzTZhuaY5V6FCKVdb2wbrYNN6gXIcj0ctW2yDcsJuC3wH1ei+PfkrCTyLciL/u4+kr4yJFXj3yjvs6hfNpm9yml75023AJd76WK/NeNqpLCn2otpbXlBRG6ruz3l9NT4vKpR11WOxmxu+SttmvbE+Pf0/2akz7rnpiP60IW23JQ/1f1xdURvqM5qpnViBd3/gn6DNGFd8Syt2xyNzrju412J3VWV1duAyVFXZlcA55dWFT/fX5v6Q0CM7o7LcS4GJpmnGW4zGZxINH7/57pq6ZWmzLQfTzMtuS0/NaW1zZ7elp+ak26Zn0zbbkTam29LmBNJars0wNZthYjdNbIaJ3TBwONI4nWmcDiPlsKebHTZjvdNhrHXa0qvsdmOl3W4utznN5TaH+YXNYaxw5BgrcqakVjqyjYbeGkGNRUQYeqHmZ0fYbQ3mhLQta7KDrIkatsk2siZopm08mjZeM+35wEQwXaZpm2yadvUZtjyw54ItB9OWpWF3gs2pmXY7ms2Oqdk0bHYNmw3smg2bhmbXNGxomgZoHbagbdyaqHdN63vANDBIY5hpDNPAMNOYpEmbbbQaLaSMZlrTatucbsSg3w9TJmrt/FNUWYv3UGvA7/VnqaOqrG4qqtvYEcCRKAdmEhVGeXt5dWGvXdL6IqFHJpmmubpt2WuvNr991/4ox+bJ3vrYqr6OrSwpvgy4AZhbURv6vK/9u+Kr8e0PPHtt/OLmuRt2nFjbmEr+vup7E/s7Tgd7wsDOwI59ha1afpdLUPYvzd71lKuzdvheDarnwqJMzldVVtfem2JHVNTVZeXVhf8bqP1dsWZk81CJh7sA33FozoI5+QXbuvN3yt8m14Ndc9CabmZVS4KvmxOsblnON61fkzKGyM1gmtgNE9vGrfHt96aBAxO7aRg2zTTsGGm7ZqbtGG02zUzZTTNlw2i1aWazHbPZhtmkYTYCjTbNbNI0GjXNbNQ0s0HTaNQwGzU2bhs009ygmWywmeY6zTDXZ6WNDflm64Ysw2hx5qdbpz2Y2OxuwmNKGCLldxhTc2ZomnXj3fhPs3X4WkPD1vlrbfMsImmqH57WH/sM01hlYiQwzYSJGW81Wj5Ktq76JJZ8Nf518zIT9dQ/pcNrOt92GNsRFacPKqTxdeBJlNP2jYraUEb+hKqyuvZeCOegmu1ko5Y1AuXVhfUZX0wHEnrkWOAR4OD1D5+/Lapc90rgWG99rNcIn8qS4ueBSRW1ofkDOTeAr8Z37C8+/ekj+zQV8NvWDcG7/3z0lQMdq7Kk+MeobnO7VNSGPuxpPytq64+oxkYPAWeNP/4vF6OqqU53B/19imI7VWV1OcDPUOXNncDtwJ/LqwvjA7yGKaiucEejMt1nWh+lUQEL9ahIri/yHBPW+Cb550zNdhfk2vPn223OjQ2BTNNYYWAsMUwjYZjpFW1m6/LWdPOKhrbk+mTr1y2rW1a2rm5Znm41mrNQ+TA5fJsbk4Np5joMY2KO2TYjm/QMm2lORs1q803INdFyDDSn9bIbmmZLo9nSmk0zNI20psHGe4Vt4z3B1n7P0GybfG/rsF9332vqnQ7Hddh2OMbW9RjNjoZGU9vbvz72zn/+fiA/l0wZU8LwdNmfWsY5J2epO6qBiYlpmlj/TDAxTWPju2o/62uM9q+tDwzD+towMdNoZkrDTIHZamI2axpNGlqDhrle0+zrNGxr7ZpztVPL+yrXNnFljiPvm2xbdpPTltNs02xp1Lqrtd7T69cbt+0hmQk9Yketc2eh/qizULkDLlTkzkRU4tIsVJjhttZ2HuoPCGvct1EO1ReAZ91Bf6fIIuvJbzaqqc7eqGJt30VNXpaiOqPdU1EbyjiHwFrrrkDd3HKBu4ArrDLSGZPQI7cCZwGT3UF/S6zAuxfq6TcPOLKj87XLNU0EVgHXV9SGftGfc3a5Ds2dl2pdkJXnOHGHivsanU1nDDS7u7KkeDbKeX9lRW0o2N0+sQJvDqqL3cnAn4EKb33MSOiRF4B8d9D/3QFexyxUee7TUT/XB1CNlF4qry7sdbpphfyegmqsdCgqj2I5avb2EsrHVF9RG+rV/5PQI9NR/TR81msn1O/rTDrnUHSkEeXTan+lgFbTNFOmssNpgiON4TQ1nCamw3rfboJdM7FrYLPm6raOj4S2zezB8OWG0E0lt1yX8TLjQBhTwiB0xhKU7VB/fHsCftR0PxurdDPKF/BgTwXpKkuKp6KWhn6EEgqbddw1FbWhFzO1paqsbhqgowRiFVBeXl34YIbXoaGWuT50B/3HtL8fK/DOBepQs54ib31sE3sqS4pLUJm5B1bUhvoVydSRsosW/7gIZ/UeeQ7O2v7XfJG1+vfR0uivBzpeZUnxG0Cqoja0f9fPYgXePFSYbiFwmbc+VgmQ0CO5qATCm91B/+UDPTdAVVndHOCnwI9RT9hfAA9a5321Y/nwypLiGajS3OcD01A/i3+hfnfeGmwWdzsv/eyFXBt8x67hNZVQzNAwp6e1thlpzZhmasZ4E/I1U8uxYXfaTZvDbtpsavFV0dEp1v49JqQ1g7TWRpuWxsAwDM0wDEzT0NJpQzMNAyNtaGabgdFmakbKwEyZmpEyMVtMzFag1dDMlKkeDNu3LaZmtphgbc1m1P5NaDSZmE0aNIHWZGK2mCataDQZGC1pzWg2TVpMzCZDM5pMaFqvNTebmM1XBfRhd66LMAidSOiRbOAAVE+E41F/gOuB+1AdwN7r6djKkuJZwJmom/t01FPirypqQy9kev6qsrrdUQ169kTdXM7toYdBR5t3QmVRl7uD/ts6fhYr8M5GdWKbCxznrY91cq5WlhTfh8p4nl5RGxrwH9wvy59JFNhssw7Nd2rXzv5b6MUJbxUDF0RLo/1yZnew69eofsuzK2pDG/soWDOFR1HJeGd662Mbs68TeuQQVG+HY9xBf2ig19IRq/heEWpmcjRqVmcCUdNoeDfV+LTbSH12AJhZ4HwazXEzZtPjPYmBtYyYg8pwn2S9Jnd4tX+/cTnTxJhqwjQbth5rPbVpKVocDTQ5G2hxNJqt9qbWlL2lMWVvWZeyt6xNa21rDC292tCMr9K2tq/SttSqlK3ly2Znw5ff5Hy1cl3uqjXAumhpVCKaEGEQesF6Et8P9SRYgvqDfhK4yh309xifb0XWnA38HBWN9A/gsora0IpMzltVVudA5Sf8DvX0eUJ5dWGPa+0JPXIxajllO3fQ/1nXz2MF3unA0ygfyUJvfewly84cVGmOBypqQ2dnYlt3/PiixQfv1up4fm12+t0zc3N2b9VSpx1XcPGpqBvpidHS6MP9HbOypHgX4APgJxW1oSrrOrJRy2NHAGd762N3dzwmoUeuQjVBmuwO+r8Z6PX0RFVZXT6wn2kaB6abXz+lreXtAsxmzebcGUfu/tjsk9p3TVmvtg4vJ9+u/fdKWmtrStlbUk2ODVpj1jpnY1Yyp9nRQLOjkWZHAylH89qUrWVlq7050eJoWtqQ9c0nzc4Nn5qa2V7D6Su5wQ8OEQYhIxJ6ZDJKIC5DPcmFgZ+7g/7/9nSMJRA6cAXqRnElUJXp0kJVWd3BKKd0HnBmT0tLCT3yJOBxB/0FPY1liUMEFUp7iLc+9m5lSXG7w/qoitrQk5nY1B2X/+Tpd9xttt1X5hmH/CQr53ngsqO8F1ajwnLnA9+Llkb7vUxVWVL8IfBVRW3okFiB14Fa7z8OOM9bH+taxI+EHnkOcLmD/j0Hei0Z2LQ9akZ3MPCcZp8eyJ7wowbUjGwKKjjBhSq348Ba30f9/JvaXwbppi/Gf5a33PXJ7NX5iXnrstds35i1brtWe6PTsBlY+32AclJ/3OG1JFoaHUifDaEfiDAI/SKhR8aj1pMvR0UwVQK/6+qo7oh1M6lCPemGgbMqakMZ1UyqKqubjboh7gtcXF5d2KkTmWXP18Dt7qD/Z72NFSvwbota3soBDnx8/va/Ri2TbFNRG8oo56Er5T9dPHenlD2+1GF8dum4nO1ROR63u4P+Cl+Nbyoqx2Ea4I+WRnsU0e6oLCkOAL+xGcasI6Of/QYV//9Tb33s1q77WkuA31jnvnQg19KHLZp1/htRzt1LgbsyFXlfjS8XVfPqAOu1L8p3AWrW9jYqCbI9GfITeeofPaQfg9Av3EH/enfQfy0qzv5e1HLRhwk90mNtn4ra0P9Q6/gXodbG368sKT4sk/OVVxcuR0W4PATcVFVW97uqsrqOYSLHopzlD/Q1lrc+9jnKQU5a057GNI8HHhqoKAA40aocaKxwGL/o0OJzNkC0NLoK5ZhvBp711fh27ufw/wK0WWs33I66KV/fnShYfBcleBk7/DOlsqQ4H+VjqkJFrO1SURu6szdR8NX4NF+Nb1dfja/CV+N7CpW09wwqlHYmannxDFQY9IxoafSoaGn0ymhp9J/R0mi9iMLoIjMGYVAk9MiBwF9QZbIrUQXsesxKqiwp3g0VBbQTcHH7+nlfWFm6twPnAf8HXFheXWgk9MgjqPDZbd1Bf0Z5FLEC74IVrvyX3vXMyMltSX3/woef6rcPAODynzydNz1tW7fGZqy/turwSQAJPfI8YHcH/f72/Xw1vgLUDbUNODhaGl2S6TluOvGoZeNaUu79lyz/J3Catz7W7TUm9EgF6ml+hjvo/3Ig19MdlSXFO6Kii3ZBlbS4tqd8FWtWcARKrBfybUe7GCrX5WnglWhpdMj9H8LQIsIgDBorTPJGVPG3d4AfuIP+TXoatFNZUjwelWlbjHoKvaSiNtRn/SRrpnANyldxx8IJjitybdqXwG19LSN15dYTjnwNk30OjS2ts8HR3vpYv1NsL/rp07ftnLJf8Fp26up7bzoiAJDQI4uA/dxB/3Yd9/XV+HZFxfM3AodES6Of9jV+rMB78P+mTXz2o1lT7LktKd+FDz/VY3Z4Qo/8C1jQ9byDobKkeAHqhq4Bp1bUhhZ33cdX4xuP5WS3tvmoJa1nrGMXR0uj/c4mF0YXWUoSBo076G9yB/3lKMfotsDrCT1yVE/7V9SG1qNCYW9AVfd8zFqu6BWrAugvgT8A5y1tMR5AJfPd3+uBXagsKZ7Z4nTsld+aesymcgHusrKIM6aqrC5rdpvt7C/tRkvcYfyuw0crgFlWRNdGoqXRD1DLaPnAy74a3269jR8r8H4HeHjWN+v/h2mmm7KdvdY9wupD0Z9r6I3KkuIDUTkgG4B9OoqCr8aX56vx/dBX43sE5d/5J8oZfS9qqW56tDR6crQ0+lcRhS0TEQZhyHAH/Y+ilnU+A8IJPaJ3vUG2U1EbSlfUhq5ARTotBBZXlhRP6m7fjlji8CvghmlO7bAWw0wubzW6zWruhR8BtjXjci9DzT5OBa7vzwCf29NXjzO17I+d6bsilUd1XA9fjvJ5TO56TLQ0+i4qiTANvOir8fm77gMQK/DOQlVYbc5NpReiaU8AZ1SWFNu729/qBTGbIRKGypLiw1FJil8A/ora0BJfjc/mq/Ed6qvx3YUqq74IlWvyfyhRmBUtjV4QLY0+Ey2NDthnI2weiDAIQ4o76F+Kijq5H7gWuC+hR3qMXa+oDd2BypHYC3i+sqR4m77OUV5daB7jctwzxWHjkxbD9WZj+upM7assKbahxOjlitrQx8B1wK1ARazAm9FyVFVZXdZUw3bRSruRfjer7YouH3fs5LYJ0dLoh6jonC+Axb4a38kdP7f6TIRRoZ9F3vrYUlSZkFmo2kPdMZg+152oLCk+HgihQkMPuvvopeN8Nb4/oNp+1qGWjO5HBQTMjZZGL46WRl8UZ/HWhQiDMOS4g/5G1FP4L1Gd1p5J6JFpPe1fURt6ABU2ugPwUmVJ8bZ9ncOmaWWmabasaDXuA35VVVaXaYhm+3luBvDWx0xUddIHgT/GCrwlfQ2w2mZclmdqeR9ktf27/vqirm0x25P4ZvV0vLW8ciDKH3O/r8ZX6avxOWMFXicqEsmHqgz7tnXIo6hWqRf3MOS+qMinHrPSM6GypPg04AFDM6MPHrx80d1HL30MVY78ClROwamoCKJzoqXR56Ol0WFpwiSMPiIMwrDgDvpNq71mCSqU8jWrWUu3VNSGnsZanwZetHIfusVKtjtD07T7m0zORIWqVlaV1WWSvXwJqkDdQ+1veOtjadTyUgS4J1bgPbSng6vK6rLzTE1fYTfMD7LS3YlRrzOGdqxQ1kNQs5VLMc1nG7K5FxXVc4G3PvZE+76WY/5W4JDKkuLduxlub+Cd/jQn6sr1pxZdYGLeu3pC65q/H75st/X5bdejEtMuBdzR0ujRVihpv7vKCVseIgzCsOIO+u9H3QDHA68m9EiPN92K2tArKGfweJQ49CQklwPjgBusip8/Qq2J31FVVndST+NXlhTvYY1/a9fcBW99rBnlPF8CPBwr8HbrHG7SzItyTW38u9ltzy25rijRzS7tM4ZehQEgWhptjZZGfwr86IfPG/vmt1Dy9vbaE6dc6bizm93/iopo6iRGViHEPYE3+jpfd/hqfPPPv+zgF+2GdltiWpP2+H5fmG0O81Zg92hpdI9oafRP0dLoFwMZW9hyEWEQhh130P8aqmrrSmBxQo+c1dO+FbWht1DOTDtKHDrdoBN6ZAZqSeUf7qA/ClBeXdgCnIByvv69qqyup2S736JCKf/S3Yfe+thaVELaeuAJK1N6I1VldVOcJlcvdaSJOdNdfQvt19qKitTpUxjauf/atsnHv2Y6X9tZSwRPth0FvOqr8e3TcZ+K2tBaVB7HaZUlxd/p8FEBqmTIm5mez1fj29lX4/uV727fe7t/7Hp352Xj/SumNCVe3H3V99N20x0tjV4aLY0OallK2LIRYRBGBKu4XXtntTsTeuQPCT3S7e9fRW3oA5Q4tKIc0nt1+Pg6VIjqVR2PKa8ubEDlRcSAh6rK6jqVq64sKd7f+vw66ybbLd762DJUlnY+8KTVRxkAA/P3Nsh9MSf1xqfXFb3Vy+WuoBcfQ0diBd4fAjcBD0//xpyHpv0QFfL7mq/G96Svxnekr8bX/v8URM0aftthiAXWtkd7rIiiBb4a31W+Gl8UqMfkdwe8P2Xq7ksmkrIbi2atzvW8fe77D0tEkQCS4CaMMAk94kQltZ2Him45u6c6S5UlxfNQkTBTgKNL5v18Bso5+3t30N9tr4OqsrptUPWQpgIHl1cXvm+FeUZQvSe2r6gN9VjXqZ1YgfcQrM50wOF1h1TtaWJG3s5Ka3V5qePiwaJHe7nGMDCzr2J2sQLvj4Aay7ajvPWxJgBfjW8cKr/jZ6iifwmUH2XxqYvdh2S32a9AhZG+lNAjN6Mq2brcQX/aOn48sDtqiekA4DBU+KwJRGxpHix5ds4+2W22dlG6NNPue8LYQIRBGHGs3IbLUE//HwGnuoP+d7vbt7Kk2A0868qaNmfhrNI2m2b/CNjfHfT3+GRbVVY3FyUOTsDfvPaPR6FugKUVtaF7ejquK7EC78lAbZs9+4UXD7xxzgabNudvE5o/b9XYOR4s6vFGmtAjfwGOcwf9PYbexgq8pagw1OeBY7z1sU3Eylfjy0ItkZ2KWuLKcrRpHP/iLANoffigFYurlv7qAEMzjAu2u+Y11CxlJp07nSVQWcjPAM+c+fjcDai6R8ejyppfNVSNdIStBxEGYdRI6JFCVKLUVFTv4mu76yPw9sWLjpyYNS2UNtvsb69+9sKSuypv72vsqrK6AiBipFcZrevuGY9qYFPc35tgrMB7ev1OP7h7xSy/Lexcw4f5uRfEg0W9Nt+x+iIEgOzuIoViBd4yVLvMZ1HNg/qM9PHV+PJQS3Hzd0jkH3rg+1OLlsxpWHWZ89dTnnW99s0tM/+xDLWEtQKVYPg28E60NLqxyY/VSOlRYA/gZxW1oZs3PZMgiDAIo0xCj0xBZR2fDaxD9TF+EnWDm4l6Yj7bNI0VTy2/6+tkatXuqCfd3/ZVX+nms245KNX8Sh1mm82Zd+iJF9154UO97d8dVWV15wB/nb3sWXPSijqz1e787uFvvvhOH9d0HsrBPdcd9G8sCWH1VLgetUQURuUqNPXXJoDKkuJbXM6pPznSfQ7A6e6g/74+9l+AavIzCfhBRW1oSDq8CVsn4nwWRhV30L/aHfSfg1oPfwI4F5V5+zbq5nk6cIum2XZNplbtB9yNqvL5YmVJsbencStLirdLNT51K8b6VFb+McvsWQV/ryqrO74/tlWV1RUDtzdr5mu3ztlWG9/a2Oje8PVLsQLvWbECb28d4jfJZbD6T7+AEoVbgOMHKgoWP9smd957AC99+dBOVr+ETagsKc6uLCn+JSpiS0P5JkQUhF6RGYOwWWFVat0b9WSbBP5jZVJvpLKk+FTUUswElPP2b8B/KmpDbZUlxVOBs4BfoJytJTmTLn0XJTJ7YZUGt0Jce6SqrO5k1Fr8+1UTmpY22lh4+ZuL9i5MvHMbqhzEs8Cl3vrY+91cw26oLORT1j98/uPAT1FilgZ+7K2P/WMg/zddiV9Rd7NNs5U/EK+0mZiPoKKW3gAMYAaqT/OlqO5q9wMXVtSGVg/FuYWtGxEGYYuksqR4GqqY3nmoXsJpVP7BRGuXZ4HzKmpDnwFUldXloEThQlQS2xXAo1aC3EaqyuqmofwDFwKvPZbXekF9Vvot4I/xYNHlsQKvHVVr6Q/WuZ5ENZ15Dkh462PmZ6fd4HbO2XdZ62cvPNfy3qLdUFFVDwOXWLWPhoSEHnnGNE3X/fHrH0QVA5yACvFt5tvuaG8Av6ioDT0zVOcVtn5EGIQtGqu3QzGqkcwkYCmwuKI21G2Uk5X8dguqUVAcdWP/CNWfeAGqyUwu8GdAv2Fi05+Bc4B58WBRe1YzVn7DhUAZ3y4ZNaOa8YwbV3wLqXjEaPng/seAG7z1sX73fO4NK7LrK+ARd9B/rvX/cAKqYVKedW1PVdSG+tVOVBBAhEEYg1SV1TlR4ZpnoyJ92p+uV6CE4sby6sKYRw9vg7rBLooHi87tbiyrj8MCVD2o7VAZ26vHFd10ITbH63NuKPz+cFxDQo/MQvkyLnIH/bcMxzmEsYtjtA0QhJGmvLowhUqU+5fVMtSF8kd8Y/V7aOciVG+FG3oay2q1+QZdahVZNaFmDLHpHZlvbaV0hTDkiDAIYxrLx7Cm6/sePTwBtVT0UDxY9NEAhk6gMo6Hi/YqqyIMwpAj4aqC0D0/RjmXrxvg8ctQLT6H6+FrPhB3B/3JYRpfGMOIMAhCFzx6OA9VsuPZeLDo9QEOk0D9fQ3XctJ8ZLYgDBMiDIKwKWWohkEZtwzthmXWds7gzelMQo/koaKqRBiEYUGEQRA6YM0WrgCeiweLIoMYqr2Jj3vwVm3Crqi/3W5DcgVhsIjzWRA6cz6q1HWfvZ/7YNhmDEhEkjDMyIxBECw8ejgX+DnwQjxY9MIgh/sG1VRnOGYM81FZ3vFhGFsQRBgEoQPnopzFg/EtAOAO+k3UrGG4hOF9d9AvzXWEYUGEQRAAjx7OAXRUN7Xnh2jYZag2nUOG1Q51PuJfEIYREQZBUJyD6oB2dTxYNFR1Yj5niIUB8ADjEf+CMIyIMAhjHo8ezkbNFl5G9ZgeKpYCMxN6JHsIxxTHszDsiDAIgspbcAOBIZwtwLfO4aGcNcxH9Vv4YAjHFIROiDAIYxqPHh6P6uvwbDxYNNQ9C9p7L8wdwjHnA590bV4kCEOJCIMw1rkUmIpqdDPUxK3tUArD7ojjWRhmRBiEMYtHD08DKoB/x4NFb/S1/wBYjuos5xmKwRJ6xGWNJf4FYVgRYRDGMlcC+ailpCHHHfS3ocRhqGYMu1lbEQZhWBFhEMYkHj08FygH7o4Hi2LDeKo4QzRjQCKShBFChEEYq1yLiu4JDPN5ljJ0M4bdgVWoFqSCMGyIMAhjDo8e3hc4FbgxHixa1tf+g2QpMHuIGvbMB96zym0IwrAhwiCMKTx6WAP+CHzBwLuz9Yc4YGeQNZMSesQJ+JCIJGEEEGEQxhqnAPsBv4oHizaMwPmGKpehAMgG3h7kOILQJyIMwpjBKpR3Hcp5e/cInfYza7vdIMfZw9q+M8hxBKFPpFGPMJa4AvXkfnY8WJQeoXMuBdqA7Qc5zp6o/g4fD9oiQegDmTEIYwKPHt4OlbdwfzxYNJSF8nrFymWIAzsOcqg9UD0YRkrQhDGMCIMwVrgJlYVcMQrnXgLsMNCDrR4MuyP+BWGEEGEQtno8evhYoBhVPTUxCiZ8AuyQ0CPaAI+fB0xA/AvCCCHCIGzVePRwHnAz8F/UrGE0WIK6sU8d4PF7WlsRBmFEEOezsLVzNcrhfEg8WJQaJRuWWNsdgK8HcPweKAe29GAQRgSZMQhbLR49vBeqrPYd8WDRC6NoSrswDNQBvQfwX3fQ3zJE9ghCr4gwCFslHj2cBdwJrAQuH2Vz4qi6TP12QFt+iT2RZSRhBK4KaD4AAAbiSURBVBFhELZWfgHsCpTFg0XJ0TTEHfS3ovIZBhKZNBOYjgiDMIKIMAhbHR49vBtKGBbFg0Wh0bbH4hMGtpQkGc/CiCPCIGxVWGUvFgFrgEtG2ZyOfAQUDCBktV0YpAeDMGKIMAhbG0HUEtKZ8WDRqtE2pgMfAOPofzG9PYBP3EH/uqE3SRC6R4RB2Grw6OEjgYuBm+PBoidH254utIea7trP4xYgy0jCCCPCIGwVePTwNFTF1P8C+uha0y39FoaEHpmBmmH8Z1gsEoQeEGEQtng8etiO8itMAn4YDxY1jbJJm2AtBX2OaraTKftZ21eH3iJB6BnJfBa2BgLA4cD58WDR+6NsS298QP+WkvYHWpHiecIIIzMGYYvGo4eLgV8BdwF/HWVz+iKKikxyZrj/fsBbkvEsjDQiDMIWi0cPbw/ci3LOlseDReYom9QXHwBZZJDPkNAjWcB3kWUkYRQQYRC2SDx6eCIQAkzgpM3Rr9AN7Q7oTPwMe6B6PIswCCOOCIOwxeHRw07gAVS7zBPiwaJPR9mkTPkQaAb2yWDfA63tK8NnjiB0jwiDsEXh0cMacBtwGHBePFj0/OhalDlWzaTXgQMy2P1woN4d9K8YXqsEYVNEGIQtjSuBc4Fr4sGimtE2ZgC8DOyZ0CN5Pe2Q0CM5wEHA0yNmlSB0QIRB2GLw6OFy4BpUzsJvRtmcgfIyKkx87172OQDIBRaPiEWC0AURBmGLwKOHS4FbgUeAs+LBImOUTRoo7c7k3paTDkd1bBvN5kLCGEaEQdjs8ejhk1BNd54BfjCKLToHjTvoX4NyQvclDK+6g/71I2OVIHRGhEHYrPHo4dOBWtST9vHxYFHzKJs0FESAAy1fQicSemQHVMe28IhbJQgWIgzCZotHD5cB9wDPA0fGg0UNo2vRkPEgMB4o6uazM1C5GYtG1CJB6IDUShI2O6yQ1J8D16KenE/aSmYK7dQBXwKnoUQCgIQesQGnA8+4g/7EKNkmCDJjEDYvrOS1v6BE4Z+oBLatSRRwB/1p1LUVJfTIxA4f+QEPsCWG4QpbESIMwmaDVebiCaw8BeC0eLCodXStGjYWoeomnQeQ0CMO4HpgFfDQKNolCLKUJGweePTwbqgyFx5UOOrdo2rQ8PMmqtbTNQk98jpwMCq34QfuoL9xVC0TxjwyYxBGHY8ePgvVpWwccNgYEAXcQb8JlALLUc71q4F/A/ePolmCAIBmmpt7pWJha8Wjh13ATagbZB2q+9qXo2vVyJLQI9sDJwGfAo+5g/6typ8ibJmIMAijgkcP/3979w4aRRDAYfzzlQjGQqKIqLDgCwKSVgOCggRk7cReLEQ7ywUrG1lsBG0UBLFOvY1iJYhgkwgWKuoKYvBFAgoaScBi9nBHLonBXC7nfj8YZu42gTk47r+7szNznDBpbSdhPOFymadz3e2VJHCMQSssyYotwBXgPPAcGCnz1M3upVXEYNCKSLJiLWHy1lVgELgGXOqRDXakRjEY1HFJVhwhBMIhwtIWo2Wejne3V5LmYzCoY5KsGCaMH6TAe+AscLeHV0aVGsFg0LJLsmKEsKHOSWAayIAbZZ76fL7UA3wqScsiyYp+4BRwgbBf8RfgOiEQprrZN0lL4xWD/kmSFfuBc8AZwqDya+AicPs/Wg1VahSDQUuWZMUuwtXBacKGM7OEndVuAQ8cQ5B6m7eStKhqGewDwAnCLN2R6tBTwiY6d8o8nexS9yQtM4NBbSVZsQ04BoxWZXd1aAIYA8bKPH3Rpe5J6iCDQSRZ0QcMAYdrZW91eBp4ANwD7pd5+qYrnZS0YgyGBkmyYh3hzH8fcBAYrsoQsKH6sw/AI8JEtIfAE9cwkprFYPiPVMtObCMsTLcT2AXsIQTBvqrdV/uXScI4wQQwDjwGyjJP/VJIDWYwrGLVoO8mwmOgg8DWWrv1eju/Q2AHv8/8W34Ar4CXf5RnZZ5+6vynkNRrDIZFVD/O64F1tbre7gM2Av1V+dv2QFU2tykDtXqhzZSmgY/AO8KGL+3KpI+PSlqKRgVDkhU3gaPEP+zz1a32mg515zvwtSrfau12rz8TZhK3ymdgqszT2Q71TVKDNW2C21vC/fRZYK5N3e69hY7NAT8Jt2tmavXMIu/NeBYvabVq1BWDJGlxC92/liQ1kMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYoYDJKkiMEgSYr8Ap2dgaeqGU73AAAAAElFTkSuQmCC\n", 30 | "text/plain": [ 31 | "
" 32 | ] 33 | }, 34 | "metadata": { 35 | "needs_background": "light" 36 | }, 37 | "output_type": "display_data" 38 | } 39 | ], 40 | "source": [ 41 | "# We could use np.tanh, but let's write our own as an example.\n", 42 | "\n", 43 | "def tanh(x):\n", 44 | " return (1.0 - np.exp(-x)) / (1.0 + np.exp(-x))\n", 45 | "\n", 46 | "x = np.linspace(-7, 7, 200)\n", 47 | "plt.plot(x, tanh(x),\n", 48 | " x, grad(tanh)(x), # first derivative\n", 49 | " x, grad(grad(tanh))(x), # second derivative\n", 50 | " x, grad(grad(grad(tanh)))(x), # third derivative\n", 51 | " x, grad(grad(grad(grad(tanh))))(x), # fourth derivative\n", 52 | " x, grad(grad(grad(grad(grad(tanh)))))(x), # fifth derivative\n", 53 | " x, grad(grad(grad(grad(grad(grad(tanh))))))(x)) # sixth derivative\n", 54 | "\n", 55 | "plt.axis('off')\n", 56 | "plt.savefig(\"tanh.png\")\n", 57 | "plt.show()" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 4, 63 | "metadata": {}, 64 | "outputs": [ 65 | { 66 | "name": "stdout", 67 | "output_type": "stream", 68 | "text": [ 69 | "3.0\n", 70 | "\n", 71 | "[[3.]\n", 72 | " [3.]]\n" 73 | ] 74 | } 75 | ], 76 | "source": [ 77 | "# How does Autograd deal with broadcasting?\n", 78 | "\n", 79 | "from autograd.numpy.numpy_vjps import unbroadcast\n", 80 | "\n", 81 | "x = 2.0\n", 82 | "\n", 83 | "# Let f(x) = x + [1, 1, 1].\n", 84 | "#\n", 85 | "# If x[], then it is numpy first prepends as many length-1 dims as necessary (in this case, 1) to get\n", 86 | "# x[1], then copies along those length-1 dims to match [1, 1, 1] (in this case, 3x).\n", 87 | "#\n", 88 | "# We can look at f(x) as f(x_1=x, x_2=x, x_3=x). Then the total derivative of f() wrt x is,\n", 89 | "# df/dx = \\sum_{i} (df/dx_i) (dx_i/dx)\n", 90 | "# = \\sum_{i} 1 * 1\n", 91 | "# = 3 = number of times x was copied.\n", 92 | "\n", 93 | "def f(x):\n", 94 | " return x + np.ones((3,))\n", 95 | "\n", 96 | "x = 2.0\n", 97 | "y = f(x)\n", 98 | "g = np.ones(np.shape(y))\n", 99 | "print unbroadcast(x, g)\n", 100 | "\n", 101 | "# Any time numpy needs to broadcast, it clones the broadcasted value along leading dimensions.\n", 102 | "# During backpropagation, we sum over these dimensions as total derivative requires.\n", 103 | "print ''\n", 104 | "\n", 105 | "x = np.arange(2).reshape((2, 1))\n", 106 | "y = f(x)\n", 107 | "g = np.ones(np.shape(y))\n", 108 | "print unbroadcast(x, g)" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 5, 114 | "metadata": {}, 115 | "outputs": [ 116 | { 117 | "data": { 118 | "text/plain": [ 119 | "array(1.)" 120 | ] 121 | }, 122 | "execution_count": 5, 123 | "metadata": {}, 124 | "output_type": "execute_result" 125 | } 126 | ], 127 | "source": [ 128 | "# What does Autograd do when a function has non-differentiable functions?\n", 129 | "#\n", 130 | "# It assumes their derivative is zero wrt their inputs.\n", 131 | "\n", 132 | "def foo(x):\n", 133 | " return np.floor(x) + x\n", 134 | "\n", 135 | "grad(foo)(1.5)" 136 | ] 137 | } 138 | ], 139 | "metadata": { 140 | "kernelspec": { 141 | "display_name": "Python 2", 142 | "language": "python", 143 | "name": "python2" 144 | }, 145 | "language_info": { 146 | "codemirror_mode": { 147 | "name": "ipython", 148 | "version": 2 149 | }, 150 | "file_extension": ".py", 151 | "mimetype": "text/x-python", 152 | "name": "python", 153 | "nbconvert_exporter": "python", 154 | "pygments_lexer": "ipython2", 155 | "version": "2.7.13" 156 | } 157 | }, 158 | "nbformat": 4, 159 | "nbformat_minor": 2 160 | } 161 | --------------------------------------------------------------------------------