├── comp_graph.png
├── grad
    ├── __init__.py
    ├── variable.py
    └── gate.py
├── README.md
└── nn.ipynb


/comp_graph.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AlisaLC/GraDino/HEAD/comp_graph.png


--------------------------------------------------------------------------------
/grad/__init__.py:
--------------------------------------------------------------------------------
 1 | from . import variable as vp
 2 | from .variable import Variable as vn
 3 | 
 4 | class no_grad:
 5 |     def __init__(self):
 6 |         pass
 7 | 
 8 |     def __enter__(self):
 9 |         vp.is_grad_enabled = False
10 | 
11 |     def __exit__(self, exc_type, exc_value, traceback):
12 |         vp.is_grad_enabled = True


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # GraDino
 2 | an autograd package in python. GraDino variables can be used in lists, tuples or even `numpy` arrays!
 3 | gradients are calculated using backpropagation. examples of linear regression and XOR neural network is in the `nn.ipynb` notebook.
 4 | ## Basic Usage
 5 | a `Variable` can be defined using a `float` or `int` datatype. every operation on a `Variable` results in another `Variable`. finally when `backward` method is called on a `Variable`. every `Variable`'s gradient is calculated using backpropagation and stored in `grad` property.
 6 | ```python
 7 | import grad
 8 | from grad.variable import Variable as vn
 9 | 
10 | x = vn(1.0, requires_grad=True)
11 | y = vn(2.0, requires_grad=True)
12 | z = (x - y) ** 2
13 | z.backward()
14 | print(x.grad, y.grad) # -2.0 2.0
15 | ````
16 | 
17 | ## Numpy Arrays
18 | an autograding array can be defined using `array` function in the module. it can work with lists and numpy arryas. to extract the gradients we can use `array_grad` function to extract the gradients.
19 | ```python
20 | import grad
21 | from grad import variable as vp
22 | 
23 | x = vp.array(np.random.randn(3, 10))
24 | y = vp.array(np.random.randn(10, 3))
25 | z = np.mean((x - y.T) ** 2)
26 | z.backward()
27 | print(vp.array_grad(x))
28 | print(vp.array_grad(y))
29 | ```
30 | ## Gradient Calculation Techniques
31 | * if we want to define a variable where there is no need for backpropagation we can set `requires_grad` attribute to `False`.
32 | * to zero the calculated gradients after applying optimization in each iteration, `zero_grad` method should be called on each `Variable`. to apply this in an array we can use `array_zero_grad` function.
33 | * to disable computation graph during optimization, `with vp.no_grad():` can be used.
34 | * `zero_grad` can be called on the final product to reset the gradients recursively.
35 | ```python
36 | import grad
37 | from grad import variable as vp
38 | 
39 | x = vn(1.0, requires_grad=True)
40 | y = vn(2.0, requires_grad=True)
41 | z = (x - y) ** 2
42 | z.backward()
43 | print(x.grad, y.grad) # -2.0 2.0
44 | z.zero_grad()
45 | print(x.grad, y.grad) # 0.0 0.0
46 | ```
47 | ## Computational Graph Drawing
48 | to draw the computational graph of a `Variable` we can use `draw_graph` function. it uses `graphviz` library to draw the graph. the graph is returned as a `graphviz.Digraph` object.
49 | ```python
50 | import grad
51 | from grad import variable as vp
52 | 
53 | x = vn(1.0, requires_grad=True)
54 | y = vn(2.0, requires_grad=True)
55 | z = (x - y) ** 2
56 | z.backward()
57 | g = z.draw_graph()
58 | g.view()
59 | ```
60 | <p align="center">
61 |   <img  src="comp_graph.png">
62 | </p>
63 | 
64 | ## Higher Order Derivatives
65 | to calculate higher order derivatives we can use `backward` method multiple times. the gradients are accumulated in `grad` property. `make_graph` argument on `backward` must be set to `True` to make the computational graph. after each `backward` we `zero_grad` to reset the gradients.
66 | ```python
67 | import grad
68 | from grad import variable as vp
69 | 
70 | x = vn(17.0, requires_grad=True)
71 | y = vn(13.0, requires_grad=True)
72 | z = (x - y) ** 2
73 | z.backward(make_graph=True)
74 | print(x.grad, y.grad) # 8.0 -8.0
75 | x_grad = x.grad # 2 * (x - y)
76 | z.zero_grad()
77 | x_grad.backward()
78 | print(x.grad, y.grad) # 2.0 -2.0
79 | ```
80 | when `make_graph` is set to `True` the computational graph is made and stored in `graph` property of `Variable`. we can use `draw_graph` method to draw the graph.


--------------------------------------------------------------------------------
/grad/variable.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | from . import gate as g
  3 | import graphviz
  4 | 
  5 | is_grad_enabled = True
  6 | 
  7 | 
  8 | class Variable:
  9 |     def __init__(self, data, gate=None, requires_grad=True):
 10 |         if isinstance(data, Variable):
 11 |             self = data
 12 |             return
 13 |         elif isinstance(data, (int, np.uint8, np.uint16, np.uint32, np.uint64, np.int8, np.int16, np.int32, np.int64)):
 14 |             data = int(data)
 15 |         elif isinstance(data, (float, np.float16, np.float32, np.float64)):
 16 |             data = float(data)
 17 |         else:
 18 |             raise TypeError('expected int or float, got ' +
 19 |                             str(type(data)) + " instead")
 20 |         self.data = data
 21 |         self.grad = 0
 22 |         if gate is None or not is_grad_enabled:
 23 |             self.gate = g.Identity()
 24 |         else:
 25 |             self.gate = gate
 26 |         self.requires_grad = requires_grad
 27 |         self.is_graph = False
 28 | 
 29 |     def __repr__(self):
 30 |         return f"{self.data}"
 31 | 
 32 |     def __int__(self):
 33 |         return int(self.data)
 34 | 
 35 |     def __float__(self):
 36 |         return float(self.data)
 37 | 
 38 |     def __str__(self):
 39 |         return str(self.data)
 40 |     
 41 |     def __format__(self, *args, **kwargs):
 42 |         return self.data.__format__(*args, **kwargs)
 43 | 
 44 |     def backward(self, grad=None, make_graph=False):
 45 |         if not self.requires_grad or not is_grad_enabled:
 46 |             return
 47 |         if grad is None:
 48 |             if make_graph:
 49 |                 grad = Variable(1, requires_grad=False)
 50 |             else:
 51 |                 grad = 1
 52 |         self.grad += grad
 53 |         self.gate.backward(grad=grad, make_graph=make_graph)
 54 |     
 55 |     def draw_graph(self, graph=None):
 56 |         if graph is None:
 57 |             graph = graphviz.Digraph()
 58 |         if not self.is_graph:
 59 |             if self.gate is not None and self.gate.name == 'identity' and self.requires_grad:
 60 |                 graph.node(str(id(self)), f'{self.data:.4g}', style='filled', fillcolor='lightblue')
 61 |             else:
 62 |                 graph.node(str(id(self)), f'{self.data:.4g}')
 63 |             self.is_graph = True
 64 |         if self.gate is not None and self.gate.name != 'identity':
 65 |             graph.node(str(id(self.gate)), self.gate.name, style='filled', fillcolor='lightgreen')
 66 |             label = None
 67 |             if self.requires_grad:
 68 |                 label = f'{self.grad:.4g}'
 69 |             graph.edge(str(id(self.gate)), str(id(self)), label=label)
 70 |             self.gate.draw_graph(graph)
 71 |         return graph
 72 |     
 73 |     def clear_graph(self):
 74 |         self.is_graph = False
 75 |         self.gate.clear_graph()
 76 | 
 77 | 
 78 |     def zero_grad(self):
 79 |         if not self.requires_grad or not is_grad_enabled:
 80 |             return
 81 |         self.grad = 0
 82 |         self.gate.zero_grad()
 83 | 
 84 |     def __eq__(self, other):
 85 |         if not isinstance(other, Variable):
 86 |             other = Variable(other, requires_grad=False)
 87 |         return self.data == other.data
 88 | 
 89 |     def __lt__(self, other):
 90 |         if not isinstance(other, Variable):
 91 |             other = Variable(other, requires_grad=False)
 92 |         return self.data < other.data
 93 | 
 94 |     def __gt__(self, other):
 95 |         if not isinstance(other, Variable):
 96 |             other = Variable(other, requires_grad=False)
 97 |         return self.data > other.data
 98 | 
 99 |     def __le__(self, other):
100 |         if not isinstance(other, Variable):
101 |             other = Variable(other, requires_grad=False)
102 |         return self.data <= other.data
103 | 
104 |     def __ge__(self, other):
105 |         if not isinstance(other, Variable):
106 |             other = Variable(other, requires_grad=False)
107 |         return self.data >= other.data
108 | 
109 |     def __ne__(self, other):
110 |         if not isinstance(other, Variable):
111 |             other = Variable(other, requires_grad=False)
112 |         return self.data != other.data
113 | 
114 |     def __pos__(self):
115 |         return self
116 | 
117 |     def __neg__(self):
118 |         gate = g.Neg()
119 |         return Variable(gate(self), gate=gate)
120 | 
121 |     def __abs__(self):
122 |         gate = g.Abs()
123 |         return Variable(gate(self), gate=gate)
124 | 
125 |     def __add__(self, other):
126 |         if not isinstance(other, Variable):
127 |             other = Variable(other, requires_grad=False)
128 |         gate = g.Add()
129 |         return Variable(gate(self, other), gate=gate)
130 | 
131 |     def __radd__(self, other):
132 |         return self.__add__(other)
133 | 
134 |     def __sub__(self, other):
135 |         if not isinstance(other, Variable):
136 |             other = Variable(other, requires_grad=False)
137 |         gate = g.Neg()
138 |         other = Variable(gate(other), gate=gate)
139 |         gate = g.Add()
140 |         return Variable(gate(self, other), gate=gate)
141 | 
142 |     def __rsub__(self, other):
143 |         if not isinstance(other, Variable):
144 |             other = Variable(other, requires_grad=False)
145 |         return other - self
146 | 
147 |     def __mul__(self, other):
148 |         if not isinstance(other, Variable):
149 |             other = Variable(other, requires_grad=False)
150 |         gate = g.Mul()
151 |         return Variable(gate(self, other), gate=gate)
152 | 
153 |     def __rmul__(self, other):
154 |         return self.__mul__(other)
155 | 
156 |     def __truediv__(self, other):
157 |         if not isinstance(other, Variable):
158 |             other = Variable(other, requires_grad=False)
159 |         gate = g.Div()
160 |         return Variable(gate(self, other), gate=gate)
161 | 
162 |     def __rtruediv__(self, other):
163 |         if not isinstance(other, Variable):
164 |             other = Variable(other, requires_grad=False)
165 |         return other / self
166 | 
167 |     def __pow__(self, other):
168 |         if not isinstance(other, Variable):
169 |             other = Variable(other, requires_grad=False)
170 |         gate = g.Pow()
171 |         return Variable(gate(self, other), gate=gate)
172 | 
173 |     def __rpow__(self, other):
174 |         if not isinstance(other, Variable):
175 |             other = Variable(other, requires_grad=False)
176 |         return other ** self
177 | 
178 |     def sqrt(self):
179 |         return self ** 0.5
180 | 
181 |     def sin(self):
182 |         gate = g.Sin()
183 |         return Variable(gate(self), gate=gate)
184 | 
185 |     def arcsin(self):
186 |         gate = g.Asin()
187 |         return Variable(gate(self), gate=gate)
188 | 
189 |     def sinh(self):
190 |         gate = g.Sinh()
191 |         return Variable(gate(self), gate=gate)
192 | 
193 |     def arcsinh(self):
194 |         gate = g.Asinh()
195 |         return Variable(gate(self), gate=gate)
196 | 
197 |     def cos(self):
198 |         gate = g.Cos()
199 |         return Variable(gate(self), gate=gate)
200 | 
201 |     def arccos(self):
202 |         gate = g.Acos()
203 |         return Variable(gate(self), gate=gate)
204 | 
205 |     def cosh(self):
206 |         gate = g.Cosh()
207 |         return Variable(gate(self), gate=gate)
208 | 
209 |     def arccosh(self):
210 |         gate = g.Acosh()
211 |         return Variable(gate(self), gate=gate)
212 | 
213 |     def tan(self):
214 |         gate = g.Tan()
215 |         return Variable(gate(self), gate=gate)
216 | 
217 |     def arctan(self):
218 |         gate = g.Atan()
219 |         return Variable(gate(self), gate=gate)
220 | 
221 |     def tanh(self):
222 |         gate = g.Tanh()
223 |         return Variable(gate(self), gate=gate)
224 | 
225 |     def arctanh(self):
226 |         gate = g.Atanh()
227 |         return Variable(gate(self), gate=gate)
228 | 
229 |     def exp(self):
230 |         gate = g.Exp()
231 |         return Variable(gate(self), gate=gate)
232 | 
233 |     def log(self):
234 |         gate = g.Log()
235 |         return Variable(gate(self), gate=gate)
236 | 
237 |     def conjugate(self):
238 |         return self
239 | 
240 | 
241 | def array(data, requires_grad=True):
242 |     if isinstance(data, Variable):
243 |         return data
244 |     if isinstance(data, (list, tuple)):
245 |         return [array(x, requires_grad=requires_grad) for x in data]
246 |     if isinstance(data, np.ndarray):
247 |         return np.array([array(x, requires_grad=requires_grad) for x in data])
248 |     return Variable(data, requires_grad=requires_grad)
249 | 
250 | 
251 | def array_grad(data):
252 |     if isinstance(data, Variable):
253 |         return data.grad
254 |     if isinstance(data, (list, tuple)):
255 |         return [array_grad(x) for x in data]
256 |     if isinstance(data, np.ndarray):
257 |         return np.array([array_grad(x) for x in data])
258 |     return data
259 | 
260 | 
261 | def array_zero_grad(data):
262 |     if isinstance(data, Variable):
263 |         data.zero_grad()
264 |     elif isinstance(data, (list, tuple)):
265 |         for x in data:
266 |             array_zero_grad(x)
267 |     elif isinstance(data, np.ndarray):
268 |         for x in data:
269 |             array_zero_grad(x)
270 | 


--------------------------------------------------------------------------------
/grad/gate.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import graphviz
  3 | 
  4 | 
  5 | class Gate:
  6 |     def __init__(self, name):
  7 |         self.name = name
  8 |         self.vars = []
  9 | 
 10 |     def __repr__(self):
 11 |         return f"Gate({self.name})"
 12 | 
 13 |     def forward(self):
 14 |         raise NotImplementedError
 15 | 
 16 |     def backward(self, grad, make_graph=False):
 17 |         raise NotImplementedError
 18 | 
 19 |     def __call__(self, *args, **kwds):
 20 |         return self.forward(*args, **kwds)
 21 | 
 22 |     def draw_graph(self, graph):
 23 |         for var in self.vars:
 24 |             if not var.is_graph:
 25 |                 if var.gate is not None and var.gate.name == 'identity' and var.requires_grad:
 26 |                     graph.node(
 27 |                         str(id(var)), f'{var.data:.4g}', style='filled', fillcolor='lightblue')
 28 |                 else:
 29 |                     graph.node(str(id(var)), f'{var.data:.4g}')
 30 |                 var.is_graph = True
 31 |             label = None
 32 |             if var.requires_grad:
 33 |                 label = f'{var.grad:.4g}'
 34 |             graph.edge(str(id(var)), str(id(self)), label=label)
 35 |             var.draw_graph(graph)
 36 |     
 37 |     def clear_graph(self):
 38 |         for var in self.vars:
 39 |             var.clear_graph()
 40 | 
 41 |     def zero_grad(self):
 42 |         for var in self.vars:
 43 |             var.zero_grad()
 44 | 
 45 | 
 46 | class Identity(Gate):
 47 |     def __init__(self):
 48 |         super().__init__("identity")
 49 | 
 50 |     def forward(self, x):
 51 |         return x
 52 | 
 53 |     def backward(self, grad, make_graph=False):
 54 |         return grad
 55 | 
 56 | 
 57 | class Add(Gate):
 58 |     def __init__(self):
 59 |         super().__init__("add")
 60 | 
 61 |     def forward(self, x, y):
 62 |         self.vars = [x, y]
 63 |         return x.data + y.data
 64 | 
 65 |     def backward(self, grad, make_graph=False):
 66 |         self.vars[0].backward(grad, make_graph=make_graph)
 67 |         self.vars[1].backward(grad, make_graph=make_graph)
 68 | 
 69 | 
 70 | class Mul(Gate):
 71 |     def __init__(self):
 72 |         super().__init__("mul")
 73 | 
 74 |     def forward(self, x, y):
 75 |         self.vars = [x, y]
 76 |         return x.data * y.data
 77 | 
 78 |     def backward(self, grad, make_graph=False):
 79 |         var0, var1 = self.vars[0], self.vars[1]
 80 |         if not make_graph:
 81 |             var0, var1 = var0.data, var1.data
 82 |         self.vars[0].backward(grad * var1, make_graph=make_graph)
 83 |         self.vars[1].backward(grad * var0, make_graph=make_graph)
 84 | 
 85 | 
 86 | class Neg(Gate):
 87 |     def __init__(self):
 88 |         super().__init__("neg")
 89 | 
 90 |     def forward(self, x):
 91 |         self.vars = [x]
 92 |         return -x.data
 93 | 
 94 |     def backward(self, grad, make_graph=False):
 95 |         self.vars[0].backward(-grad, make_graph=make_graph)
 96 | 
 97 | 
 98 | class Abs(Gate):
 99 |     def __init__(self):
100 |         super().__init__("abs")
101 | 
102 |     def forward(self, x):
103 |         self.vars = [x]
104 |         return abs(x.data)
105 | 
106 |     def backward(self, grad, make_graph=False):
107 |         self.vars[0].backward(
108 |             grad * (-1 if self.vars[0].data < 0 else 1), make_graph=make_graph)
109 | 
110 | 
111 | class Div(Gate):
112 |     def __init__(self):
113 |         super().__init__("div")
114 | 
115 |     def forward(self, x, y):
116 |         self.vars = [x, y]
117 |         return x.data / y.data
118 | 
119 |     def backward(self, grad, make_graph=False):
120 |         var0, var1 = self.vars[0], self.vars[1]
121 |         if not make_graph:
122 |             var0, var1 = var0.data, var1.data
123 |         if self.vars[0].requires_grad:
124 |             self.vars[0].backward(grad / var1, make_graph=make_graph)
125 |         if self.vars[1].requires_grad:
126 |             self.vars[1].backward(-grad * var0 / var1 **
127 |                                   2, make_graph=make_graph)
128 | 
129 | 
130 | class Pow(Gate):
131 |     def __init__(self):
132 |         super().__init__("pow")
133 | 
134 |     def forward(self, x, y):
135 |         self.vars = [x, y]
136 |         return x.data ** y.data
137 | 
138 |     def backward(self, grad, make_graph=False):
139 |         var0, var1 = self.vars[0], self.vars[1]
140 |         if not make_graph:
141 |             var0, var1 = var0.data, var1.data
142 |         if self.vars[0].requires_grad:
143 |             self.vars[0].backward(grad * var1 * var0 **
144 |                                   (var1 - 1), make_graph=make_graph)
145 |         if self.vars[1].requires_grad:
146 |             self.vars[1].backward(grad * var0 ** var1 *
147 |                                   np.log(var0), make_graph=make_graph)
148 | 
149 | 
150 | class Sin(Gate):
151 |     def __init__(self):
152 |         super().__init__("sin")
153 | 
154 |     def forward(self, x):
155 |         self.vars = [x]
156 |         return np.sin(x.data)
157 | 
158 |     def backward(self, grad, make_graph=False):
159 |         var0 = self.vars[0]
160 |         if not make_graph:
161 |             var0 = var0.data
162 |         self.vars[0].backward(grad * np.cos(var0), make_graph=make_graph)
163 | 
164 | 
165 | class Asin(Gate):
166 |     def __init__(self):
167 |         super().__init__("asin")
168 | 
169 |     def forward(self, x):
170 |         self.vars = [x]
171 |         return np.arcsin(x.data)
172 | 
173 |     def backward(self, grad, make_graph=False):
174 |         var0 = self.vars[0]
175 |         if not make_graph:
176 |             var0 = var0.data
177 |         self.vars[0].backward(
178 |             grad / np.sqrt(1 - var0 ** 2), make_graph=make_graph)
179 | 
180 | 
181 | class Sinh(Gate):
182 |     def __init__(self):
183 |         super().__init__("sinh")
184 | 
185 |     def forward(self, x):
186 |         self.vars = [x]
187 |         return np.sinh(x.data)
188 | 
189 |     def backward(self, grad, make_graph=False):
190 |         var0 = self.vars[0]
191 |         if not make_graph:
192 |             var0 = var0.data
193 |         self.vars[0].backward(grad * np.cosh(var0), make_graph=make_graph)
194 | 
195 | 
196 | class Asinh(Gate):
197 |     def __init__(self):
198 |         super().__init__("asinh")
199 | 
200 |     def forward(self, x):
201 |         self.vars = [x]
202 |         return np.arcsinh(x.data)
203 | 
204 |     def backward(self, grad, make_graph=False):
205 |         var0 = self.vars[0]
206 |         if not make_graph:
207 |             var0 = var0.data
208 |         self.vars[0].backward(
209 |             grad / np.sqrt(1 + var0 ** 2), make_graph=make_graph)
210 | 
211 | 
212 | class Cos(Gate):
213 |     def __init__(self):
214 |         super().__init__("cos")
215 | 
216 |     def forward(self, x):
217 |         self.vars = [x]
218 |         return np.cos(x.data)
219 | 
220 |     def backward(self, grad, make_graph=False):
221 |         var0 = self.vars[0]
222 |         if not make_graph:
223 |             var0 = var0.data
224 |         self.vars[0].backward(-grad * np.sin(var0), make_graph=make_graph)
225 | 
226 | 
227 | class Acos(Gate):
228 |     def __init__(self):
229 |         super().__init__("acos")
230 | 
231 |     def forward(self, x):
232 |         self.vars = [x]
233 |         return np.arccos(x.data)
234 | 
235 |     def backward(self, grad, make_graph=False):
236 |         var0 = self.vars[0]
237 |         if not make_graph:
238 |             var0 = var0.data
239 |         self.vars[0].backward(-grad / np.sqrt(1 - var0 **
240 |                               2), make_graph=make_graph)
241 | 
242 | 
243 | class Cosh(Gate):
244 |     def __init__(self):
245 |         super().__init__("cosh")
246 | 
247 |     def forward(self, x):
248 |         self.vars = [x]
249 |         return np.cosh(x.data)
250 | 
251 |     def backward(self, grad, make_graph=False):
252 |         var0 = self.vars[0]
253 |         if not make_graph:
254 |             var0 = var0.data
255 |         self.vars[0].backward(grad * np.sinh(var0), make_graph=make_graph)
256 | 
257 | 
258 | class Acosh(Gate):
259 |     def __init__(self):
260 |         super().__init__("acosh")
261 | 
262 |     def forward(self, x):
263 |         self.vars = [x]
264 |         return np.arccosh(x.data)
265 | 
266 |     def backward(self, grad, make_graph=False):
267 |         var0 = self.vars[0]
268 |         if not make_graph:
269 |             var0 = var0.data
270 |         self.vars[0].backward(
271 |             grad / np.sqrt(var0 ** 2 - 1), make_graph=make_graph)
272 | 
273 | 
274 | class Tan(Gate):
275 |     def __init__(self):
276 |         super().__init__("tan")
277 | 
278 |     def forward(self, x):
279 |         self.vars = [x]
280 |         return np.tan(x.data)
281 | 
282 |     def backward(self, grad, make_graph=False):
283 |         var0 = self.vars[0]
284 |         if not make_graph:
285 |             var0 = var0.data
286 |         self.vars[0].backward(grad / np.cos(var0) ** 2, make_graph=make_graph)
287 | 
288 | 
289 | class Atan(Gate):
290 |     def __init__(self):
291 |         super().__init__("atan")
292 | 
293 |     def forward(self, x):
294 |         self.vars = [x]
295 |         return np.arctan(x.data)
296 | 
297 |     def backward(self, grad, make_graph=False):
298 |         var0 = self.vars[0]
299 |         if not make_graph:
300 |             var0 = var0.data
301 |         self.vars[0].backward(grad / (1 + var0 ** 2), make_graph=make_graph)
302 | 
303 | 
304 | class Tanh(Gate):
305 |     def __init__(self):
306 |         super().__init__("tanh")
307 | 
308 |     def forward(self, x):
309 |         self.vars = [x]
310 |         return np.tanh(x.data)
311 | 
312 |     def backward(self, grad, make_graph=False):
313 |         var0 = self.vars[0]
314 |         if not make_graph:
315 |             var0 = var0.data
316 |         self.vars[0].backward(
317 |             grad * (1 - np.tanh(var0) ** 2), make_graph=make_graph)
318 | 
319 | 
320 | class Atanh(Gate):
321 |     def __init__(self):
322 |         super().__init__("atanh")
323 | 
324 |     def forward(self, x):
325 |         self.vars = [x]
326 |         return np.arctanh(x.data)
327 | 
328 |     def backward(self, grad, make_graph=False):
329 |         var0 = self.vars[0]
330 |         if not make_graph:
331 |             var0 = var0.data
332 |         self.vars[0].backward(grad / (1 - var0 ** 2), make_graph=make_graph)
333 | 
334 | 
335 | class Exp(Gate):
336 |     def __init__(self):
337 |         super().__init__("exp")
338 | 
339 |     def forward(self, x):
340 |         self.vars = [x]
341 |         return np.exp(x.data)
342 | 
343 |     def backward(self, grad, make_graph=False):
344 |         var0 = self.vars[0]
345 |         if not make_graph:
346 |             var0 = var0.data
347 |         self.vars[0].backward(grad * np.exp(var0), make_graph=make_graph)
348 | 
349 | 
350 | class Log(Gate):
351 |     def __init__(self):
352 |         super().__init__("log")
353 | 
354 |     def forward(self, x):
355 |         self.vars = [x]
356 |         return np.log(x.data)
357 | 
358 |     def backward(self, grad, make_graph=False):
359 |         var0 = self.vars[0]
360 |         if not make_graph:
361 |             var0 = var0.data
362 |         self.vars[0].backward(grad / var0, make_graph=make_graph)
363 | 


--------------------------------------------------------------------------------
/nn.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 1,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "import numpy as np\n",
 10 |     "import grad\n",
 11 |     "from grad import variable as vp\n",
 12 |     "from grad.variable import Variable as vn"
 13 |    ]
 14 |   },
 15 |   {
 16 |    "cell_type": "code",
 17 |    "execution_count": 2,
 18 |    "metadata": {},
 19 |    "outputs": [
 20 |     {
 21 |      "name": "stdout",
 22 |      "output_type": "stream",
 23 |      "text": [
 24 |       "8.0 -8.0\n",
 25 |       "2.0 -2.0\n"
 26 |      ]
 27 |     },
 28 |     {
 29 |      "data": {
 30 |       "text/plain": [
 31 |        "'x_grad.pdf'"
 32 |       ]
 33 |      },
 34 |      "execution_count": 2,
 35 |      "metadata": {},
 36 |      "output_type": "execute_result"
 37 |     }
 38 |    ],
 39 |    "source": [
 40 |     "x = vn(17.0, requires_grad=True)\n",
 41 |     "y = vn(13.0, requires_grad=True)\n",
 42 |     "z = (x - y) ** 2\n",
 43 |     "z.backward(make_graph=True)\n",
 44 |     "print(x.grad, y.grad)\n",
 45 |     "z.draw_graph().render('z')\n",
 46 |     "z.clear_graph()\n",
 47 |     "x_grad = x.grad\n",
 48 |     "z.zero_grad()\n",
 49 |     "x_grad.backward()\n",
 50 |     "print(x.grad, y.grad)\n",
 51 |     "x_grad.draw_graph().render('x_grad')"
 52 |    ]
 53 |   },
 54 |   {
 55 |    "attachments": {},
 56 |    "cell_type": "markdown",
 57 |    "metadata": {},
 58 |    "source": [
 59 |     "## Linear Regression"
 60 |    ]
 61 |   },
 62 |   {
 63 |    "cell_type": "code",
 64 |    "execution_count": 3,
 65 |    "metadata": {},
 66 |    "outputs": [
 67 |     {
 68 |      "name": "stderr",
 69 |      "output_type": "stream",
 70 |      "text": [
 71 |       "100%|██████████| 300/300 [00:00<00:00, 2153.92it/s]\n"
 72 |      ]
 73 |     },
 74 |     {
 75 |      "data": {
 76 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAGdCAYAAADqsoKGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABH2ElEQVR4nO3deVgTd+I/8PckkHAIQURBBMXbIhoUJaStrQcttV1XbT3rgahobevaZdtd7Xe3bnf3V/utu92uC1tRq3iLdqu9L2nVtnIoGO+7KCgCIpJAkADJ/P7Yb9OlYAUNmRzv1/PM8ywzk5l3Pps0b2cmGUEURRFERERETk4mdQAiIiIiW2CpISIiIpfAUkNEREQugaWGiIiIXAJLDREREbkElhoiIiJyCSw1RERE5BJYaoiIiMgleEgdwF4sFgtKSkrg5+cHQRCkjkNEREStIIoiqqurERoaCpns54/FuE2pKSkpQXh4uNQxiIiI6C4UFxcjLCzsZ9dxm1Lj5+cH4D+D4u/vL3EaIiIiag2DwYDw8HDr5/jPcZtS88MpJ39/f5YaIiIiJ9OaS0d4oTARERG5BJYaIiIicgksNUREROQSWGqIiIjIJbDUEBERkUtgqSEiIiKXwFJDRERELoGlhoiIiFwCSw0RERG5BJYaIiIicgksNUREROQSWGqIiIjIJbDU3COLRcRv3z2KnYeKpY5CRETk1tzmLt3t5cNjJdh5+Ap2Hr6CukYzZmsjpI5ERETklnik5h79Uh2KpAciAACvvH8Saw5clDYQERGRm2KpuUeCIOCVX0Ti2ZG9AQCvfXIG/9h7HqIoSpyMiIjIvbDU2IAgCPjtYwPw4qP9AAB/33sOb3x+lsWGiIjIjlhqbOj50X3x+yfuAwC8ve8iXv3wFCwWFhsiIiJ7YKmxsfkjeuHPE6IAABkHL+Hl3cdhZrEhIiJqdyw17WBWXA/8dbIaMgHYcagYv9mpQ6PZInUsIiIil8ZS004mxYThH9OGQC4TsEdXgsXbj6C+kcWGiIiovbDUtKNx6lC8PWMoFHIZPj1Rime25KOuwSx1LCIiIpfEUtPOHh0YgrWJw6D0kOGrM+WYv/EwausbpY5FRETkclhq7ODhfp2RkRQLH4Uc316oQOL6PFTXNUgdi4iIyKU4XamZOHEiOnbsiEmTJkkdpU20vTth8zwN/Lw8cOjSTcxcl4uq2nqpYxEREbkMpys1S5YswaZNm6SOcVdienTE9uQ4dPTxxNErekxfm4uKGpPUsYiIiFyC05WakSNHws/PT+oYdy2qmwo7FmgR1EGJ09cMmLYmB2WGOqljEREROT27lpoDBw5g3LhxCA0NhSAI2LNnT7N10tLSEBERAS8vL2g0GuTl5dkzol30D/HDzoVx6KrywoXyGkxJz8aVm7VSxyIiInJqdi01RqMRarUaaWlpLS7PzMxESkoKli9fjoKCAqjVaiQkJKC8vNyeMe2iV+cO2LlQi7CO3rh8oxZT03NwqcIodSwiIiKnZddSM3bsWPzlL3/BxIkTW1z+5ptvIjk5GUlJSYiMjMTq1avh4+OD9evXt3lfJpMJBoOhyeRowgN9sOsZLXoF+eJq1S1MSc/GhfJqqWMRERE5JYe5pqa+vh75+fmIj4+3zpPJZIiPj0d2dnabt7dixQqoVCrrFB4ebsu4NtNV5Y3MhVr0D/ZDebUJU9NzcKrE8QoYERGRo3OYUlNRUQGz2Yzg4OAm84ODg1FaWmr9Oz4+HpMnT8Ynn3yCsLCw2xaeZcuWQa/XW6fi4uJ2zX8vOvspsWNBHKK6+eOGsR7T1mRDV1wldSwiIiKn4iF1gLbau3dvq9ZTKpVQKpXtnMZ2OvoqsHV+HJI25KGgqAoz1+Vi/ZzhiO0ZKHU0IiIip+AwR2qCgoIgl8tRVlbWZH5ZWRlCQkIkSmVfKm9PbJ6nQVyvQNSYGpG4Pg/fnq+QOhYREZFTcJhSo1AoEBMTg6ysLOs8i8WCrKwsaLVaCZPZl6/SAxvmxOKhfp1xq8GMuRsP4aszZXd+IBERkZuza6mpqamBTqeDTqcDABQWFkKn06GoqAgAkJKSgrVr12Ljxo04ffo0Fi1aBKPRiKSkJHvGlJy3Qo61s2PwSGQw6hstWLg5H58evyZ1LCIiIocmiKIo2mtn+/btw6hRo5rNT0xMREZGBgAgNTUVK1euRGlpKaKjo7Fq1SpoNJp73rfBYIBKpYJer4e/v/89b88eGswWpOw8ig+PlkAmAH+bosbEIWFSxyIiIrKbtnx+27XUSMkZSw0AmC0ifvfvY3g3/woEAVgxcRCmxXaXOhYREZFdtOXz22GuqaGWyWUC3nhqMGbGdYcoAkvfO44N3xVKHYuIiMjhsNQ4AZlMwJ/HRyF5RE8AwKsfnsLb+y5KnIqIiMixsNQ4CUEQ8PLj9+FXo/sAAP73szN488tzcJOzh0RERHfEUuNEBEFAyqP98dvH+gMAVmWdx4pPz7DYEBERgaXGKT07sg+Wj4sEAKw58D1eef8kLBYWGyIicm8sNU4q6YGeeG3iIAgCsDnnMn7372Mws9gQEZEbY6lxYk9ruuPNKWrIBGBX/hW8kKlDg9kidSwiIiJJsNQ4uYlDwpD69FB4yAR8eLQEz20tgKnRLHUsIiIiu2OpcQGPD+qK9FkxUMhl+OJUGRZsykddA4sNERG5F5YaFzHmvmC8M2cYvDxl2H/uOpI2HILR1Ch1LCIiIrthqXEhI/p2xqa5Gvgq5Mj+/gZmr8+Doa5B6lhERER2wVLjYmJ7BmJrchz8vTyQf/kmZqzNxU1jvdSxiIiI2h1LjQuKDg/A9gVxCPRV4PhVPaavzcH1apPUsYiIiNoVS42LGhiqQuaCOHT2U+JMaTWmrsnGNf0tqWMRERG1G5YaF9Y32A87F2oRqvLC99eNmJKejeLKWqljERERtQuWGhfXM8gXO5/RonugD4orb2FKeja+v14jdSwiIiKbY6lxA2EdfbBzoRa9O/vimr4OU9JzcLa0WupYRERENsVS4yZCVF7IXKjFgBA/VNSYMG1NNk5c1Usdi4iIyGZYatxIUAcldiyIw+AwFW7WNmD62hwUFN2UOhYREZFNsNS4mQAfBbbM12BYj46ormvErHW5yPn+htSxiIiI7hlLjRvy9/LEpnmxuL93JxjrzZizIQ/7z12XOhYREdE9YalxUz4KD6yfMxyj+ndGXYMFyRsP48tTZVLHIiIiumssNW7My1OO9FnDMDYqBPVmCxZtycdHx0qkjkVERHRXWGrcnMJDhn9OH4Lx0aFotIj41fYjeDf/itSxiIiI2oylhuAhl+HNKdGYNjwcFhF4cddRbMm5LHUsIiKiNmGpIQCAXCbgtYmDMOf+CADA7/ecwLpvvpc2FBERURuw1JCVTCZg+bhIPPNwbwDAXz4+jdSvzkucioiIqHVYaqgJQRDwu8f649fx/QAAf/3iHFZ+fgaiKEqcjIiI6Oex1FAzgiBgSXxfLBs7AACQ9vVF/Pmj0yw2RETk0Jym1FRVVWHYsGGIjo5GVFQU1q5dK3Ukl7fw4d740/iBAID13xXif/acgMXCYkNERI7JQ+oAreXn54cDBw7Ax8cHRqMRUVFRePLJJ9GpUyepo7m02doIeHnI8bv3jmFbbhHqGsx446nB8JA7TR8mIiI34TSfTHK5HD4+PgAAk8kEURR5OsROpgwPx1tToyGXCXiv4CqWZOrQYLZIHYuIiKgJm5WaAwcOYNy4cQgNDYUgCNizZ0+zddLS0hAREQEvLy9oNBrk5eW1aR9VVVVQq9UICwvDSy+9hKCgIBulpzsZH90NaU8PhadcwMfHrmHRlnzUNZiljkVERGRls1JjNBqhVquRlpbW4vLMzEykpKRg+fLlKCgogFqtRkJCAsrLy63r/HC9zE+nkpL//HR/QEAAjh49isLCQmzbtg1lZbxXkT09FhWCNbOHQekhw97T5UjedBi36llsiIjIMQhiO5zDEQQBu3fvxoQJE6zzNBoNhg8fjtTUVACAxWJBeHg4Fi9ejKVLl7Z5H88++yxGjx6NSZMmtbjcZDLBZDJZ/zYYDAgPD4der4e/v3+b90c/OnihAvM3HUZtvRmxPQOxfs5wdFA6zeVZRETkRAwGA1QqVas+v+1yTU19fT3y8/MRHx//445lMsTHxyM7O7tV2ygrK0N1dTUAQK/X48CBA+jfv/9t11+xYgVUKpV1Cg8Pv7cnQVb39wnCprmx8FN6IK+wEjPX5UJ/q0HqWERE5ObsUmoqKipgNpsRHBzcZH5wcDBKS0tbtY3Lly9jxIgRUKvVGDFiBBYvXoxBgwbddv1ly5ZBr9dbp+Li4nt6DtTUsIhAbE3WQOXtCV1xFZ5em4NKY73UsYiIyI05zTmD2NhY6HS6Vq+vVCqhVCrbLxBhcFgAdiyIw6x3cnGyxICp6dnYOl+DLv5eUkcjIiI3ZJcjNUFBQZDL5c0u7C0rK0NISIg9IlA7ua+rP3Ys0CLYX4nz5TWYuiYHJVW3pI5FRERuyC6lRqFQICYmBllZWdZ5FosFWVlZ0Gq19ohA7ahPlw7YuVCLbgHeKKwwYvLqbBTdqJU6FhERuRmblZqamhrodDrrKaLCwkLodDoUFRUBAFJSUrB27Vps3LgRp0+fxqJFi2A0GpGUlGSrCCShHp18sesZLSI6+eBq1S1MTj+IC+U1UsciIiI3YrOvdO/btw+jRo1qNj8xMREZGRkAgNTUVKxcuRKlpaWIjo7GqlWroNFobLH7O2rLV8Lo7pUb6jBjXS7Ol9cgqIMCm+dpcF9XjjcREd2dtnx+t8vv1Dgilhr7uVFjwqx38nDqmgEBPp7YNDcWg8MCpI5FREROyOF+p4bcS6cOSmxPjkN0eACqahswY20u8i9XSh2LiIhcHEsNtQuVjye2zNcgtmcgqk2NmPVOHg5eqJA6FhERuTCWGmo3HZQe2JgUixF9g1Bbb0ZSxiF8fbb8zg8kIiK6Cyw11K68FXKsnT0M8fd1ganRggWbDuOzE637FWkiIqK2YKmhduflKcfbM2PwxKCuaDCLeG5bAd7XXZU6FhERuRiWGrILT7kM/5gWjSeHdoPZIuKFTB12HuL9uIiIyHZYashuPOQy/HWSGk9rukMUgd/++xg2ZV+SOhYREbkIlhqyK5lMwP+bEIW5D/QEALzy/kmsOXBR4lREROQKWGrI7gRBwB9+cR+eG9UbAPDaJ2fwj73n4Sa/A0lERO2EpYYkIQgCXkoYgBcf7QcA+Pvec/jfz86y2BAR0V1jqSFJPT+6L37/xH0AgNX7L+LVD0/BYmGxISKitmOpIcnNH9ELf5kQBQDIOHgJL+8+DjOLDRERtRFLDTmEmXE98NfJasgEYMehYvxmpw6NZovUsYiIyImw1JDDmBQThlXTh8BDJmCPrgSLtx9BfSOLDRERtQ5LDTmUXwwOxdszY6CQy/DpiVI8syUfdQ1mqWMREZETYKkhh/NIZDDWJQ6Dl6cMX50px7yNh1Bb3yh1LCIicnAsNeSQHurXGRlJsfBRyPHdhRtIXJ+H6roGqWMREZEDY6khhxXXqxO2zNfAz8sDhy7dxMx1uaiqrZc6FhEROSiWGnJoQ7t3xPbkOHT08cTRK3pMW5ODihqT1LGIiMgBsdSQw4vqpsKOBVoEdVDiTGk1pqZno8xQJ3UsIiJyMCw15BT6h/hh58I4dFV54eJ1I6akZ+PKzVqpYxERkQNhqSGn0atzB+xcqEV4oDcu36jFlNXZuFRhlDoWERE5CJYacirhgT7YuVCLXkG+KNHXYUp6Ns6XVUsdi4iIHABLDTmdripvZC7UYkCIH8qrTZi6JgcnS/RSxyIiIomx1JBT6uynxPbkOAzqpkKlsR7T1+RAV1wldSwiIpIQSw05rY6+CmxN1iCmR0cY6hoxc10u8gorpY5FREQSYakhp+bv5YlNc2MR1ysQNaZGJK7Pw7fnK6SORUREEmCpIafnq/RARlIsHu7XGbcazJi78RC+OlMmdSwiIrIzpyk1Z8+eRXR0tHXy9vbGnj17pI5FDsLLU441s2PwaGQw6hstWLApH58cvyZ1LCIisiNBFEVR6hBtVVNTg4iICFy+fBm+vr6teozBYIBKpYJer4e/v387JySpNJgtSNl5FB8eLYFMAP42RY2JQ8KkjkVERHepLZ/fTnOk5r998MEHGDNmTKsLDbkPT7kMb02NxuSYMFhEIGXnUWzPK5I6FhER2YHNSs2BAwcwbtw4hIaGQhCEFk8NpaWlISIiAl5eXtBoNMjLy7urfe3cuRNTp069x8TkquQyAf/71GDMiusBUQSWvXccG74rlDoWERG1M5uVGqPRCLVajbS0tBaXZ2ZmIiUlBcuXL0dBQQHUajUSEhJQXl5uXSc6OhpRUVHNppKSEus6BoMBBw8exOOPP26r6OSCZDIBfxo/EAse6gUAePXDU/jXvgsSpyIiovbULtfUCIKA3bt3Y8KECdZ5Go0Gw4cPR2pqKgDAYrEgPDwcixcvxtKlS1u97c2bN+Pzzz/Hli1bfnY9k8kEk8lk/dtgMCA8PJzX1LgZURTx973nsSrrPADgV6P74NeP9IMgCBInIyKi1nC4a2rq6+uRn5+P+Pj4H3cskyE+Ph7Z2dlt2lZrTz2tWLECKpXKOoWHh7c5Nzk/QRCQ8kg//Pax/gCAVV9dwGufnIYTXh9PRER3YJdSU1FRAbPZjODg4Cbzg4ODUVpa2urt6PV65OXlISEh4Y7rLlu2DHq93joVFxe3OTe5jmdH9sHycZEAgLXfFOKV90/CYmGxISJyJR5SB2gLlUqFsrLW/aiaUqmEUqls50TkTJIe6AkvTzle3n0cm3Muo67BjNefGgy5jKeiiIhcgV2O1AQFBUEulzcrJGVlZQgJCbFHBCIAwPTY7nhzihpymYBd+VfwQqYODWaL1LGIiMgG7FJqFAoFYmJikJWVZZ1nsViQlZUFrVZrjwhEVhOHhCF1+hB4yAR8eLQEz20tgKnRLHUsIiK6RzYrNTU1NdDpdNDpdACAwsJC6HQ6FBX954fPUlJSsHbtWmzcuBGnT5/GokWLYDQakZSUZKsIRK02dlBXrJkdA4WHDF+cKsOCTfmoa2CxISJyZjb7Sve+ffswatSoZvMTExORkZEBAEhNTcXKlStRWlqK6OhorFq1ChqNxha7vyPeJoFa8u35CiRvOoxbDWZoe3XCusRh8FU61aVmREQurS2f305576e7wVJDt5NXWIm5GYdQY2rE0O4ByJgbC38vT6ljERERHPB3aogcWWzPQGyZr4G/lwcKiqowY20ubhrrpY5FRERtxFJDBCA6PAA7FmgR6KvA8at6TFuTg+vVpjs/kIiIHAZLDdH/iQz1R+aCOHTxU+JsWTWmpmfjmv6W1LGIiKiVWGqI/kvfYD/sXKhFtwBvfF9hxJT0bBRX1kodi4iIWoGlhugnIoJ8kbkwDj06+aC48hampGfj++s1UsciIqI7YKkhakFYRx/sXKhFny4dcE1fhynpOThbWi11LCIi+hksNUS3EezvhR0L4nBfV39U1JgwbU02TlzVSx2LiIhug6WG6GcEdVBie7IG6jAVbtY2YPraHBQU3ZQ6FhERtYClhugOAnwU2DJfg+ERHVFd14hZ63KR8/0NqWMREdFPsNQQtYKflyc2zo3FA306wVhvRuL6POw/d13qWERE9F9YaohayUfhgXcSh2P0gC4wNVqQvPEwvjxVJnUsIiL6Pyw1RG3g5SnH6pkxGBsVgnqzBYu25OPDoyVSxyIiIrDUELWZwkOGf04fggnRoWi0iFiy4wjezb8idSwiIrfHUkN0FzzkMvxtSjSmDQ+HRQRe3HUUW3IuSx2LiMitsdQQ3SW5TMCKJwdhzv0RAIDf7zmBdd98L20oIiI3xlJDdA8EQcDycZF45uHeAIC/fHwaqV+dlzgVEZF7YqkhukeCIOB3j/VHyiP9AAB//eIcVn5+BqIoSpyMiMi9sNQQ2YAgCPjVmL54+fEBAIC0ry/izx+dZrEhIrIjlhoiG1rwUG/8efxAAMD67wrxP3tOwGJhsSEisgeWGiIbm6WNwBuTBkMQgG25RXhx11E0mi1SxyIicnksNUTtYMqwcLw1NRpymYD3jlzFkh06NLDYEBG1K5YaonYyProb0p4eCk+5gI+PX8OiLfmoazBLHYuIyGWx1BC1o8eiQrB29jAoPWTYe7ocyZsO41Y9iw0RUXtgqSFqZyP7d8GGpOHwUcjxzfkKJG7IQ42pUepYREQuh6WGyA7u7x2EzfNi4af0QF5hJWauy4W+tkHqWERELoWlhshOYnoEYltyHAJ8PKErrsL0tTm4UWOSOhYRkctgqSGyo0FhKuxYEIegDgqcumbAtDU5KDfUSR2LiMglsNQQ2dmAEH9kLtQixN8L58trMCU9G1erbkkdi4jI6bHUEEmgd+cO2LlQi7CO3rh0oxZTVmej6Eat1LGIiJyaQ5aaiRMnomPHjpg0aVKblhE5k+6dfLBzoRY9g3xxteoWJqcfxIXyGqljERE5LYcsNUuWLMGmTZvavIzI2YQGeCNzYRz6BXdAmcGEqenZOH3NIHUsIiKn5JClZuTIkfDz82vzMiJn1MXPCzsWaDEw1B83jPWYtiYHx65USR2LiMjptLnUHDhwAOPGjUNoaCgEQcCePXuarZOWloaIiAh4eXlBo9EgLy/PFlmJXFagrwLbkuMwpHsA9LcaMGNtLg5fqpQ6FhGRU2lzqTEajVCr1UhLS2txeWZmJlJSUrB8+XIUFBRArVYjISEB5eXl1nWio6MRFRXVbCopKbn7Z0Lk5FTentg8T4PYnoGoNjVi1jt5OHihQupYREROw6OtDxg7dizGjh172+VvvvkmkpOTkZSUBABYvXo1Pv74Y6xfvx5Lly4FAOh0urtL2wYmkwkm048/bGYw8DoFcnwdlB7YmBSLBZsP45vzFUjKOITVs2Iwqn8XqaMRETk8m15TU19fj/z8fMTHx/+4A5kM8fHxyM7OtuWu7mjFihVQqVTWKTw83K77J7pb3go51iUOQ/x9wTA1WrBg02F8dqJU6lhERA7PpqWmoqICZrMZwcHBTeYHBwejtLT1/1GOj4/H5MmT8cknnyAsLKxJIfq5Zf9t2bJl0Ov11qm4uPjunhSRBJQecrw9cyieGNwVDWYRz20rwPu6q1LHIiJyaG0+/WQPe/fuvatl/02pVEKpVNoqEpHdecplWDVtCLw85Ph3wRW8kKmDqcGCKcN51JGIqCU2PVITFBQEuVyOsrKyJvPLysoQEhJiy10RuQW5TMDKSYMxQ9Mdogj89t/HsCn7ktSxiIgckk1LjUKhQExMDLKysqzzLBYLsrKyoNVqbbkrIrchkwn4y4QozHuwJwDglfdPIn3/RYlTERE5njaffqqpqcGFCxesfxcWFkKn0yEwMBDdu3dHSkoKEhMTMWzYMMTGxuKtt96C0Wi0fhuKiNpOEAT8/on74O0pR+rXF7Di0zO41WDGkjF9IQiC1PGIiBxCm0vN4cOHMWrUKOvfKSkpAIDExERkZGRg6tSpuH79Ol555RWUlpYiOjoan332WbOLh4mobQRBwIsJ/eGtkGPl52fx1t7zqGuw4HeP9WexISICIIiiKEodwh4MBgNUKhX0ej38/f2ljkN0T975thB//ugUAGDO/RF45ReRkMlYbIjI9bTl89sh7/1ERD9v3oM98f8mRgEAMg5ewsu7j8NscYt/nxAR3RZLDZGTmqHpgb9NVkMmADsOFSNlpw6NZovUsYiIJMNSQ+TEnooJwz+nD4WHTMD7uhI8v+0I6htZbIjIPbHUEDm5JwZ3xeqZMVDIZfjsZCkWbj6Mugaz1LGIiOyOpYbIBcRHBuOdOcPg5SnD12evY97GQ6itb5Q6FhGRXbHUELmIEX07IyMpFr4KOb67cAOz38lDdV2D1LGIiOyGpYbIhcT16oTN8zXw9/LA4cs3MWNdLqpq66WORURkFyw1RC5maPeO2JYch44+njh2RY9pa3JQUWOSOhYRUbtjqSFyQVHdVMhcqEVQByXOlFZjano2ygx1UsciImpXLDVELqpfsB92LoxDV5UXLl43Ykp6Nq7crJU6FhFRu2GpIXJhvTp3wM6FWoQHeuPyjVpMWZ2NSxVGqWMREbULlhoiFxce6INdC+9Hr86+KNHXYUp6Ns6XVUsdi4jI5lhqiNxAiMoLmQu0GBDih/JqE6auycHJEr3UsYiIbIqlhshNdPZTYntyHAZ1U6HSWI/pa3KgK66SOhYRkc2w1BC5kY6+CmxN1iCmR0cY6hoxc10u8gorpY5FRGQTLDVEbsbfyxOb5sZC26sTakyNmL0+F9+er5A6FhHRPWOpIXJDvkoPbEgajpH9O6OuwYK5Gw8h63SZ1LGIiO4JSw2Rm/LylCN9VgwSBgajvtGChZvz8cnxa1LHIiK6ayw1RG5M6SFH6tND8Ut1KBotIp7fVoDdR65IHYuI6K6w1BC5OU+5DH+fGo0pw8JgEYGUnUexLbdI6lhERG3GUkNEkMsEvP7kYMzW9oAoAi/vPo713xZKHYuIqE1YaogIACCTCXj1lwOx8KFeAIA/fXQK/9p3QeJUREStx1JDRFaCIGDp2AFYMqYvAOCNz87izS/OQhRFiZMREd0ZSw0RNSEIAn79SD/87rEBAIBVX13Aa5+cZrEhIofHUkNELVo0sjf+OC4SALD2m0K88v5JWCwsNkTkuFhqiOi25jzQE68/OQiCAGzOuYzf/vsYzCw2ROSgWGqI6GdNi+2Ov0+Jhlwm4N38K1iy4wgazBapYxERNcNSQ0R3NGFIN6ROHwJPuYCPjl3Ds1sLYGo0Sx2LiKgJlhoiapWxg7oifVYMFB4yfHmqDMmb8nGrnsWGiByHQ5aaiRMnomPHjpg0aVKzZRERERg8eDCio6MxatQoCdIRua/RA4KxYc5weHvKceDcdSRl5MFoapQ6FhERAActNUuWLMGmTZtuu/zgwYPQ6XT4+uuv7ZiKiADggT5B2DQvFh2UHsj5vhKz3smF/laD1LGIiByz1IwcORJ+fn5SxyCi2xgeEYit8zVQeXuioKgKM9bl4KaxXupYROTm2lxqDhw4gHHjxiE0NBSCIGDPnj3N1klLS0NERAS8vLyg0WiQl5dni6wA/vPDYA8//DCGDx+OrVu32my7RNQ26vAAbE+OQydfBU5cNWDamhyUV9dJHYuI3FibS43RaIRarUZaWlqLyzMzM5GSkoLly5ejoKAAarUaCQkJKC8vt64THR2NqKioZlNJSckd9//tt98iPz8fH3zwAV577TUcO3asxfVMJhMMBkOTiYhsKzLUH5kL49DFT4mzZdWYlp6Da/pbUsciIjcliPfw2+eCIGD37t2YMGGCdZ5Go8Hw4cORmpoKALBYLAgPD8fixYuxdOnSVm973759SE1NxbvvvnvbdV566SUMHDgQc+bMabbsj3/8I1599dVm8/V6Pfz9/Vudg4ju7FKFETPW5eJq1S2EB3pj2/w4hAf6SB2LiFyAwWCASqVq1ee3Ta+pqa+vR35+PuLj43/cgUyG+Ph4ZGdn3/P2jUYjqqurAQA1NTX46quvMHDgwBbXXbZsGfR6vXUqLi6+5/0TUcsignyx8xktenTyQXHlLUxJz8b312ukjkVEbsampaaiogJmsxnBwcFN5gcHB6O0tLTV24mPj8fkyZPxySefICwszFqIysrK8OCDD0KtViMuLg6zZ8/G8OHDW9yGUqmEv79/k4mI2k+3AG/sXKhFny4dcE1fhynpOThbWi11LCJyIx5SB2jJ3r17W5zfq1cvHD161M5piKi1gv29kLkgDjPfycPpawZMW5ONzfM0iOqmkjoaEbkBmx6pCQoKglwuR1lZWZP5ZWVlCAkJseWuiMhBdeqgxI7kOKjDA3CztgHT1+Yg//JNqWMRkRuwaalRKBSIiYlBVlaWdZ7FYkFWVha0Wq0td0VEDkzl44kt82IRGxGI6rpGzHonF9kXb0gdi4hcXJtLTU1NDXQ6HXQ6HQCgsLAQOp0ORUVFAICUlBSsXbsWGzduxOnTp7Fo0SIYjUYkJSXZNDgROTY/L09kzB2OB/sEobbejDkb8rD/3HWpYxGRC2vzV7r37dvX4j2XEhMTkZGRAQBITU3FypUrUVpaiujoaKxatQoajcYmge9WW74SRkS2U9dgxrNbC/DVmXIo5DKkPj0Ejw7k6Wgiap22fH7f0+/UOBOWGiLp1Dda8ELmEXxyvBQeMgF/nxqNcepQqWMRkROQ7HdqiIhaovCQYdW0IZg4pBsaLSKW7DiCd/OvSB2LiFwMSw0R2YWHXIa/TVZjemw4LCLw4q6j2JxzWepYRORCWGqIyG5kMgGvTRyEOfdHAAD+sOcE1n3zvbShiMhlsNQQkV0JgoDl4yKxaGRvAMBfPj6Nf2adlzgVEbkClhoisjtBEPDbhP74zSP9AAB/+/IcVn5+Bm7yvQUiaicsNUQkCUEQsHhMX/zP4/cBANK+vog/fXSKxYaI7hpLDRFJKvmhXvjz+IEAgA3fXcLLu0/AYmGxIaK2Y6khIsnN0kbgjUmDIROA7XlFeHHXUTSaLVLHIiInw1JDRA5hyrBwvDVtCOQyAe8duYolO3Sob2SxIaLWY6khIofxS3Uo/jVjKBRyGT4+fg3Pbs1HXYNZ6lhE5CRYaojIoSQMDMGa2TFQesiw93Q5kjcdxq16FhsiujOWGiJyOCP7d8GGpOHwUcjxzfkKJK7PQ42pUepYROTgWGqIyCHd3zsIm+fFwk/pgbxLlZi5Lhf62gapYxGRA2OpISKHFdMjENuS4xDg4wldcRWmr83BjRqT1LGIyEGx1BCRQxsUpsKOBXEI6qDEqWsGTFuTg3JDndSxiMgBsdQQkcMbEOKPzIVxCPH3wvnyGkxJz8bVqltSxyIiB8NSQ0ROoXfnDti5UIuwjt64dKMWU1Zn4/INo9SxiMiBsNQQkdPo3skHOxdq0SvIF1erbmFKejYulNdIHYuIHARLDRE5ldAAb+xYGId+wR1QZjBhano2Tl8zSB2LiBwASw0ROZ0ufl7YsUCLqG7+uGGsx7Q1OTh2pUrqWEQkMZYaInJKgb4KbJ0fhyHdA6C/1YAZa3Nx+FKl1LGISEIsNUTktFTentg8TwNNz0BUmxox6508HLxQIXUsIpIISw0RObUOSg9kJMViRN8g3GowY07GIXx9plzqWEQkAZYaInJ63go51iUOwyORwahvtGDB5sP47MQ1qWMRkZ2x1BCRS1B6yPGvGUPxxOCuaDCLeG7bEbyvuyp1LCKyI5YaInIZnnIZVk0bgqeGhsFsEfFCpg47DxVLHYuI7ISlhohcilwmYOWkwZgZ1x2iCPz238ew8eAlqWMRkR2w1BCRy5HJBPx5fBTmP9gTALD8g5NI339R4lRE1N4cstRMnDgRHTt2xKRJk5ot++tf/4qBAwciKioKW7ZskSAdETkDQRDwP0/ch8Wj+wAAVnx6Bm/tPQdRFCVORkTtxSFLzZIlS7Bp06Zm848fP45t27YhPz8fhw4dQmpqKqqqquwfkIicgiAI+M2j/fFSQn8AwFt7z+P1z86w2BC5KIcsNSNHjoSfn1+z+adPn4ZWq4WXlxe8vb2hVqvx2WefSZCQiJzJc6P64A+/iAQApO//Hn/84CQsFhYbIlfT5lJz4MABjBs3DqGhoRAEAXv27Gm2TlpaGiIiIuDl5QWNRoO8vDxbZEVUVBT27duHqqoq3Lx5E/v27cPVq/zKJhHd2bwHe+K1iYMgCMDG7MtY9t5xmFlsiFyKR1sfYDQaoVarMXfuXDz55JPNlmdmZiIlJQWrV6+GRqPBW2+9hYSEBJw9exZdunQBAERHR6OxsbHZY7/44guEhobedt+RkZH41a9+hdGjR0OlUiEuLg5yubytT4GI3NTTmu7w8pThxV1HkXm4GHWNZvxtshoecoc8aE1EbdTmUjN27FiMHTv2tsvffPNNJCcnIykpCQCwevVqfPzxx1i/fj2WLl0KANDpdHeXFsDChQuxcOFCAMD8+fPRt2/fFtczmUwwmUzWvw0Gw13vk4hcx5NDw6D0kGPJjiN4X1cCU4MFq6YPgcKDxYbI2dn0XVxfX4/8/HzEx8f/uAOZDPHx8cjOzrbJPsrL/3NPl7NnzyIvLw8JCQktrrdixQqoVCrrFB4ebpP9E5Hze2JwV6yeGQOFXIbPTpZi4ebDqGswSx2LiO6RTUtNRUUFzGYzgoODm8wPDg5GaWlpq7cTHx+PyZMn45NPPkFYWFiTQjR+/HhERkZi5syZ2LBhAzw8Wj7YtGzZMuj1eutUXMxfFSWiH8VHBuOdOcPg5SnD12evY27GIdTWNz8tTkTOo82nn+xh7969t13W2iM+SqUSSqXSVpGIyAWN6NsZG5NiMTfjEA5evIHZ7+RhfdJw+Ht5Sh2NiO6CTY/UBAUFQS6Xo6ysrMn8srIyhISE2HJXREQ2oenVCVvma+Dv5YHDl29i5rpcVNXWSx2LiO6CTUuNQqFATEwMsrKyrPMsFguysrKg1WptuSsiIpsZ0r0jti+IQ6CvAseu6DFtTQ4qakx3fiAROZQ2l5qamhrodDrrN5gKCwuh0+lQVFQEAEhJScHatWuxceNGnD59GosWLYLRaLR+G4qIyBENDFVhx4I4dPZT4kxpNaamZ6NUXyd1LCJqA0Fs4++F79u3D6NGjWo2PzExERkZGQCA1NRUrFy5EqWlpYiOjsaqVaug0WhsEvhuGQwGqFQq6PV6+Pv7S5qFiBxXYYURM9bmoERfh+6BPtiWrEFYRx+pYxG5rbZ8fre51Dgrlhoiaq3iylrMWJeLospahKq8sDU5Dj2DfKWOReSW2vL5zV+bIiL6ifBAH+xcqEXvzr4o0ddhSno2zpdVSx2LiO6ApYaIqAUhKi9kLtRiQIgfrlebMHVNDk6W6KWORUQ/g6WGiOg2gjoosWNBHAaHqVBprMf0NTk4UnRT6lhEdBssNUREPyPAR4Et8zUY1qMjDHWNmLkuF7nf35A6FhG1gKWGiOgO/L08sXFuLO7v3QnGejMSN+Thm/PXpY5FRD/BUkNE1Aq+Sg+snzMcI/t3Rl2DBfMyDmPvqbI7P5CI7Ialhoiolbw85UifFYOEgcGoN1vwzJZ8fHzsmtSxiOj/sNQQEbWB0kOOtKeHYnx0KBotIhZvL8B7BVekjkVEYKkhImozD7kMb06JxtRh4bCIwG92HcW23CKpYxG5PZYaIqK7IJcJWPHkICRqe0AUgZd3H8f6bwuljkXk1lhqiIjukkwm4I+/HIiFD/UCAPzpo1NI+/qCxKmI3BdLDRHRPRAEAUvHDsAL8X0BACs/P4u/fXEWbnJbPSKHwlJDRHSPBEHAC/H9sHTsAADAP7+6gP/38WkWGyI7Y6khIrKRZx7ujVd/ORAAsO7bQvzh/ROwWFhsiOyFpYaIyIYS74/A/z41CIIAbMkpwm//fQxmFhsiu2CpISKysanDu+OtqdGQywS8m38FS3YcQYPZInUsIpfHUkNE1A7GR3dD2tND4CkX8NGxa3h2awFMjWapYxG5NJYaIqJ28lhUV6yZNQwKDxm+PFWG5E35uFXPYkPUXlhqiIja0agBXbBhznB4e8px4Nx1JGXkocbUKHUsIpfEUkNE1M4e6BOEzfNi0UHpgZzvKzH7nVzobzVIHYvI5bDUEBHZwbCIQGydr4HK2xMFRVWYsS4HlcZ6qWMRuRSWGiIiO1GHB2DHgjh08lXgxFUDpq/JQXl1ndSxiFwGSw0RkR3d19UfmQu1CPZX4mxZNaal5+Ca/pbUsYhcAksNEZGd9enSATsXatEtwBvfVxgxJT0bxZW1UscicnosNUREEujRyRc7n9EiopMPiitvYfLqbHx/vUbqWEROjaWGiEgi3QK8sXOhFn27dECpoQ5T0nNwtrRa6lhEToulhohIQl38vbBjQRwiu/qjosaEqWuyceKqXupYRE6JpYaISGKdOiixPTkO6vAAVNU2YPraHORfvil1LCKnw1JDROQAVD6e2DIvFrERgaiua8Ssd3KRffGG1LGInIrDlZri4mKMHDkSkZGRGDx4MHbt2mVdVlVVhWHDhiE6OhpRUVFYu3athEmJiGzLz8sTGXOHY0TfINTWmzFnQx72n7sudSwipyGIoihKHeK/Xbt2DWVlZYiOjkZpaSliYmJw7tw5+Pr6wmw2w2QywcfHB0ajEVFRUTh8+DA6dep0x+0aDAaoVCro9Xr4+/vb4ZkQEd2dugYznttagKwz5VDIZUh9eggeHRgidSwiSbTl89vhjtR07doV0dHRAICQkBAEBQWhsrISACCXy+Hj4wMAMJlMEEURDtbJiIjumZenHG/PjMETg7qi3mzBoq0F+PBoidSxiBxem0vNgQMHMG7cOISGhkIQBOzZs6fZOmlpaYiIiICXlxc0Gg3y8vLuKlx+fj7MZjPCw8Ot86qqqqBWqxEWFoaXXnoJQUFBd7VtIiJHpvCQ4R/TovHkkG4wW0Qs2XEEuw4XSx2LyKG1udQYjUao1WqkpaW1uDwzMxMpKSlYvnw5CgoKoFarkZCQgPLycus6P1wT89OppOTHf4lUVlZi9uzZWLNmTZPtBwQE4OjRoygsLMS2bdtQVlbW1qdAROQUPOQy/HWyGtNju8MiAi+9ewybcy5LHYvIYd3TNTWCIGD37t2YMGGCdZ5Go8Hw4cORmpoKALBYLAgPD8fixYuxdOnSVm3XZDLhkUceQXJyMmbNmnXb9Z599lmMHj0akyZNanEbJpPJ+rfBYEB4eDivqSEipyOKIv700Sls+O4SAOD3T9yH+SN6SRuKyE4ku6amvr4e+fn5iI+P/3EHMhni4+ORnZ3dqm2Ioog5c+Zg9OjRzQpNWVkZqqv/82uber0eBw4cQP/+/VvczooVK6BSqazTf5/CIiJyJoIg4JVfROLZkb0BAH/5+DT+mXVe4lREjsempaaiogJmsxnBwcFN5gcHB6O0tLRV2/juu++QmZmJPXv2IDo6GtHR0Th+/DgA4PLlyxgxYgTUajVGjBiBxYsXY9CgQS1uZ9myZdDr9dapuJjnoonIeQmCgN8+NgAvPtoPAPC3L8/hjc/O8MsSRP/FQ+oAP/Xggw/CYrG0uCw2NhY6na5V21EqlVAqlTZMRkQkvedH94WXpxx/+fg0/rXvIm41mPHKLyIhCILU0YgkZ9MjNUFBQZDL5c0u3i0rK0NICH9jgYjIFuaP6IU/T4gCAGz47hJe3n0CFguP2BDZtNQoFArExMQgKyvLOs9isSArKwtardaWuyIicmuz4npg5aTBkAnA9rwivLjrKBrNLR/lJnIXbT79VFNTgwsXLlj/LiwshE6nQ2BgILp3746UlBQkJiZi2LBhiI2NxVtvvQWj0YikpCSbBicicneTh4XDy1OOFzJ1eO/IVdQ1mvHW1CFQeDjc76oS2UWbS83hw4cxatQo698pKSkAgMTERGRkZGDq1Km4fv06XnnlFZSWliI6OhqfffZZs4uHiYjo3o1Th0LpIcPz247gk+OlMDXkI23GUHh5yqWORmR3Dnfvp/bCez8RkSvbf+46Fmw6DFOjBQ/2CcKa2THwUTjcd0GI2syp7/1ERERt93C/zshIioWPQo5vL1RgzvpDqK5rkDoWkV2x1BARuQht707YPE8DPy8P5F2qxMx38qCvZbEh98FSQ0TkQmJ6dMT25Dh09PHE0eIqTF+bgxs1pjs/kMgFsNQQEbmYqG4q7FigRVAHJU5dM2DamhyUG+qkjkXU7lhqiIhcUP8QP2QujEOIvxfOl9dgSno2rlbdkjoWUbtiqSEiclG9O3fArme0COvojUs3ajFldTYu3zBKHYuo3bDUEBG5sPBAH+x6RoteQb64WnULU9KzcaG8RupYRO2CpYaIyMV1VXkjc6EW/YP9UGYwYWp6Nk6VGKSORWRzLDVERG6gs58S2xfEIaqbP24Y6zF9bQ6OFldJHYvIplhqiIjcRKCvAlvnx2Fo9wDobzVgxrpcHLpUKXUsIpthqSEiciMqb09snqdBXK9A1JgaMfudPBy8UCF1LCKbYKkhInIzvkoPbJgTi4f6dcatBjPmZBzC12fKpY5FdM9YaoiI3JC3Qo61s2PwSGQw6hstWLD5MD47cU3qWET3hKWGiMhNKT3k+NeMofjF4K5oMIt4btsRvK+7KnUsorvGUkNE5MY85TL8Y9oQTIoJg9ki4oVMHTIPFUkdi+iusNQQEbk5uUzAG08Nxsy47hBF4Hf/Po6NBy9JHYuozVhqiIgIMpmAP4+PQvKIngCA5R+cxOr9FyVORdQ2LDVERAQAEAQBLz9+H341ug8A4PVPz+DvX56DKIoSJyNqHZYaIiKyEgQBKY/2x0sJ/QEA/8g6j9c/PcNiQ06BpYaIiJp5blQfvPKLSABA+oHvsfyDk7BYWGzIsbHUEBFRi+Y+2BOvTRwEQQA2ZV/GsveOw8xiQw6MpYaIiG7raU13vDlFDZkAZB4uRspOHRrMFqljEbWIpYaIiH7WxCFhSH16KDxkAt7XleD5bQWob2SxIcfDUkNERHf0+KCuSJ8VA4Vchs9PlmH2+lzcNNZLHYuoCZYaIiJqlTH3BWP9nOHooPRAzveVmPCv73BNf0vqWERWLDVERNRqD/YNwnvP3o+wjt64fKMWM9fl4kaNSepYRABYaoiIqI36Bfthx4I4dFV54eJ1I2avz4OhrkHqWEQsNURE1HZhHX2wZb4GnXwVOFliwJz1eajgERuSmMOVmuLiYowcORKRkZEYPHgwdu3aZV129uxZREdHWydvb2/s2bNHurBERG6sd+cO2DxPA38vDxQUVWHsP75BQdFNqWORGxNEB/vt62vXrqGsrAzR0dEoLS1FTEwMzp07B19f3ybr1dTUICIiApcvX262rCUGgwEqlQp6vR7+/v7tFZ+IyO2cL6vGs1sLcL68BipvT7z7jBZ9g/2kjkUuoi2f3w53pKZr166Ijo4GAISEhCAoKAiVlZXN1vvggw8wZsyYVhUaIiJqP32D/fD+8w9gSPcA6G81IHF9Hi5VGKWORW6ozaXmwIEDGDduHEJDQyEIQounf9LS0hAREQEvLy9oNBrk5eXdVbj8/HyYzWaEh4c3W7Zz505MnTr1rrZLRES25aPwwDuJw9Grsy9K9HWYnJ6NM6UGqWORm2lzqTEajVCr1UhLS2txeWZmJlJSUrB8+XIUFBRArVYjISEB5eXl1nWio6MRFRXVbCopKbGuU1lZidmzZ2PNmjXN9mEwGHDw4EE8/vjjbY1PRETtJNBXgR0L4jAgxA/Xq02Ymp6DI7zGhuzonq6pEQQBu3fvxoQJE6zzNBoNhg8fjtTUVACAxWJBeHg4Fi9ejKVLl7ZquyaTCY888giSk5Mxa9asZss3b96Mzz//HFu2bPnZbZhMP16JbzAYEB4ezmtqiIjamb62AUkZeSgoqoKPQo4dC+IwOCxA6ljkpCS7pqa+vh75+fmIj4//cQcyGeLj45Gdnd2qbYiiiDlz5mD06NEtFhqgdaeeVqxYAZVKZZ1aOoVFRES2p/LxxJb5GtzfuxNq682Ym3EIl2/wGhtqfzYtNRUVFTCbzQgODm4yPzg4GKWlpa3axnfffYfMzEzs2bPH+tXt48ePW5fr9Xrk5eUhISHhZ7ezbNky6PV661RcXNz2J0RERHfFR+GB9FkxiOzqj4qaeoxP+w5fnSmTOha5OA+pA/zUgw8+CIvl9nd/ValUKCu78xtDqVRCqVTaMhoREbWBn5cnMpKGY/6mwzh2RY95Gw/jT78ciFnaCKmjkYuy6ZGaoKAgyOXyZqWjrKwMISEhttwVERE5gS7+Xtj1jBbTY7tDFIE/vH8SGw9ekjoWuSiblhqFQoGYmBhkZWVZ51ksFmRlZUGr1dpyV0RE5CSUHnK8NjEKz43qDQD4y8encKqEX/cm22tzqampqYFOp4NOpwMAFBYWQqfToaioCACQkpKCtWvXYuPGjTh9+jQWLVoEo9GIpKQkmwYnIiLnIQgCXny0Px6NDEaDWcTz2wvw6fFraDDf/nIDorZq81e69+3bh1GjRjWbn5iYiIyMDABAamoqVq5cidLSUkRHR2PVqlXQaDQ2CXy3eJsEIiLp3agxIeGtb6w3v4y/rwvWzBoGmUyQOBk5qrZ8fjvcvZ/aC0sNEZFjKK6sxZbcy8j47hJMjRb8/on7MH9EL6ljkYNy6ns/ERGRawsP9MGysffhD7+IBAC8/ukZvPnlOdQ1mCVORs6OpYaIiCQxQ9MdTw0NQ6NFxKqs83jyXwdRXFkrdSxyYiw1REQkCUEQ8NfJg/GvGUPRyVeBU9cM+MU/v8U73xbC1MijNtR2LDVERCQZQRDw+KCu+HDxgxgcpoL+VgP+/NEpPP6Pb5B/mTfDpLZhqSEiIsmFBnjjvUX34/UnByGogxIXrxsxefVBrP+2EG7yfRayAX77iYiIHEpVbT3++MFJ7NGVAAD8lB5Q+XgiYWAIenX2RRc/L4wZ0IVfA3cT/Ep3C1hqiIichyiKeOfbQqz49AzMluYfUyP6BuG1iYMQHugjQTqyJ5aaFrDUEBE5n6raetysbcCF8hp8dqIUhroGfHP+OuoaLBAEYGCoP7w95db1BUHAU0O7Yerw7hKmJltqy+e3w92lm4iI6AcBPgoE+CjQM8gXj0QGAwAulFfjlfdP4uDFGzhxtfk9pPIKK1HXYLGuDwCCAAT7efGUlYvjkRoiInJKV27W/l+p+fFjLOf7SmTc5i7gD/TphC3zNBAEFhtnwiM1RETk8sI6+iCsY9NrahIGhkDpKcPm7Mto/K9rceobLfjuwg28ryvBhCHd7B2V7IRHaoiIyOWlfnUef/3iHLr4KTEpJuyethUdHoBHIoN5xMdOeKSGiIjov8wf0QuZh4tRXHkL/9p38Z6392CfIPQN7mCDZK6ls58Sz47sI9n+eaSGiIjcwulrBrxXcAVmy91vw2hqxO4jV1F/LxtxYX27dMCXKQ/bdJs8UkNERPQT93X1x/88EXnP20l+qBc+OlaCBhabZjr5KiXdP0sNERFRG/Tp0gEvxPeTOga1gPd+IiIiIpfAUkNEREQugaWGiIiIXAJLDREREbkElhoiIiJyCSw1RERE5BJYaoiIiMglsNQQERGRS2CpISIiIpfAUkNEREQugaWGiIiIXAJLDREREbkElhoiIiJyCW5zl25RFAEABoNB4iRERETUWj98bv/wOf5z3KbUVFdXAwDCw8MlTkJERERtVV1dDZVK9bPrCGJrqo8LsFgsKCkpgZ+fHwRBsOm2DQYDwsPDUVxcDH9/f5tu29VwrNqG49V6HKvW41i1Dcer9dpjrERRRHV1NUJDQyGT/fxVM25zpEYmkyEsLKxd9+Hv788XfCtxrNqG49V6HKvW41i1Dcer9Ww9Vnc6QvMDXihMRERELoGlhoiIiFwCS40NKJVKLF++HEqlUuooDo9j1TYcr9bjWLUex6ptOF6tJ/VYuc2FwkREROTaeKSGiIiIXAJLDREREbkElhoiIiJyCSw1RERE5BJYau5RWloaIiIi4OXlBY1Gg7y8PKkjOYQ//vGPEAShyTRgwADr8rq6Ojz33HPo1KkTOnTogKeeegplZWUSJrafAwcOYNy4cQgNDYUgCNizZ0+T5aIo4pVXXkHXrl3h7e2N+Ph4nD9/vsk6lZWVmDFjBvz9/REQEIB58+ahpqbGjs/CPu40VnPmzGn2OnvsscearOMuY7VixQoMHz4cfn5+6NKlCyZMmICzZ882Wac177uioiI88cQT8PHxQZcuXfDSSy+hsbHRnk/FLlozXiNHjmz2+nrmmWearOMO4/X2229j8ODB1h/U02q1+PTTT63LHel1xVJzDzIzM5GSkoLly5ejoKAAarUaCQkJKC8vlzqaQxg4cCCuXbtmnb799lvrsl//+tf48MMPsWvXLuzfvx8lJSV48sknJUxrP0ajEWq1GmlpaS0uf+ONN7Bq1SqsXr0aubm58PX1RUJCAurq6qzrzJgxAydPnsSXX36Jjz76CAcOHMCCBQvs9RTs5k5jBQCPPfZYk9fZ9u3bmyx3l7Hav38/nnvuOeTk5ODLL79EQ0MDHn30URiNRus6d3rfmc1mPPHEE6ivr8fBgwexceNGZGRk4JVXXpHiKbWr1owXACQnJzd5fb3xxhvWZe4yXmFhYXj99deRn5+Pw4cPY/To0Rg/fjxOnjwJwMFeVyLdtdjYWPG5556z/m02m8XQ0FBxxYoVEqZyDMuXLxfVanWLy6qqqkRPT09x165d1nmnT58WAYjZ2dl2SugYAIi7d++2/m2xWMSQkBBx5cqV1nlVVVWiUqkUt2/fLoqiKJ46dUoEIB46dMi6zqeffioKgiBevXrVbtnt7adjJYqimJiYKI4fP/62j3HXsRJFUSwvLxcBiPv37xdFsXXvu08++USUyWRiaWmpdZ23335b9Pf3F00mk32fgJ39dLxEURQffvhhccmSJbd9jDuPV8eOHcV169Y53OuKR2ruUn19PfLz8xEfH2+dJ5PJEB8fj+zsbAmTOY7z588jNDQUvXr1wowZM1BUVAQAyM/PR0NDQ5OxGzBgALp37+72Y1dYWIjS0tImY6NSqaDRaKxjk52djYCAAAwbNsy6Tnx8PGQyGXJzc+2eWWr79u1Dly5d0L9/fyxatAg3btywLnPnsdLr9QCAwMBAAK1732VnZ2PQoEEIDg62rpOQkACDwWD9V7mr+ul4/WDr1q0ICgpCVFQUli1bhtraWusydxwvs9mMHTt2wGg0QqvVOtzrym1uaGlrFRUVMJvNTf5PAoDg4GCcOXNGolSOQ6PRICMjA/3798e1a9fw6quvYsSIEThx4gRKS0uhUCgQEBDQ5DHBwcEoLS2VJrCD+OH5t/S6+mFZaWkpunTp0mS5h4cHAgMD3W78HnvsMTz55JPo2bMnLl68iJdffhljx45FdnY25HK5246VxWLBCy+8gAceeABRUVEA0Kr3XWlpaYuvvR+WuaqWxgsAnn76afTo0QOhoaE4duwYfve73+Hs2bN47733ALjXeB0/fhxarRZ1dXXo0KEDdu/ejcjISOh0Ood6XbHUULsYO3as9X8PHjwYGo0GPXr0wM6dO+Ht7S1hMnIl06ZNs/7vQYMGYfDgwejduzf27duHMWPGSJhMWs899xxOnDjR5Do2ur3bjdd/X3s1aNAgdO3aFWPGjMHFixfRu3dve8eUVP/+/aHT6aDX6/Huu+8iMTER+/fvlzpWMzz9dJeCgoIgl8ubXeFdVlaGkJAQiVI5roCAAPTr1w8XLlxASEgI6uvrUVVV1WQdjh2sz//nXlchISHNLkZvbGxEZWWl249fr169EBQUhAsXLgBwz7F6/vnn8dFHH+Hrr79GWFiYdX5r3nchISEtvvZ+WOaKbjdeLdFoNADQ5PXlLuOlUCjQp08fxMTEYMWKFVCr1fjHP/7hcK8rlpq7pFAoEBMTg6ysLOs8i8WCrKwsaLVaCZM5ppqaGly8eBFdu3ZFTEwMPD09m4zd2bNnUVRU5PZj17NnT4SEhDQZG4PBgNzcXOvYaLVaVFVVIT8/37rOV199BYvFYv2Prru6cuUKbty4ga5duwJwr7ESRRHPP/88du/eja+++go9e/Zssrw17zutVovjx483KYJffvkl/P39ERkZaZ8nYid3Gq+W6HQ6AGjy+nKX8fopi8UCk8nkeK8rm1527GZ27NghKpVKMSMjQzx16pS4YMECMSAgoMkV3u7qN7/5jbhv3z6xsLBQ/O6778T4+HgxKChILC8vF0VRFJ955hmxe/fu4ldffSUePnxY1Gq1olarlTi1fVRXV4tHjhwRjxw5IgIQ33zzTfHIkSPi5cuXRVEUxddff10MCAgQ33//ffHYsWPi+PHjxZ49e4q3bt2ybuOxxx4ThwwZIubm5orffvut2LdvX3H69OlSPaV283NjVV1dLb744otidna2WFhYKO7du1ccOnSo2LdvX7Gurs66DXcZq0WLFokqlUrct2+feO3aNetUW1trXedO77vGxkYxKipKfPTRR0WdTid+9tlnYufOncVly5ZJ8ZTa1Z3G68KFC+Kf/vQn8fDhw2JhYaH4/vvvi7169RIfeugh6zbcZbyWLl0q7t+/XywsLBSPHTsmLl26VBQEQfziiy9EUXSs1xVLzT365z//KXbv3l1UKBRibGysmJOTI3UkhzB16lSxa9euokKhELt16yZOnTpVvHDhgnX5rVu3xGeffVbs2LGj6OPjI06cOFG8du2ahInt5+uvvxYBNJsSExNFUfzP17r/8Ic/iMHBwaJSqRTHjBkjnj17tsk2bty4IU6fPl3s0KGD6O/vLyYlJYnV1dUSPJv29XNjVVtbKz766KNi586dRU9PT7FHjx5icnJys39UuMtYtTROAMQNGzZY12nN++7SpUvi2LFjRW9vbzEoKEj8zW9+IzY0NNj52bS/O41XUVGR+NBDD4mBgYGiUqkU+/TpI7700kuiXq9vsh13GK+5c+eKPXr0EBUKhdi5c2dxzJgx1kIjio71uhJEURRte+yHiIiIyP54TQ0RERG5BJYaIiIicgksNUREROQSWGqIiIjIJbDUEBERkUtgqSEiIiKXwFJDRERELoGlhoiIiFwCSw0RERG5BJYaIiIicgksNUREROQSWGqIiIjIJfx/PiFSdr7y13gAAAAASUVORK5CYII=",
 77 |       "text/plain": [
 78 |        "<Figure size 640x480 with 1 Axes>"
 79 |       ]
 80 |      },
 81 |      "metadata": {},
 82 |      "output_type": "display_data"
 83 |     },
 84 |     {
 85 |      "name": "stdout",
 86 |      "output_type": "stream",
 87 |      "text": [
 88 |       "[[16.999999999999996 12.999999999999998 18.999999999999996\n",
 89 |       "  22.999999999999996]]\n"
 90 |      ]
 91 |     }
 92 |    ],
 93 |    "source": [
 94 |     "import matplotlib.pyplot as plt\n",
 95 |     "from tqdm import tqdm\n",
 96 |     "x = vn(2)\n",
 97 |     "y = vn(3)\n",
 98 |     "z = vn(4)\n",
 99 |     "a = np.array([[x, y, z]])\n",
100 |     "W = vp.array(np.random.randn(4, 3))\n",
101 |     "Y = vp.array(np.array([17, 13, 19, 23]), requires_grad=False)\n",
102 |     "losses = []\n",
103 |     "for i in tqdm(range(300)):\n",
104 |     "    b = a @ W.T\n",
105 |     "    loss = np.mean((Y - b) ** 2)\n",
106 |     "    losses.append(loss)\n",
107 |     "    loss.backward()\n",
108 |     "    with grad.no_grad():\n",
109 |     "        W = W - 0.01 * vp.array_grad(W)\n",
110 |     "    if i != 299:\n",
111 |     "        loss.zero_grad()\n",
112 |     "loss.draw_graph().view()\n",
113 |     "plt.plot(losses)\n",
114 |     "plt.yscale('log')\n",
115 |     "plt.show()\n",
116 |     "print(a @ W.T)\n"
117 |    ]
118 |   },
119 |   {
120 |    "attachments": {},
121 |    "cell_type": "markdown",
122 |    "metadata": {},
123 |    "source": [
124 |     "## XOR Neural Net"
125 |    ]
126 |   },
127 |   {
128 |    "cell_type": "code",
129 |    "execution_count": 4,
130 |    "metadata": {},
131 |    "outputs": [
132 |     {
133 |      "name": "stderr",
134 |      "output_type": "stream",
135 |      "text": [
136 |       "100%|██████████| 500/500 [02:13<00:00,  3.74it/s]\n"
137 |      ]
138 |     },
139 |     {
140 |      "name": "stdout",
141 |      "output_type": "stream",
142 |      "text": [
143 |       "[[-0.9984565423358374]\n",
144 |       " [0.9974001865275044]\n",
145 |       " [0.9977095500300808]\n",
146 |       " [-0.9985410651493936]]\n"
147 |      ]
148 |     },
149 |     {
150 |      "data": {
151 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGdCAYAAADaPpOnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBmElEQVR4nO3de3xT9eE+8OckaZLekrb03qaUO5RLC6UtRRGRKqLD4W14B6Y4sTJZnRtsX2F+f98NNy9DR6YbXtB5AXWCmwiC5a5c2kK5WC4FCi29pDeatGmbtMn5/dESrIC0NOnJ5Xm/XnlBTk5znhydfXby+ZyPIIqiCCIiIiIPIZM6ABEREVFPsLwQERGRR2F5ISIiIo/C8kJEREQeheWFiIiIPArLCxEREXkUlhciIiLyKCwvRERE5FEUUgdwNrvdjoqKCgQHB0MQBKnjEBERUTeIoojGxkbExsZCJvvxayteV14qKiqg0+mkjkFERETXoKysDPHx8T+6j9eVl+DgYAAdH16j0UichoiIiLrDZDJBp9M5fo//GK8rLxe+KtJoNCwvREREHqY7Qz44YJeIiIg8CssLEREReRSWFyIiIvIoLC9ERETkUVheiIiIyKOwvBAREZFHYXkhIiIij8LyQkRERB7FLcvLF198gWHDhmHIkCF48803pY5DREREbsTt7rDb3t6OnJwcbN26FVqtFqmpqbjzzjvRr18/qaMRERGRG3C7Ky/79u3DyJEjERcXh6CgIEyfPh2bNm2SOhYRERG5CaeXlx07dmDGjBmIjY2FIAhYt27dJfvo9XokJiZCrVYjIyMD+/btc7xWUVGBuLg4x/O4uDiUl5c7OyYRERF5KKeXF7PZjOTkZOj1+su+vmbNGuTk5GDp0qXYv38/kpOTMW3aNFRXVzs7ilM1trZh3nv5WL2vFAVnz+N0TRPOm61os9mljkZERORTnD7mZfr06Zg+ffoVX3/llVcwb948zJ07FwDwxhtvYP369Xj77bexaNEixMbGdrnSUl5ejvT09Cu+n8VigcVicTw3mUxO+BSX2na8BpuLDNhcZLjkNbWfDEEqP2jUCgSpFQhWKxCkUiBY7QeN2g+RGhWiNWpEadSI1qoRG6KGSiF3SU4iIiJv16cDdq1WKwoKCrB48WLHNplMhqysLOzevRsAkJ6ejiNHjqC8vBxarRYbNmzAc889d8X3XLZsGZ5//nmXZ0/RheCZm4di+4kaGBpb0WBuQ6OlHQDQ2mZHa5sFtU2Wq7xLB7lMwIDwQAyLDsbwqGCMTwzDuP4hLDRERETd0Kflpba2FjabDVFRUV22R0VF4dixYx2BFAq8/PLLmDJlCux2O37zm9/86EyjxYsXIycnx/HcZDJBp9M5PbsuLAALpg7BgqlDHNvabHaYLe1obL3waEPTheeWjufGljZUmyyoMrbCYGpFlakVzVYbTlY34WR1E9ajEkDH1ZuMAf1w59g4TBsZDX8liwwREdHluN1UaQC44447cMcdd3RrX5VKBZVK5eJEl+cnlyEkQImQAGW3f0YURRhMFhyrMuGEoRFHyk349lQdapss2H6iBttP1CBIpcCdY+PwxI2DEBfi78JPQERE5Hn6tLyEh4dDLpfDYOg6bsRgMCA6OrpX763X66HX62Gz2Xr1Pq4mCAKitR1jX24cFgmgo9CcMDRh45EqfLq/DGX1LfjXnrNYnVeKBzP6I+eWodCo/SROTkRE5B769D4vSqUSqampyM3NdWyz2+3Izc1FZmZmr947OzsbRUVFyMvL623MPicIAoZFB+PprCHY/usp+PCxDEwc1A9tNhGrvj2DqS9vx9eXGShMRETki5x+5aWpqQknT550PC8pKUFhYSHCwsKQkJCAnJwczJ49G+PHj0d6ejqWL18Os9nsmH3k62QyARMHh2Pi4HDsLK7Bks+/Q0mtGY+9l4/Hrh+A304fDj+5291bkIiIqM8IoiiKznzDbdu2YcqUKZdsnz17NlatWgUAWLFiBV588UVUVVUhJSUFr732GjIyMnp13O9/bXTixAkYjUZoNJpevac7aG2z4cWvjuOtXSUAgBuGRuD1B8chUOWWw5WIiIiuiclkglar7dbvb6eXF6n15MN7ko1HqvCrNYVoabNhdJwW/3o0vUcDhYmIiNxZT35/8/sHD3HrqGh89PgEhAUqcbjciNlv70Nja5vUsYiIiPqc15QXvV6PpKQkpKWlSR3FZVJ0IVjTWWAOnjPi0VX5sLS79+wqIiIiZ+PXRh7oSLkR96/cg8bWdtw1Lg4v35sMQRCkjkVERHTN+LWRlxsVp8XfHxwHuUzAZ/vLsXLnaakjERER9RmWFw81aUgElvwkCQDw543HkX+mXuJEREREfYPlxYM9ktkfM1NiYbOLeHp1IRqarVJHIiIicjmvKS++MGD3hwRBwP/dORqJ/QJQ3tCC/1l3ROpIRERELscBu17g0LkG3Pn3b2Gzi/jnw6m4ZWTv1okiIiLqaxyw62PGxIdg3qSBAIDnPj8CYwvv/0JERN6L5cVLLMwaggHhgTCYLHhhw1Gp4xAREbkMy4uXUPvJ8cJdowEAH+0rw57TdRInIiIicg2vKS++OGD3hzIG9sMDGQkAgOf/WwSb3auGMxEREQHggF2vc95sxY0vbYOxpQ1/unO0o8wQERG5Mw7Y9WGhgUoszBoCAHhp03EO3iUiIq/D8uKFHprQH4Mjg1BvtuJvucVSxyEiInIqlhcv5CeX4bnOpQNWfXsGZ+vMEiciIiJyHpYXLzV5aARuGBqBdruIv24+IXUcIiIip/Ga8sLZRpf6zbRhAIDPD1bgWJVJ4jRERETO4TXlJTs7G0VFRcjLy5M6itsYFafF7aNjIIrAS1/x6gsREXkHrykvdHk5twyFXCbg66MGFJw9L3UcIiKiXmN58XKDIoJwz7h4AMDLm45LnIaIiKj3WF58wC+zhsBPLuDbU3UoOFsvdRwiIqJeYXnxAXEh/rgntePqy2u5JyVOQ0RE1DssLz5i/uTBkMsEbD9Rg4NlDVLHISIiumZeU144VfrHJfQLwE9TYgEAK7by6gsREXkuLszoQ07VNCHrle0QRWDD05MwIobnh4iI3AMXZqTLGhQRhNtHxwAAVmzh1RciIvJMLC8+5qmbBgMAvjxSiZPVjRKnISIi6jmWFx8zPFqDaSOjIIrAP3ecljoOERFRj7G8+KBfTB4EAFh3oALVplaJ0xAREfUMy4sPGpcQivH9Q2G12bHq2zNSxyEiIuoRlhcf9fgNAwEA7+85C7OlXeI0RERE3cfy4qOyRkRhQHggTK3t+Di/TOo4RERE3cby4qNkMgGPTRoAAHhrVwnabXaJExEREXUPy4sPu3tcPPoFKnHufAs2HKmSOg4REVG3eE154fIAPaf2k+ORzEQAHdOmvexmy0RE5KW8prxkZ2ejqKgIeXl5UkfxKA9n9odKIcPhciP2ltRLHYeIiOiqvKa80LUJC1Ti3vHxAHjTOiIi8gwsL4THrh8IQQC2HKtGSa1Z6jhEREQ/iuWFkBgeiKnDIwEA7/KmdURE5OZYXggAMGdix7TpT/LL0NjaJnEaIiKiK2N5IQDAdYP7YXBkEMxWGz4tOCd1HCIioitieSEAgCAImDMxEUDHV0d2O6dNExGRe2J5IYe7xsUhWK3AmbpmbD9RI3UcIiKiy2J5IYcApQL3pekAAO9w4C4REbkplhfq4pHMRAgCsONEDU5WN0kdh4iI6BIsL9SFLiwAWSOiAHDaNBERuSeWF7rE3M6Bu//efw7GFk6bJiIi9+KW5eXOO+9EaGgo7rnnHqmj+KTMQf0wLCoYzVYbPskvkzoOERFRF25ZXp5++mm89957UsfwWYIgYM51iQCA93afhY3TpomIyI24ZXm58cYbERwcLHUMnzYzJQ5afz+U1jdj67FqqeMQERE59Li87NixAzNmzEBsbCwEQcC6desu2Uev1yMxMRFqtRoZGRnYt2+fM7JSH/JXyh3Tpt/bc1biNERERBf1uLyYzWYkJydDr9df9vU1a9YgJycHS5cuxf79+5GcnIxp06ahuvri/3tPSUnBqFGjLnlUVFRc+ychp3sgI8ExbfoMV5smIiI3oejpD0yfPh3Tp0+/4uuvvPIK5s2bh7lz5wIA3njjDaxfvx5vv/02Fi1aBAAoLCy8trSXYbFYYLFYHM9NJpPT3tvX9e8XiMlDI7DteA0+2HsWv789SepIREREzh3zYrVaUVBQgKysrIsHkMmQlZWF3bt3O/NQDsuWLYNWq3U8dDqdS47jqx7J7A8A+Dj/HFrbbBKnISIicnJ5qa2thc1mQ1RUVJftUVFRqKqq6vb7ZGVl4d5778WXX36J+Pj4Hy0+ixcvhtFodDzKyji115kmD41EfKg/jC1t+M9Bfq1HRETS6/HXRn3h66+/7va+KpUKKpXKhWl8m1wm4MGM/vjzxmN4f89Z/Gw8r2wREZG0nHrlJTw8HHK5HAaDoct2g8GA6OhoZx7qEnq9HklJSUhLS3PpcXzRrDQdlAoZDp0z4mBZg9RxiIjIxzm1vCiVSqSmpiI3N9exzW63Izc3F5mZmc481CWys7NRVFSEvLw8lx7HF4UFKvGT0TEAOm5aR0REJKUel5empiYUFhY6ZgyVlJSgsLAQpaWlAICcnBysXLkS7777Lo4ePYr58+fDbDY7Zh+RZ3qoc+Dufw9V4LzZKnEaIiLyZT0e85Kfn48pU6Y4nufk5AAAZs+ejVWrVmHWrFmoqanBkiVLUFVVhZSUFGzcuPGSQbzOptfrodfrYbNxRowrjNWFYFScBkfKTfikoAyP3zBI6khEROSjBFEUvWrhGpPJBK1WC6PRCI1GI3Ucr7ImrxS//fdhJIQFYNuvb4RMJkgdiYiIvERPfn+75dpG5J7uSI6DRq1AaX0zthfXSB2HiIh8FMsLdZu/Uo57O6dKv8+Bu0REJBGvKS+cKt03HsxIAABsOV6NsvpmidMQEZEv8prywqnSfWNgRBAmDQmHKAIf7C2VOg4REfkgrykv1HcennBhvaMyrndERER9juWFeuym4ZGI1apRb7biy8OVUschIiIf4zXlhWNe+o5CLsMDnWNf/rWHA3eJiKhveU154ZiXvjUrLQF+cgEHShtwpNwodRwiIvIhXlNeqG9FBKswfVTHekfv8+oLERH1IZYXumYPd653tK6wHMbmNonTEBGRr2B5oWs2vn8ohkcHo7XNjk8KyqSOQ0REPsJrygsH7PY9QRAcV18+2FsKu92rlskiIiI35TXlhQN2pTEzJQ7BKgVKas3YdbJW6jhEROQDvKa8kDQCVQrcnRoPAHiP6x0REVEfYHmhXnuo8467W44ZcO481zsiIiLXYnmhXhscGYTrBveDXQQ+5HpHRETkYiwv5BQPT0gEAKzJK4OlnesdERGR63hNeeFsI2lljYhEjFaNOrMVGw5XSR2HiIi8mNeUF842kpZCLsMD6R3rHb23+4y0YYiIyKt5TXkh6c1K18FPLmA/1zsiIiIXYnkhp4kMVuPWzvWO/sVp00RE5CIsL+RUj3Tecffzg1zviIiIXIPlhZyK6x0REZGrsbyQU31/vaP395zlekdEROR0LC/kdBfWOzpT18z1joiIyOm8przwPi/ug+sdERGRKwmiKHrVdX2TyQStVguj0QiNRiN1HJ91qqYJU1/eDpkA7PjNFMSHBkgdiYiI3FhPfn97zZUXci+DIi6ud/QB1zsiIiInYnkhl+F6R0RE5AosL+QyF9Y7qjdb8eXhSqnjEBGRl2B5IZfput4RB+4SEZFzsLyQS92XngA/uYADXO+IiIichOWFXCoiWIXpXO+IiIiciOWFXO5hrndEREROxPJCLvf99Y7W5HPaNBER9Q7LC7mcIAiYMzERQMfAXRvXOyIiol7wmvLC5QHc28yxcQgJ8MO58y34+qhB6jhEROTBvKa8ZGdno6ioCHl5eVJHoctQ+8lxf+e06Xe+KZE4DREReTKvKS/k/h6e0B9ymYA9p+txtNIkdRwiIvJQLC/UZ2JD/HHryGgAwKpvzkgbhoiIPBbLC/WpudclAgDWFZaj3myVNgwREXkklhfqU6n9QzEqTgNLux2r8zhtmoiIeo7lhfqUIAiYO3EAgI477rbZ7BInIiIiT8PyQn3uJ8kxCA9SotLYik3fcdo0ERH1DMsL9TmVQo4HMjqWDOC0aSIi6imWF5LEQxkJUMgE5J89j8PnuNo0ERF1H8sLSSJSo8btYzpWm37nW159ISKi7mN5IcnMva5j4O4XBytR02iROA0REXkKlheSTIouBCm6EFhtdny4l9OmiYioe9yuvJSVleHGG29EUlISxowZg08++UTqSORCP7++c9r0njNobbNJnIaIiDyB25UXhUKB5cuXo6ioCJs2bcLChQthNpuljkUuMn1UNGK1atQ2WfF5YbnUcYiIyAO4XXmJiYlBSkoKACA6Ohrh4eGor6+XNhS5jJ9c5hj78ubOEoiiKHEiIiJydz0uLzt27MCMGTMQGxsLQRCwbt26S/bR6/VITEyEWq1GRkYG9u3bd03hCgoKYLPZoNPprunnyTPMStchSKVAcXUTtp2okToOERG5uR6XF7PZjOTkZOj1+su+vmbNGuTk5GDp0qXYv38/kpOTMW3aNFRXVzv2SUlJwahRoy55VFRUOPapr6/HI488gn/+85/X8LHIk2jUfrgvraOgvrnztMRpiIjI3QliL67TC4KAtWvXYubMmY5tGRkZSEtLw4oVKwAAdrsdOp0OCxYswKJFi7r1vhaLBTfffDPmzZuHhx9++Kr7WiwXp9maTCbodDoYjUZoNJqefyiSRHlDC274y1bY7CLW//J6jIzVSh2JiIj6kMlkglar7dbvb6eOebFarSgoKEBWVtbFA8hkyMrKwu7du7v1HqIoYs6cObjpppuuWlwAYNmyZdBqtY4Hv2LyTHEh/rhtdMdN697ayZvWERHRlTm1vNTW1sJmsyEqKqrL9qioKFRVVXXrPb755husWbMG69atQ0pKClJSUnD48OEr7r948WIYjUbHo6ysrFefgaQzb1LHwN3/HKxAlbFV4jREROSuFFIH+KHrr78edru92/urVCqoVCro9Xro9XrYbLxXiKcaEx+C9AFh2FdSj1XfnsGi6cOljkRERG7IqVdewsPDIZfLYTAYumw3GAyIjo525qEukZ2djaKiIuTl5bn0OORa8yYNBAB8uPcszJZ2idMQEZE7cmp5USqVSE1NRW5urmOb3W5Hbm4uMjMznXko8lJTh0diYHggTK3t+DifXwESEdGlelxempqaUFhYiMLCQgBASUkJCgsLUVrasTZNTk4OVq5ciXfffRdHjx7F/PnzYTabMXfuXKcGJ+8kkwmOJQPe/qYENjtvWkdERF31eMxLfn4+pkyZ4niek5MDAJg9ezZWrVqFWbNmoaamBkuWLEFVVRVSUlKwcePGSwbxOhvHvHiPu8fF4+VNx1FW34KvvqtyzEIiIiICenmfF3fUk3ni5L5e2XQcr205iRRdCNY+ORGCIEgdiYiIXEiy+7wQOcvDmYlQKmQoLGvAvhKubUVERBd5TXnR6/VISkpCWlqa1FHICSKCVbg3NR4A8PdtpyROQ0RE7oRfG5HbKq1rxo0vbYVdBJcMICLycvzaiLxCQr8A/GRMLADgdV59ISKiTiwv5Nbm3zgIAPDl4UqcqTVLnIaIiNyB15QXjnnxTiNiNJgyLAJ2EfjHjtNSxyEiIjfAMS/k9vLO1OPeN3ZDKZdh12+nIFKjljoSERE5Gce8kFdJSwzD+P6hsNrseOubEqnjEBGRxFheyCM8OaVj7MsHe0phbGmTOA0REUmJ5YU8wpRhkRgeHYwmSzve33NW6jhERCQhrykvHLDr3QRBcMw8entXCVqsXMOKiMhXeU15yc7ORlFREfLy8qSOQi5y++gY6ML8UWe2YnVeqdRxiIhIIl5TXsj7KeQyPDG54+rLG9tPobWNV1+IiHwRywt5lHtS4xGjVcNgsuCTgnNSxyEiIgmwvJBHUSnkjrEvr289CWu7XeJERETU11heyOP8bLwOkcEqVBhb8SmvvhAR+RyvKS+cbeQ71H5yx9gX/daTaLPx6gsRkS/xmvLC2Ua+5YGMBIQHqVDe0IK1+8uljkNERH3Ia8oL+ZaOqy8DAQArtp5EO6++EBH5DJYX8lgPZCSgX6ASpfXN+LywQuo4RETUR1heyGMFKBWYdwOvvhAR+RqWF/JoD0/oj9AAP5TUmvHFoUqp4xARUR9geSGPFqhS4LFJHVdfXsst5tUXIiIfwPJCHu+RzI6rL6drzVh7gDOPiIi8ndeUF97nxXcFq/0c9315NbeYd90lIvJyXlNeeJ8X3/ZIZiIiglU4d74Fa/LLpI5DREQu5DXlhXybv1KOp6YMBgCs2FLMFaeJiLwYywt5jfvSdYgL8YfBZMH7e85KHYeIiFyE5YW8hkohxy+ndlx9+fu2U2iytEuciIiIXIHlhbzK3ePiMSA8EPVmK1Z9UyJ1HCIicgGWF/IqCrkMC7OGAAD+seM0jM1tEiciIiJnY3khrzNjTCyGRQWjsbUdK3eeljoOERE5GcsLeR2ZTEDOLUMBAG9/U4LaJovEiYiIyJlYXsgr3ZIUheR4LZqtNryWWyx1HCIiciKWF/JKgiDgt9OHAwA+3FuKklqzxImIiMhZvKa8cHkA+qGJg8IxZVgE2u0iXvzqmNRxiIjISQRRFEWpQziTyWSCVquF0WiERqOROg5J7HhVI6a/ugN2EfjsyYkYlxAqdSQiIrqMnvz+9porL0SXMyw6GPekxgMAln15FF7W1YmIfBLLC3m9X908FGo/GfLOnMfmIoPUcYiIqJdYXsjrxWj98ej1AwAAf954DO02u8SJiIioN1heyCf8YvIghAUqcarGjI/zz0kdh4iIeoHlhXyCRu2HBTd1LNr4169PwMxFG4mIPBbLC/mMBzP6IyEsADWNFi4bQETkwVheyGcoFTL85tZhAIA3tp9CRUOLxImIiOhasLyQT7l9dAzSEkPR2mbHnzfyxnVERJ6I5YV8iiAIWDpjJAQB+LywAgVn66WOREREPcTyQj5nVJwWP0vVAQCe/28R7HbeuI6IyJOwvJBP+vW0YQhSKXDonBH/3s+p00REnsTtyktDQwPGjx+PlJQUjBo1CitXrpQ6EnmhiGCVY+r0X746jiZOnSYi8hhuV16Cg4OxY8cOFBYWYu/evfjTn/6Euro6qWORF5pzXSIS+3VMndZvPSl1HCIi6ia3Ky9yuRwBAQEAAIvFAlEUuZgeuYRKIcfvb08CALy1swSldc0SJyIiou7ocXnZsWMHZsyYgdjYWAiCgHXr1l2yj16vR2JiItRqNTIyMrBv374eHaOhoQHJycmIj4/Hs88+i/Dw8J7GJOqWrBGRmDQkHFabHf+3vkjqOERE1A09Li9msxnJycnQ6/WXfX3NmjXIycnB0qVLsX//fiQnJ2PatGmorq527HNhPMsPHxUVFQCAkJAQHDx4ECUlJfjwww9hMHAlYHINQRDw3E+SIJcJ2FRkwPYTNVJHIiKiqxDEXnwnIwgC1q5di5kzZzq2ZWRkIC0tDStWrAAA2O126HQ6LFiwAIsWLerxMZ588kncdNNNuOeeey77usVigcVicTw3mUzQ6XQwGo3QaDQ9Ph75pv/9bxHe/qYEif0CsHHhDVD7yaWORETkU0wmE7Rabbd+fzt1zIvVakVBQQGysrIuHkAmQ1ZWFnbv3t2t9zAYDGhsbAQAGI1G7NixA8OGDbvi/suWLYNWq3U8dDpd7z4E+aRf3TwEkcEqnKlrxj+2c90jIiJ35tTyUltbC5vNhqioqC7bo6KiUFVV1a33OHv2LCZNmoTk5GRMmjQJCxYswOjRo6+4/+LFi2E0Gh2PsrKyXn0G8k3Baj8895OOwbv6bSdxts4scSIiIroShdQBfig9PR2FhYXd3l+lUkGlUrkuEPmMn4yJwcf5ZdhZXIsln3+HVXPTIAiC1LGIiOgHnHrlJTw8HHK5/JIBtgaDAdHR0c481CX0ej2SkpKQlpbm0uOQ9xIEAc/fMRJKuQzbT9Tgq++6d7WQiIj6llPLi1KpRGpqKnJzcx3b7HY7cnNzkZmZ6cxDXSI7OxtFRUXIy8tz6XHIuw2MCMITkwcC6Fj3yMw77xIRuZ0el5empiYUFhY6vtopKSlBYWEhSktLAQA5OTlYuXIl3n33XRw9ehTz58+H2WzG3LlznRqcyFWenDIYujB/VBpb8VpusdRxiIjoB3pcXvLz8zF27FiMHTsWQEdZGTt2LJYsWQIAmDVrFl566SUsWbIEKSkpKCwsxMaNGy8ZxOts/NqInEXtJ8f/3jEKAPDWrhIcr2qUOBEREX1fr+7z4o56Mk+c6Mf84l/5+Oo7A8YmhODTJyZCLuPgXSIiV5HsPi9E3uQPd4xEkEqBA6UNeG/3GanjEBFRJ5YXoiuI0fpj0fThAIC/bDyOsnou3EhE5A68prxwzAu5wgPpCUgfEIaWNht+t/YwVzgnInIDHPNCdBWna5pw66s7YW234+V7k3F3arzUkYiIvA7HvBA50cCIICzMGgIA+H/ri1DbZLnKTxARkSuxvBB1w7xJA5EUo0FDcxv+8J/vpI5DROTTvKa8cMwLuZKfXIa/3DMGcpmALw5VYnOR4eo/RERELsExL0Q9sGzDUfxj+2lEaVTYtHAytAF+UkciIvIKHPNC5CK/yhqKgeGBMJgsWPqfI1LHISLySSwvRD2g9pPj5Z8lQyYA6worsOFwpdSRiIh8DssLUQ+NTQjFkzcOBgD8bu1h1DRy9hERUV/ymvLCAbvUl345dQhGxGhwvrkNiz/jzeuIiPoSB+wSXaOjlSbcsWIX2mwiXrxnDO4dr5M6EhGRx+KAXaI+MCJGg1/dPBQA8L//LUJ5Q4vEiYiIfAPLC1Ev/OKGQRiXEIJGSzue/eQg7HavupBJROSWWF6IekEuE/Dyz1Lg7yfHt6fq8PY3JVJHIiLyeiwvRL00IDwQv799BADgzxuP4Ui5UeJERETezWvKC2cbkZQezEjALUlRaLOJ+OVHB2C2tEsdiYjIa3G2EZGTnDdbMf3VnagyteJn4+Pxl3uSpY5EROQxONuISAKhgUr8dVYKBAH4OP8cvjhUIXUkIiKvxPJC5ESZg/rhqSkdd99d/NlhlNU3S5yIiMj7sLwQOdnTU4d0TJ9ubcfTqw+g3WaXOhIRkVdheSFyMoVchlfvG4tglQL7Sxvwam6x1JGIiLwKywuRC+jCAvCnu0YDAFZsPYltx6slTkRE5D28prxwqjS5mxnJsXhoQgJEEVi4ppDLBxAROQmnShO5kKXdhnvf2I1D54xI1oXg419MgEohlzoWEZHb4VRpIjehUsihf2ActP5+OFjWgD+uPyp1JCIij8fyQuRiurAA/HVWxw3r3tt9Fp8XlkuciIjIs7G8EPWBm4ZHdbn/S7GhUeJERESei+WFqI/86uahmDioH5qtNsz/YD+auP4REdE1YXkh6iNymYBX7xuLKI0KJ6ub8Ks1hbDbvWq8PBFRn2B5IepDEcEqvP5QKpRyGTYXGbCcN7AjIuoxlheiPjYuIdRxA7vXcoux4XClxImIiDwLywuRBO5JjcfPrxsAAMj5+CCOVpokTkRE5DlYXogk8rvbhmPSkHC0tNkw77181JutUkciIvIIXlNeuDwAeRqFXIa/3T8W/fsF4Nz5Fjz5QQHauAI1EdFVcXkAIomdMDTiTv03MFtteCAjAX+cOQqCIEgdi4ioT3F5ACIPMjQqGK/eNxaCAHy4txQrd56WOhIRkVtjeSFyA1lJUfif25MAAH/68hi+5AwkIqIrYnkhchM/vy4RszP7AwB+taYQBWfPS5yIiMg9sbwQuQlBELBkxkhkjYiEpd2Oee/l42ydWepYRERuh+WFyI1cWEJgVJwG9WYr5r6Th4ZmTqEmIvo+lhciNxOoUuDt2WmI1apxutaMx98rQGubTepYRERug+WFyA1FatR4e24aglUK7DtTj19+dADtvAcMEREAlhcitzU8WoN/PjIeSoUMm4oM+P3aI/Cy2zIREV0TlhciN5Y5qB/+dv9YyARgTX4Z/vLVcakjERFJjuWFyM1NGxmNZZ2rUL++7RTe5E3siMjHsbwQeYBZaQn47a3DAQD/t/4o/l1wTuJERETScdvy0tzcjP79++PXv/611FGI3MITkwdi3qQBAIDf/PsQNn1XJXEiIiJpuG15+eMf/4gJEyZIHYPIbQiCgN/dNgL3pMbDZheR/eF+bD1WLXUsIqI+55blpbi4GMeOHcP06dOljkLkVgRBwAt3jcbto2PQZhPxi/cLsLO4RupYRER9qsflZceOHZgxYwZiY2MhCALWrVt3yT56vR6JiYlQq9XIyMjAvn37enSMX//611i2bFlPoxH5BIVchuX3peCWpChY2+147N187D5VJ3UsIqI+0+PyYjabkZycDL1ef9nX16xZg5ycHCxduhT79+9HcnIypk2bhurqi5e3U1JSMGrUqEseFRUV+PzzzzF06FAMHTr02j8VkZfzk8uw4oFxuGl4xzpIj76bh7wz9VLHIiLqE4LYi7teCYKAtWvXYubMmY5tGRkZSEtLw4oVKwAAdrsdOp0OCxYswKJFi676nosXL8b7778PuVyOpqYmtLW14ZlnnsGSJUsuu7/FYoHFYnE8N5lM0Ol0MBqN0Gg01/rRiDxCa5sN897Lx87iWgSpFPjXo+kYmxAqdSwioh4zmUzQarXd+v3t1DEvVqsVBQUFyMrKungAmQxZWVnYvXt3t95j2bJlKCsrw5kzZ/DSSy9h3rx5VywuF/bXarWOh06n6/XnIPIUaj85/vnweGQO7IcmSzseeWsf8nkFhoi8nFPLS21tLWw2G6Kiorpsj4qKQlWVa6Z1Ll68GEaj0fEoKytzyXGI3JW/Uo43Z49HxoAwNFra8cjb+zgGhoi8mkLqAD9mzpw5V91HpVJBpVK5PgyRGwtUKbBqbjoe/1fHV0hz3tmHlY+Mxw1DI6SORkTkdE698hIeHg65XA6DwdBlu8FgQHR0tDMPdQm9Xo+kpCSkpaW59DhE7spfKcfKR8Y7BvE+9m4+vi4yXP0HiYg8jFPLi1KpRGpqKnJzcx3b7HY7cnNzkZmZ6cxDXSI7OxtFRUXIy8tz6XGI3JnaT443HkrFrSOjYbXZ8cT7BdhwuFLqWERETtXj8tLU1ITCwkIUFhYCAEpKSlBYWIjS0lIAQE5ODlauXIl3330XR48exfz582E2mzF37lynBieiy1MqZFjxwFjckRyLdruIpz46gI/zORaMiLxHj8e85OfnY8qUKY7nOTk5AIDZs2dj1apVmDVrFmpqarBkyRJUVVUhJSUFGzduvGQQr7Pp9Xro9XrYbDaXHofIEyjkMvx1VgrUfjJ8nH8Ov/n0EOrNVvzihoEQBEHqeEREvdKr+7y4o57MEyfydqIo4oWNx/CP7acBAI9dPwC/u20EZDIWGCJyL5Ld54WI3IsgCFg8fQR+f9sIAMCbu0rwzCcH0WazS5yMiOjasbwQ+YB5NwzEy/cmQy4TsPZAOea9l49ma7vUsYiIronXlBdOlSb6cXenxuPNR8ZD7SfDtuM1uP+fe1Dd2Cp1LCKiHuOYFyIfU3D2PB59Nw8NzW2IC/HHO3PTMDQqWOpYROTjOOaFiK4otX8o1j55HQaEB6K8oQV3//1b7CyukToWEVG3sbwQ+aAB4YH4bP5EpCd2rIc05508fLSvVOpYRETd4jXlhWNeiHomNFCJfz2WjpkpsbDZRSz+7DCWbTgKm92rvkkmIi/EMS9EPk4URbyaW4zlXxcDAG4aHonl96VAo/aTOBkR+RKOeSGibhMEAQuzhuLV+1KgUsiw5Vg1Zq74Bierm6SORkR0WSwvRAQA+GlKHD59YiJitWqcrjXjTv03yD3KVamJyP2wvBCRw+h4Lf6z4HrHQN7H3svH33KL4WXfLhORh/Oa8sIBu0TOER6kwvuPZeDhCf0hisDLm0/g8X8VwNjSJnU0IiIAHLBLRD9i9b5SLPn8O1htdujC/PH3B1IxOl4rdSwi8kIcsEtETnFfegL+PX8idGH+KKtvwd2vf4t/7TnLr5GISFIsL0T0o0bHa/HFU5OQNSIKVpsdz607goVrCmG2cGFHIpIGywsRXZU2wA8rH0nF724bDrlMwOeFFZixYheOlBuljkZEPojlhYi6RRAEPH7DIKx+fAKiNWqcrjHjzr9/g39sPwU778pLRH3Ia8oLZxsR9Y20xDBseHoSpo2MQptNxLINx/DQW3tRZWyVOhoR+QjONiKiayKKItbkleH5/xahpc0Grb8f/nz3aNw6KkbqaETkgTjbiIhcThAE3JeegC9+eT1Gx2lhbGnDE+/vx28/PYQmDuYlIhdieSGiXhkUEYR/z5+IJyYPgiAAa/LLMO2vO7CruFbqaETkpVheiKjXlAoZFk0fjg8fm4D4UH+UN7Tgobf2YvFnh9HYyjvzEpFzsbwQkdNkDuqHrxbegEcy+wMAPtpXiluX78TO4hqJkxGRN2F5ISKnClQp8L8/HYWP5k2ALqzjKszDb+3Don8fgolXYYjICVheiMglMgf1w8anb8CciYkAgNV5Zch6eTu+OFTB5QWIqFe8przwPi9E7idQpcAf7hiJ1Y9PwIDwQFQ3WvDUhwcw5508lNY1Sx2PiDwU7/NCRH2itc2G17edwuvbTsFqs0OlkOGXU4dg3qSBUCq85v9HEdE14n1eiMjtqP3k+NXNQ7Fx4SRcN7gfLO12vPjVcdz22k7sOV0ndTwi8iAsL0TUpwZGBOH9RzOwfFYKwoOUOFndhPv+uQfZH+7HufP8KomIro7lhYj6nCAImDk2Drk5N+KhCQmQCcD6Q5WY+vJ2vLLpOJqtvEMvEV0Zx7wQkeSOVprw/H+/w57T9QCAaI0ai6YPx09TYiEIgsTpiKgv9OT3N8sLEbkFURTx1XdV+L/1R3HufAsAYFxCCH532wiMTwyTOB0RuRrLC8sLkcdqbbPhrV0l0G89iWarDQCQNSIKv7l1GIZGBUucjohcheWF5YXI4xlMrVj+dTE+zi+DzS5CJgB3j4vHr24eitgQf6njEZGTsbywvBB5jZPVTXjpq+PY+F0VgI5FIOdMTMT8yYMQGqiUOB0ROQvLC8sLkdfZX3oeL2w4hn0lHYN6A5VyzLkuEfMmDURIAEsMkafzyfKi1+uh1+ths9lw4sQJlhciLySKIrYdr8FfvjqOo5UmAECQSoE5ExPx2KQBLDFEHswny8sFvPJC5P3sdhGbjxqw/OviLiVm7nWJePR6lhgiT8TywvJC5BPsdhGbigxY/vUJHKtqBNDxddL96Ql4dNIAxGg5sJfIU7C8sLwQ+ZSOElOF5V8XO0qMn1zAT1Pi8MTkgRgcySnWRO6O5YXlhcgniaKIbSdq8Ma2U9jbObAX6LhPzPwbByK1P292R+SuWF5YXoh83oHS83hj+ylsKjLgwn/lxvcPxdzrBmDayCgo5FzajcidsLywvBBRp1M1Tfjn9tP47MA5tNk6/nMXo1XjoQn9cX96AsJ4rxgit8DywvJCRD9gMLXigz1n8cHeUtSZrQA6bng3MyUWsycmYmSsVuKERL6N5YXlhYiuoLXNhvWHKrHq2zM4XG50bE9PDMODExIwbWQ01H5yCRMS+SaWF5YXIroKURSxv/Q8Vn17FhsOV6Ld3vGfwpAAP9w1Nh73p+swhAtBEvUZlheWFyLqgSpjK9bklWFNXikqjK2O7an9Q3F/egJuHx0DfyWvxhC5EssLywsRXQObXcSO4hp8tLcUuceqYeu8GhOsVuAnY2Jx17g4jO8fCkEQJE5K5H1YXlheiKiXqk2t+KTgHFbnlaKsvsWxPSEsADPHxuGusXFIDA+UMCGRd/H48pKYmAiNRgOZTIbQ0FBs3bq12z/L8kJEzmS3i9hzug6fHSjHhsOVMFttjtfGJYTgrnHxuH10DEI55ZqoV7yivBw5cgRBQUE9/lmWFyJylWZrOzYXGfDv/eXYVVyDzm+VoJAJuG5wOG4fHYNbRkZxYUiia8DywvJCRC5WbWrF54UV+OxAuWNla6CjyFw/pLPIJEVDG+AnYUoiz9GT3989vj/2jh07MGPGDMTGxkIQBKxbt+6SffR6PRITE6FWq5GRkYF9+/b16BiCIGDy5MlIS0vDBx980NOIREQuF6lRY94NA7Hh6UnIfWYynrl5KIZHB6PdLmLb8Ro8++khjP/jZvx8VR4+zi9DXZNF6shEXkPR0x8wm81ITk7Gz3/+c9x1112XvL5mzRrk5OTgjTfeQEZGBpYvX45p06bh+PHjiIyMBACkpKSgvb39kp/dtGkTYmNjsWvXLsTFxaGyshJZWVkYPXo0xowZcw0fj4jI9QZFBGHB1CFYMHUITlY34cvDlVh/qBLHDY3YcqwaW45VQyZ0TL3OGhGFm5OiMDCi51eWiahDr742EgQBa9euxcyZMx3bMjIykJaWhhUrVgAA7HY7dDodFixYgEWLFvX4GM8++yxGjhyJOXPmXPZ1i8UCi+Xi/6MxmUzQ6XT82oiIJFdsaMT6w5XYXGTAdxWmLq8NjAjEzUlRuHlEFMYmhEIu4/Rr8m09+dqox1defozVakVBQQEWL17s2CaTyZCVlYXdu3d36z3MZjPsdjuCg4PR1NSELVu24Gc/+9kV91+2bBmef/75XmcnInK2IVHBWBgVjIVZQ1He0ILcowZsLjJgz+k6nK4x4x/bT+Mf208jNMAPk4ZE4IahEbhhSDgiNWqpoxO5NaeWl9raWthsNkRFRXXZHhUVhWPHjnXrPQwGA+68804AgM1mw7x585CWlnbF/RcvXoycnBzH8wtXXoiI3ElciD8eyUzEI5mJMLW2YceJGmwuMmDrsWqcb27Dfw5W4D8HKwAAI2I0uGFoOCYPiUBqYihUCt7dl+j7nFpenGHgwIE4ePBgt/dXqVRQqVTQ6/XQ6/Ww2WxX/yEiIglp1H74yZhY/GRMLNpsdhwobcCOEzXYUVyDw+VGHK004WilCf/YfhoBSjkmDOyHSUPCkTmoH4ZGBkPGr5jIxzm1vISHh0Mul8NgMHTZbjAYEB0d7cxDXSI7OxvZ2dmO78yIiDyBn1yG9AFhSB8Qhl9PG4a6Jgt2nazF9hM12Flci5pGi2PQLwCEBSqRMSAMmYP6YcLAfhgSGcTlCsjnOLW8KJVKpKamIjc31zGI1263Izc3F0899ZQzD0VE5JX6Banw05Q4/DQlDqIo4mhlI3YU1+Cbk7XIP3Me9WYrNhypwoYjVR37ByoxYWA/TBjUDxMGhGFQRBCvzJDX63F5aWpqwsmTJx3PS0pKUFhYiLCwMCQkJCAnJwezZ8/G+PHjkZ6ejuXLl8NsNmPu3LlODU5E5O0EQUBSrAZJsRo8MXkQrO12HC5vwJ7T9dh9qg75Z+tRZ7Zi/eFKrD9cCQDQ+vthXEIIUvuHIrV/GJJ1WgQo3W6EAFGv9Hiq9LZt2zBlypRLts+ePRurVq0CAKxYsQIvvvgiqqqqkJKSgtdeew0ZGRlOCXwl3x/zcuLECU6VJiKvZ2234+C5Buw5VYfdp+twoLQBLW1dx/3JZQKSYjRI7R+Kcf1DMS4hBHEh/vyqidyOxy8P0BtcHoCIfFWbzY5jlY3IP1uPgrPnsf/seVQYWy/ZLzxIidFxWoyOD0FyvBaj47WIDOb0bJIWywvLCxERAKCioQUFZ887HkcrTWi3X/qf/RitGqPjtEjWhXQUmzgtV8qmPuWT5YVfGxERXV1rmw1HK004dM7Y+WjAyZomXO43QbRGjRExwRgRo8HwGA2SYoKR2C8QCnmPl8UjuiqfLC8X8MoLEVHPmC3tOFJuxOFyIw52Fpqzdc2X3VelkGFYdDCGR3eUmhExGgyPDkZIAK/SUO+wvLC8EBH1SmNrG45XNeJopQlFlY04VmXC8apGNFsvfyPQ8CAlBkcGdTwigjAkKhiDI4MQGazi4GDqFpYXlhciIqez20WU1jc77gB8tLPcnDvfcsWfCVYrvldogjAgPAgDwgOgCwvgsgfUhU+WF455ISKShtnSjtM1ZhRXN+JkdROKq5twqroJZ+rMuMzYYACAIACxWn8MCA9E/34BnX8GYkB4AOJDA6D2Y7HxNT5ZXi7glRciIvdgabfhTG1zZ6HpKDYltWacrWtGk6X9ij93odgkhgdAFxqA+FB/xH/vz8hgFe8i7IV68vubt10kIiKXUCnkGBYdjGHRwQBiHNtFUURtkxVn68yOMlNSZ8bZOjPO1HYUm/KGFpQ3tACou+R9/eQCYkP8O8pMSGepCesoNnEh/ogMVnFGlJdjeSEioj4lCAIiglWICFZhfGJYl9dEUUSd2YoztWacqWvGufPNOHe+xfFnpbEVbTYRZ+uaO2dEXVpuZAIQHqRCjFaNKI2640+t+nvP/RGtUcNfya+mPBXLCxERuQ1BEBAepEJ40KXFBgDabXYYGi04V3+h1FwsNucamlHZ0Ip2u4jqRguqGy0AjFc8ltbfz1FoIjvLVHiQylGsLvxdo1ZwxpSb8Zry8v0Bu0RE5J0UchniQvwRF+KPy62YZ7eLqDVbYDBaUGlsgcHUikpjK6pMraj63p/NVhuMLW0wtrThWFXjjx5TqZAhIkiF8GAVIoKUHeWm83l4kAqhAUqEBSoRGuiH0AAl/PiVlctxwC4REfkUURRham13FBuDsRU1TRbUNHY+miyo7fyzsfXKA4uvJFitQGiAEqGBSoQF+HX+2fk8UNnxWoBfZ+FRQqP2g1LBwsMBu0RERFcgCAK0/n7Q+vthaFTwj+7b2mZDbWexqW2yOgrOxW0WnG+24nxzGxqarbCLQGNrOxpb21Faf/m7FF+Ov58cWn8/aPwVHX+q/Tqfdz7UCsdzx+sBHduDVL73tRbLCxER0RWo/eSd07QDrrqv3S7C1NqGerMV55utqDe34bzZivpma8efndvPN1/c3tDcBgBoabOhpc2GKlPPM8oEIEjVUWKC1AoEdv49WK1AoLJj24XXA6+w/cLfVQqZRxQhlhciIiInkMkEhAQoe7TOk80uorG1DaaWdhhb2mBq7RiHY+ocj3PxebtjjI6p9eLrbTYRdhEwtbbD1Nr+Y+OTu0UhExCglCNAqUCASn7x70o5ApUK+CvlCFTKMTQ6GA9m9O/dwXqTU7IjExER+Tj5NRSeC0RRRGubHcaWNjRZ2mG2tKPpwqO1HWZrx9dXl9ve1Hpxm9lic9w0sN0uXixCP2Ly0AiWF2fgbCMiIvIlgiDAXyl3yv1q7HYRzW02NLW2o9najmarDWZLO5rbbGi22C5us7ajxWpDQtjVv0ZzJc42IiIiIsn15Pc352YRERGRR2F5ISIiIo/C8kJEREQeheWFiIiIPArLCxEREXkUlhciIiLyKF5TXvR6PZKSkpCWliZ1FCIiInIh3ueFiIiIJMf7vBAREZHXYnkhIiIij8LyQkRERB6F5YWIiIg8CssLEREReRSF1AGc7cLkKZPJJHESIiIi6q4Lv7e7Mwna68pLY2MjAECn00mchIiIiHqqsbERWq32R/fxuvu82O12VFRUIDg4GIIgOPW9TSYTdDodysrKeA8ZF+J57hs8z32H57pv8Dz3HVeca1EU0djYiNjYWMhkPz6qxeuuvMhkMsTHx7v0GBqNhv/D6AM8z32D57nv8Fz3DZ7nvuPsc321Ky4XcMAuEREReRSWFyIiIvIoLC89oFKpsHTpUqhUKqmjeDWe577B89x3eK77Bs9z35H6XHvdgF0iIiLybrzyQkRERB6F5YWIiIg8CssLEREReRSWFyIiIvIoLC/dpNfrkZiYCLVajYyMDOzbt0/qSB5nx44dmDFjBmJjYyEIAtatW9fldVEUsWTJEsTExMDf3x9ZWVkoLi7usk99fT0efPBBaDQahISE4NFHH0VTU1Mffgr3tmzZMqSlpSE4OBiRkZGYOXMmjh8/3mWf1tZWZGdno1+/fggKCsLdd98Ng8HQZZ/S0lLcfvvtCAgIQGRkJJ599lm0t7f35Udxe6+//jrGjBnjuElXZmYmNmzY4Hid59k1XnjhBQiCgIULFzq28Vw7xx/+8AcIgtDlMXz4cMfrbnWeRbqq1atXi0qlUnz77bfF7777Tpw3b54YEhIiGgwGqaN5lC+//FL8/e9/L3722WciAHHt2rVdXn/hhRdErVYrrlu3Tjx48KB4xx13iAMGDBBbWloc+9x6661icnKyuGfPHnHnzp3i4MGDxfvvv7+PP4n7mjZtmvjOO++IR44cEQsLC8XbbrtNTEhIEJuamhz7PPHEE6JOpxNzc3PF/Px8ccKECeLEiRMdr7e3t4ujRo0Ss7KyxAMHDohffvmlGB4eLi5evFiKj+S2/vOf/4jr168XT5w4IR4/flz83e9+J/r5+YlHjhwRRZHn2RX27dsnJiYmimPGjBGffvppx3aea+dYunSpOHLkSLGystLxqKmpcbzuTueZ5aUb0tPTxezsbMdzm80mxsbGisuWLZMwlWf7YXmx2+1idHS0+OKLLzq2NTQ0iCqVSvzoo49EURTFoqIiEYCYl5fn2GfDhg2iIAhieXl5n2X3JNXV1SIAcfv27aIodpxTPz8/8ZNPPnHsc/ToURGAuHv3blEUO0qmTCYTq6qqHPu8/vrrokajES0WS99+AA8TGhoqvvnmmzzPLtDY2CgOGTJE3Lx5szh58mRHeeG5dp6lS5eKycnJl33N3c4zvza6CqvVioKCAmRlZTm2yWQyZGVlYffu3RIm8y4lJSWoqqrqcp61Wi0yMjIc53n37t0ICQnB+PHjHftkZWVBJpNh7969fZ7ZExiNRgBAWFgYAKCgoABtbW1dzvPw4cORkJDQ5TyPHj0aUVFRjn2mTZsGk8mE7777rg/Tew6bzYbVq1fDbDYjMzOT59kFsrOzcfvtt3c5pwD/nXa24uJixMbGYuDAgXjwwQdRWloKwP3Os9ctzOhstbW1sNlsXf5hAEBUVBSOHTsmUSrvU1VVBQCXPc8XXquqqkJkZGSX1xUKBcLCwhz70EV2ux0LFy7Eddddh1GjRgHoOIdKpRIhISFd9v3heb7cP4cLr9FFhw8fRmZmJlpbWxEUFIS1a9ciKSkJhYWFPM9OtHr1auzfvx95eXmXvMZ/p50nIyMDq1atwrBhw1BZWYnnn38ekyZNwpEjR9zuPLO8EHmp7OxsHDlyBLt27ZI6itcaNmwYCgsLYTQa8emnn2L27NnYvn271LG8SllZGZ5++mls3rwZarVa6jhebfr06Y6/jxkzBhkZGejfvz8+/vhj+Pv7S5jsUvza6CrCw8Mhl8svGVFtMBgQHR0tUSrvc+Fc/th5jo6ORnV1dZfX29vbUV9fz38WP/DUU0/hiy++wNatWxEfH+/YHh0dDavVioaGhi77//A8X+6fw4XX6CKlUonBgwcjNTUVy5YtQ3JyMl599VWeZycqKChAdXU1xo0bB4VCAYVCge3bt+O1116DQqFAVFQUz7WLhISEYOjQoTh58qTb/TvN8nIVSqUSqampyM3NdWyz2+3Izc1FZmamhMm8y4ABAxAdHd3lPJtMJuzdu9dxnjMzM9HQ0ICCggLHPlu2bIHdbkdGRkafZ3ZHoijiqaeewtq1a7FlyxYMGDCgy+upqanw8/Prcp6PHz+O0tLSLuf58OHDXYri5s2bodFokJSU1DcfxEPZ7XZYLBaeZyeaOnUqDh8+jMLCQsdj/PjxePDBBx1/57l2jaamJpw6dQoxMTHu9++0U4f/eqnVq1eLKpVKXLVqlVhUVCQ+/vjjYkhISJcR1XR1jY2N4oEDB8QDBw6IAMRXXnlFPHDggHj27FlRFDumSoeEhIiff/65eOjQIfGnP/3pZadKjx07Vty7d6+4a9cucciQIZwq/T3z588XtVqtuG3bti7THZubmx37PPHEE2JCQoK4ZcsWMT8/X8zMzBQzMzMdr1+Y7njLLbeIhYWF4saNG8WIiAhOK/2BRYsWidu3bxdLSkrEQ4cOiYsWLRIFQRA3bdokiiLPsyt9f7aRKPJcO8szzzwjbtu2TSwpKRG/+eYbMSsrSwwPDxerq6tFUXSv88zy0k1/+9vfxISEBFGpVIrp6eninj17pI7kcbZu3SoCuOQxe/ZsURQ7pks/99xzYlRUlKhSqcSpU6eKx48f7/IedXV14v333y8GBQWJGo1GnDt3rtjY2CjBp3FPlzu/AMR33nnHsU9LS4v45JNPiqGhoWJAQIB45513ipWVlV3e58yZM+L06dNFf39/MTw8XHzmmWfEtra2Pv407u3nP/+52L9/f1GpVIoRERHi1KlTHcVFFHmeXemH5YXn2jlmzZolxsTEiEqlUoyLixNnzZolnjx50vG6O51nQRRF0bnXcoiIiIhch2NeiIiIyKOwvBAREZFHYXkhIiIij8LyQkRERB6F5YWIiIg8CssLEREReRSWFyIiIvIoLC9ERETkUVheiIiIyKOwvBAREZFHYXkhIiIij8LyQkRERB7l/wOF8vn/IEsWXAAAAABJRU5ErkJggg==",
152 |       "text/plain": [
153 |        "<Figure size 640x480 with 1 Axes>"
154 |       ]
155 |      },
156 |      "metadata": {},
157 |      "output_type": "display_data"
158 |     }
159 |    ],
160 |    "source": [
161 |     "x = vp.array(np.array([[-1, -1], [-1, 1], [1, -1], [1, 1]]), requires_grad=False)\n",
162 |     "y = vp.array(np.array([[-1], [1], [1], [-1]]), requires_grad=False)\n",
163 |     "W_1 = vp.array(np.random.uniform(-1./np.sqrt(2),1./np.sqrt(2),(64, 2)))\n",
164 |     "b_1 = vp.array(np.zeros(64))\n",
165 |     "W_2 = vp.array(np.random.uniform(-1./np.sqrt(64),1./np.sqrt(64),(16, 64)))\n",
166 |     "b_2 = vp.array(np.zeros(16))\n",
167 |     "W_3 = vp.array(np.random.uniform(-1./np.sqrt(16),1./np.sqrt(16),(1, 16)))\n",
168 |     "b_3 = vp.array(np.zeros(1))\n",
169 |     "losses = []\n",
170 |     "lr = 0.1\n",
171 |     "for i in tqdm(range(500)):\n",
172 |     "    f_1 = np.tanh(x @ W_1.T + b_1)\n",
173 |     "    f_2 = np.tanh(f_1 @ W_2.T + b_2)\n",
174 |     "    z_3 = f_2 @ W_3.T + b_3\n",
175 |     "    loss = np.mean((z_3 - y) ** 2)\n",
176 |     "    losses.append(loss)\n",
177 |     "    loss.backward()\n",
178 |     "    with grad.no_grad():\n",
179 |     "        W_1 = W_1 - lr * vp.array_grad(W_1)\n",
180 |     "        b_1 = b_1 - lr * vp.array_grad(b_1)\n",
181 |     "        W_2 = W_2 - lr * vp.array_grad(W_2)\n",
182 |     "        b_2 = b_2 - lr * vp.array_grad(b_2)\n",
183 |     "        W_3 = W_3 - lr * vp.array_grad(W_3)\n",
184 |     "        b_3 = b_3 - lr * vp.array_grad(b_3)\n",
185 |     "    loss.zero_grad()\n",
186 |     "    lr *= 0.99\n",
187 |     "print(z_3)\n",
188 |     "plt.plot(losses)\n",
189 |     "plt.yscale('log')"
190 |    ]
191 |   }
192 |  ],
193 |  "metadata": {
194 |   "kernelspec": {
195 |    "display_name": "Python 3",
196 |    "language": "python",
197 |    "name": "python3"
198 |   },
199 |   "language_info": {
200 |    "codemirror_mode": {
201 |     "name": "ipython",
202 |     "version": 3
203 |    },
204 |    "file_extension": ".py",
205 |    "mimetype": "text/x-python",
206 |    "name": "python",
207 |    "nbconvert_exporter": "python",
208 |    "pygments_lexer": "ipython3",
209 |    "version": "3.9.13"
210 |   },
211 |   "orig_nbformat": 4
212 |  },
213 |  "nbformat": 4,
214 |  "nbformat_minor": 2
215 | }
216 | 


--------------------------------------------------------------------------------