├── .gitignore ├── python ├── setup.py ├── aapy.pyx ├── example.py └── aa.ipynb ├── .github └── workflows │ ├── valgrind.yml │ └── build.yml ├── Makefile ├── LICENSE.txt ├── test ├── minunit.h └── run_tests.c ├── include ├── aa_blas.h └── aa.h ├── README.md ├── examples └── gd.c └── src └── aa.c /.gitignore: -------------------------------------------------------------------------------- 1 | out 2 | *.o 3 | *.swp 4 | python/aapy.c 5 | python/build 6 | python/.* 7 | -------------------------------------------------------------------------------- /python/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | from distutils.extension import Extension 3 | from Cython.Build import cythonize 4 | 5 | aa_extension = Extension( 6 | name="aa", 7 | sources=["aapy.pyx"], 8 | library_dirs=["../src"], 9 | include_dirs=["../include"], 10 | libraries=['lapack', 'blas'] 11 | ) 12 | 13 | setup( 14 | name="aa", 15 | version='0.0.1', 16 | ext_modules=cythonize([aa_extension]) 17 | ) 18 | 19 | -------------------------------------------------------------------------------- /.github/workflows/valgrind.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Valgrind 3 | 4 | on: [push, pull_request] 5 | 6 | jobs: 7 | linux: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - run: sudo apt-get install libopenblas-dev liblapack-dev valgrind 12 | - run: make 13 | - run: make test 14 | - run: valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --show-reachable=yes --error-exitcode=1 out/run_tests 15 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Build 3 | 4 | on: [push, pull_request] 5 | 6 | jobs: 7 | linux: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - run: sudo apt-get install libopenblas-dev liblapack-dev 12 | - run: make 13 | - run: make test 14 | - run: out/run_tests 15 | 16 | # runs-on: windows-latest 17 | # steps: 18 | # - uses: actions/checkout@v2 19 | # - run: choco install clapack 20 | # - run: make 21 | # - run: make test 22 | # - run: test/run_tests 23 | 24 | mac: 25 | runs-on: macos-latest 26 | steps: 27 | - uses: actions/checkout@v2 28 | - run: brew install openblas lapack 29 | - run: make 30 | - run: make test 31 | - run: out/run_tests 32 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # MAKEFILE for aa 2 | .PHONY: default clean purge 3 | 4 | OBJECTS = src/aa.o 5 | 6 | PROFILING = 0 7 | CFLAGS += -g -Wall -O3 -Iinclude -DPROFILING=$(PROFILING) 8 | 9 | SRC_FILES = $(wildcard src/*.c) 10 | INC_FILES = $(wildcard include/*.h) 11 | 12 | OUT = out 13 | ARCHIVE = ar -rv 14 | RANLIB = ranlib 15 | 16 | default: $(OUT)/libaa.a $(OUT)/gd 17 | 18 | %.o : src/%.c 19 | $(CC) $(CFLAGS) -c $< -o $@ 20 | 21 | src/aa.o : $(SRC_FILES) $(INC_FILES) 22 | 23 | $(OUT)/libaa.a: $(OBJECTS) 24 | mkdir -p $(OUT) 25 | $(ARCHIVE) $@ $^ 26 | - ranlib $@ 27 | 28 | $(OUT)/gd: examples/gd.c $(OUT)/libaa.a 29 | $(CC) $(CFLAGS) -o $@ $^ -lblas -llapack 30 | 31 | clean: 32 | @rm -rf $(OBJECTS) 33 | purge: clean 34 | @rm -rf $(OUT) 35 | 36 | test: $(OUT)/run_tests 37 | 38 | $(OUT)/run_tests: test/run_tests.c $(OUT)/libaa.a 39 | $(CC) $(CFLAGS) -o $@ $^ -lblas -llapack 40 | 41 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Brendan O'Donoghue (bodonoghue85@gmail.com) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/minunit.h: -------------------------------------------------------------------------------- 1 | /* Taken from http://www.jera.com/techinfo/jtns/jtn002.html */ 2 | 3 | /* Simple Macros for testing */ 4 | #define mu_assert_less(message, a, b) \ 5 | do { \ 6 | if (a > b) { \ 7 | printf("%s: %1.3e > %1.3e\n", message, a, b); \ 8 | return message; \ 9 | } \ 10 | } while (0) 11 | #define mu_assert(message, test) \ 12 | do { \ 13 | if (!(test)) \ 14 | return message; \ 15 | } while (0) 16 | #define mu_run_test(test) \ 17 | do { \ 18 | const char *message = test(); \ 19 | tests_run++; \ 20 | if (message) \ 21 | return message; \ 22 | } while (0) 23 | -------------------------------------------------------------------------------- /python/aapy.pyx: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | cdef extern from "../src/aa.c": 4 | pass 5 | 6 | cdef extern from "../include/aa.h": 7 | ctypedef struct AaWork: 8 | pass 9 | AaWork *aa_init(int, int, int, float, float, float, float, int) 10 | double aa_apply(double*, const double*, AaWork*) 11 | int aa_safeguard(double*, double*, AaWork*) 12 | void aa_reset(AaWork*) 13 | void aa_finish(AaWork *) 14 | 15 | cdef class AndersonAccelerator(object): 16 | cdef AaWork* _wrk 17 | cdef int _dim 18 | 19 | def __cinit__(self, dim, mem, type1=False, regularization=1e-12, 20 | relaxation=1.0, safeguard_factor=1.0, max_weight_norm=1e6, 21 | verbosity=0): 22 | self._wrk = aa_init(dim, mem, type1, regularization, relaxation, 23 | safeguard_factor, max_weight_norm, verbosity) 24 | self._dim = dim 25 | 26 | def _validate(self, f, x): 27 | f = np.squeeze(f) 28 | x = np.squeeze(x) 29 | if (f.shape != (self._dim,) or x.shape != (self._dim,)): 30 | raise ValueError("Incorrect input dimension") 31 | 32 | if not f.flags['C_CONTIGUOUS']: 33 | # Makes a contiguous copy of the numpy array. 34 | f = np.ascontiguousarray(f) 35 | if not x.flags['C_CONTIGUOUS']: 36 | # Makes a contiguous copy of the numpy array. 37 | x = np.ascontiguousarray(x) 38 | 39 | return f, x 40 | 41 | def apply(self, f, x): 42 | f, x = self._validate(f, x) 43 | cdef double[::1] f_memview = f 44 | cdef double[::1] x_memview = x 45 | return aa_apply(&f_memview[0], &x_memview[0], self._wrk) 46 | 47 | def safeguard(self, f_new, x_new): 48 | f_new , x_new = self._validate(f_new, x_new) 49 | cdef double[::1] f_memview = f_new 50 | cdef double[::1] x_memview = x_new 51 | return aa_safeguard(&f_memview[0], &x_memview[0], self._wrk) 52 | 53 | def reset(self): 54 | aa_reset(self._wrk) 55 | 56 | def __dealloc__(self): 57 | aa_finish(self._wrk) 58 | 59 | -------------------------------------------------------------------------------- /include/aa_blas.h: -------------------------------------------------------------------------------- 1 | #ifndef AA_BLAS_H_GUARD 2 | #define AA_BLAS_H_GUARD 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include "aa.h" 9 | 10 | /* Default to underscore for blas / lapack */ 11 | #ifndef BLASSUFFIX 12 | #define BLASSUFFIX _ 13 | #endif 14 | 15 | /* annoying hack because some preprocessors can't handle empty macros */ 16 | #if defined(NOBLASSUFFIX) && NOBLASSUFFIX > 0 17 | /* single or double precision */ 18 | #ifndef SFLOAT 19 | #define BLAS(x) d##x 20 | #else 21 | #define BLAS(x) s##x 22 | #endif 23 | #else 24 | /* this extra indirection is needed for BLASSUFFIX to work correctly as a 25 | * variable */ 26 | #define stitch_(pre, x, post) pre##x##post 27 | #define stitch__(pre, x, post) stitch_(pre, x, post) 28 | /* single or double precision */ 29 | #ifndef SFLOAT 30 | #define BLAS(x) stitch__(d, x, BLASSUFFIX) 31 | #else 32 | #define BLAS(x) stitch__(s, x, BLASSUFFIX) 33 | #endif 34 | #endif 35 | 36 | #ifdef MATLAB_MEX_FILE 37 | typedef ptrdiff_t blas_int; 38 | #elif defined BLAS64 39 | #include 40 | typedef int64_t blas_int; 41 | #else 42 | typedef int blas_int; 43 | #endif 44 | 45 | /* BLAS functions used */ 46 | aa_float BLAS(nrm2)(blas_int *n, aa_float *x, blas_int *incx); 47 | void BLAS(axpy)(blas_int *n, aa_float *a, const aa_float *x, blas_int *incx, 48 | aa_float *y, blas_int *incy); 49 | void BLAS(gemv)(const char *trans, const blas_int *m, const blas_int *n, 50 | const aa_float *alpha, const aa_float *a, const blas_int *lda, 51 | const aa_float *x, const blas_int *incx, const aa_float *beta, 52 | aa_float *y, const blas_int *incy); 53 | void BLAS(gesv)(blas_int *n, blas_int *nrhs, aa_float *a, blas_int *lda, 54 | blas_int *ipiv, aa_float *b, blas_int *ldb, blas_int *info); 55 | void BLAS(gemm)(const char *transa, const char *transb, blas_int *m, 56 | blas_int *n, blas_int *k, aa_float *alpha, aa_float *a, 57 | blas_int *lda, aa_float *b, blas_int *ldb, aa_float *beta, 58 | aa_float *c, blas_int *ldc); 59 | void BLAS(scal)(const blas_int *n, const aa_float *a, aa_float *x, 60 | const blas_int *incx); 61 | 62 | #ifdef __cplusplus 63 | } 64 | #endif 65 | 66 | #endif /* AA_BLAS_H_GUARD */ 67 | -------------------------------------------------------------------------------- /python/example.py: -------------------------------------------------------------------------------- 1 | # min (1/2) x'Q'x - q'x 2 | 3 | from __future__ import print_function 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | import aa 7 | 8 | dim = 100 9 | mems = [5, 10, 20, 50] 10 | N = int(1e4) 11 | 12 | np.random.seed(1234) 13 | 14 | Q = np.random.randn(dim, dim) 15 | Q = 0.1 * Q.T.dot(Q) 16 | q = np.random.randn(dim) 17 | x_0 = np.random.randn(dim) 18 | x_star = np.linalg.solve(Q, q) 19 | 20 | step = 1.0 / np.max(np.linalg.eigvals(Q)) 21 | 22 | f = lambda x: 0.5 * x.T @ Q @ x - q.T @ x 23 | 24 | f_star = f(x_star) 25 | print('f^* = ', f_star) 26 | 27 | print('No acceleration') 28 | 29 | results = {} 30 | 31 | fs = [] 32 | x = x_0.copy() 33 | for i in range(N): 34 | x_prev = np.copy(x) 35 | x -= step * (Q.dot(x) - q) 36 | fs.append(f(x) - f_star) 37 | if i % 1000 == 0: 38 | print('i: ', i,' f - f^*: ', np.abs(f(x) - f_star)) 39 | 40 | results['No accel'] = fs 41 | 42 | RELAXATION = 1.0 43 | 44 | for mem in mems: 45 | print('Type-I acceleration, mem:', mem) 46 | fs = [] 47 | x = x_0.copy() 48 | aa_wrk = aa.AndersonAccelerator(dim, mem, True, regularization=1e-8, 49 | relaxation=RELAXATION, verbosity=1, 50 | max_weight_norm=1e6) 51 | for i in range(N): 52 | if i > 0: aa_wrk.apply(x, x_prev) 53 | x_prev = np.copy(x) 54 | x -= step * (Q.dot(x) - q) 55 | aa_wrk.safeguard(x, x_prev) 56 | fs.append(f(x) - f_star) 57 | if i % 1000 == 0: 58 | print('i: ', i,' f - f^*: ', np.abs(f(x) - f_star)) 59 | 60 | results[f'AA-I {mem}'] = fs 61 | 62 | print('Type-II acceleration, mem:', mem) 63 | fs = [] 64 | x = x_0.copy() 65 | aa_wrk = aa.AndersonAccelerator(dim, mem, False, regularization=1e-12, 66 | relaxation=RELAXATION, verbosity=1, 67 | max_weight_norm=1e6) 68 | for i in range(N): 69 | if i > 0: aa_wrk.apply(x, x_prev) 70 | x_prev = np.copy(x) 71 | x -= step * (Q.dot(x) - q) 72 | aa_wrk.safeguard(x, x_prev) 73 | fs.append(f(x) - f_star) 74 | if i % 1000 == 0: 75 | print('i: ', i,' f - f^*: ', np.abs(f(x) - f_star)) 76 | 77 | results[f'AA-II {mem}'] = fs 78 | 79 | for k,v in results.items(): 80 | plt.semilogy(v, label=k) 81 | 82 | plt.legend() 83 | plt.show() 84 | -------------------------------------------------------------------------------- /include/aa.h: -------------------------------------------------------------------------------- 1 | #ifndef AA_H_GUARD 2 | #define AA_H_GUARD 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | typedef double aa_float; 13 | typedef int aa_int; 14 | 15 | typedef struct ACCEL_WORK AaWork; 16 | 17 | /** 18 | * Initialize Anderson Acceleration, allocates memory. 19 | * 20 | * @param dim the dimension of the variable for AA 21 | * @param mem the memory (number of past iterations used) for AA 22 | * @param type1 if True use type 1 AA, otherwise use type 2 23 | * @param regularization type-I and type-II different, for type-I: 1e-8 works 24 | * well, type-II: more stable can use 1e-12 often 25 | * @param relaxation float \in [0,2], mixing parameter (1.0 is vanilla) 26 | * @param safeguard_factor factor that controls safeguarding checks 27 | * larger is more aggressive but less stable 28 | * @param max_weight_norm float, maximum norm of AA weights 29 | * @param verbosity if greater than 0 prints out various info 30 | * 31 | * @return pointer to AA workspace 32 | * 33 | */ 34 | AaWork *aa_init(aa_int dim, aa_int mem, aa_int type1, aa_float regularization, 35 | aa_float relaxation, aa_float safeguard_factor, 36 | aa_float max_weight_norm, aa_int verbosity); 37 | 38 | /** 39 | * Apply Anderson Acceleration. The usage pattern should be as follows: 40 | * 41 | * - for i = 0 .. N: 42 | * - if (i > 0): aa_apply(x, x_prev, a) 43 | * - x_prev = x.copy() 44 | * - x = F(x) 45 | * - aa_safeguard(x, x_prev, a) // optional but helps stability 46 | * 47 | * Here F is the map we are trying to find the fixed point for. We put the AA 48 | * before the map so that any properties of the map are maintained at the end. 49 | * Eg if the map contains a projection onto a set then the output is guaranteed 50 | * to be in the set. 51 | * 52 | * 53 | * @param f output of map at current iteration, overwritten with AA output 54 | * @param x input to map at current iteration 55 | * @param a workspace from aa_init 56 | * 57 | * @return (+ or -) norm of AA weights vector. If positive then update 58 | * was accepted and f contains new point, if negative then update was 59 | * rejected and f is unchanged 60 | * 61 | */ 62 | aa_float aa_apply(aa_float *f, const aa_float *x, AaWork *a); 63 | 64 | /** 65 | * Apply safeguarding. 66 | * 67 | * This step is optional but can improve stability. 68 | * 69 | * @param f_new output of map after AA step 70 | * @param x_new AA output that is input to the map 71 | * @param a workspace from aa_init 72 | * 73 | * @returns 0 if AA step is accepted otherwise -1, if AA step is rejected then 74 | * this overwrites f_new and x_new with previous values 75 | * 76 | */ 77 | aa_int aa_safeguard(aa_float *f_new, aa_float *x_new, AaWork *a); 78 | 79 | /** 80 | * Finish Anderson Acceleration, clears memory. 81 | * 82 | * @param a AA workspace from aa_init 83 | * 84 | */ 85 | void aa_finish(AaWork *a); 86 | 87 | /** 88 | * Reset Anderson Acceleration. 89 | * 90 | * Resets AA as if at the first iteration, reuses original memory allocations. 91 | * 92 | * @param a AA workspace from aa_init 93 | * 94 | */ 95 | void aa_reset(AaWork *a); 96 | 97 | #ifdef __cplusplus 98 | } 99 | #endif 100 | #endif 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | AA 2 | === 3 | 4 | [![Build Status](https://github.com/cvxgrp/aa/actions/workflows/build.yml/badge.svg)](https://github.com/cvxgrp/aa/actions/workflows/build.yml) 5 | 6 | AA (`Anderson Acceleration`) 7 | 8 | C (with python interface) implementation of the Anderson Acceleration algorithm as described in our paper [Globally Convergent Type-I Anderson Acceleration for Non-Smooth Fixed-Point Iterations](https://web.stanford.edu/~boyd/papers/nonexp_global_aa1.html) 9 | 10 | NOTE: This implementation is a simple proof-of-concept and does not include all 11 | the necessary stabilizations required to guarantee convergence. However, it 12 | works well in many cases. 13 | 14 | MATLAB code (and the experiments presented in the paper) available [here](https://github.com/cvxgrp/nonexp_global_aa1/): 15 | 16 | ---- 17 | 18 | Python 19 | ---- 20 | 21 | To install the package use: 22 | ```bash 23 | cd python 24 | python setup.py install 25 | ``` 26 | To test, run in the same directory: 27 | ```bash 28 | python example.py 29 | ``` 30 | 31 | The Python API is as follows. To initialize the accelerator: 32 | ```python 33 | import aa 34 | aa_wrk = aa.AndersonAccelerator(dim, mem, type1, eta) 35 | ``` 36 | where: 37 | * `dim` is the integer problem dimension. 38 | * `mem` is the integer amount of memory (or lookback) you want the algorithm to use, around 10 is a good number for this. 39 | * `type1` is a boolean, if `True` uses type-1 AA, otherwise uses type-2 AA. 40 | * `regularization`: float, regularization param, type-I: 1e-8 works well, type-II: more stable can use 1e-10 often 41 | * `relaxation`: float in [0,2], mixing parameter (1.0 is vanilla AA) 42 | * `verbosity`: verbosity level, if greater than 0 prints out various info 43 | 44 | To use the accelerator: 45 | ```python 46 | aa_wrk.apply(x, x_prev) 47 | ``` 48 | where: 49 | * `x` is the numpy array consisting of the current iterate and it will be overwritten with the accelerated iterate. 50 | * `x_prev` is the numpy array consisting of the previous iterate (the input to the update function). 51 | 52 | 53 | C 54 | ---- 55 | 56 | At the command prompt type `make` to compile the library and the example. The 57 | example can be run by `out/gd`. 58 | 59 | The C API is as follows: 60 | 61 | ```C 62 | /* Initialize Anderson Acceleration, allocates memory. 63 | * 64 | * Args: 65 | * dim: the dimension of the variable for aa 66 | * mem: the memory (number of past iterations used) for aa 67 | * type1: bool, if True use type 1 aa, otherwise use type 2 68 | * regularization: float, regularization param, type-I and type-II different 69 | * for type-I: 1e-8 works well, type-II: more stable can use 1e-10 often 70 | * relaxation: float \in [0,2], mixing parameter (1.0 is vanilla AA) 71 | * verbosity: if greater than 0 prints out various info 72 | 73 | * Reurns: 74 | * Pointer to aa workspace 75 | */ 76 | AaWork *aa_init(aa_int dim, aa_int mem, aa_int type1, aa_float regularization, 77 | aa_float relaxation, aa_int verbosity); 78 | 79 | /* Apply Anderson Acceleration. 80 | * 81 | * Args: 82 | * f: output of map at current iteration, overwritten with aa output at end. 83 | * x: input to map at current iteration 84 | * a: aa workspace from aa_init 85 | * 86 | * Returns: 87 | * (float) (+ or -) norm of AA weights vector: 88 | * if positive then update was accepted and f contains new point 89 | * if negative then update was rejected and f is unchanged 90 | */ 91 | aa_float aa_apply(aa_float *f, const aa_float *x, AaWork *a); 92 | 93 | /* Finish Anderson Acceleration, clears memory. 94 | * 95 | * Args: 96 | * a: aa workspace from aa_init. 97 | */ 98 | void aa_finish(AaWork *a); 99 | ``` 100 | 101 | -------------------------------------------------------------------------------- /examples/gd.c: -------------------------------------------------------------------------------- 1 | /* Gradient descent (GD) on convex quadratic */ 2 | #include "aa.h" 3 | #include "aa_blas.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | /* default parameters */ 10 | #define SEED (1234) 11 | #define TYPE1 (1) 12 | #define DIM (1000) 13 | #define MEM (5) 14 | #define REGULARIZATION (0) 15 | #define SAFEGUARD_TOLERANCE (2.0) 16 | #define MAX_AA_NORM (1e10) 17 | #define RELAXATION (1.0) 18 | #define ITERS (30000) 19 | #define STEPSIZE (0.001) 20 | #define PRINT_INTERVAL (500) 21 | #define VERBOSITY (1) 22 | 23 | 24 | /* duplicate these with underscore prefix */ 25 | typedef struct _timer { 26 | struct timespec tic; 27 | struct timespec toc; 28 | } _timer; 29 | 30 | void _tic(_timer *t) { 31 | clock_gettime(CLOCK_MONOTONIC, &t->tic); 32 | } 33 | 34 | aa_float _tocq(_timer *t) { 35 | struct timespec temp; 36 | 37 | clock_gettime(CLOCK_MONOTONIC, &t->toc); 38 | 39 | if ((t->toc.tv_nsec - t->tic.tv_nsec) < 0) { 40 | temp.tv_sec = t->toc.tv_sec - t->tic.tv_sec - 1; 41 | temp.tv_nsec = 1e9 + t->toc.tv_nsec - t->tic.tv_nsec; 42 | } else { 43 | temp.tv_sec = t->toc.tv_sec - t->tic.tv_sec; 44 | temp.tv_nsec = t->toc.tv_nsec - t->tic.tv_nsec; 45 | } 46 | return (aa_float)temp.tv_sec * 1e3 + (aa_float)temp.tv_nsec / 1e6; 47 | } 48 | 49 | /* uniform random number in [-1,1] */ 50 | static aa_float rand_float(void) { 51 | return 2 * (((aa_float)rand()) / RAND_MAX) - 1; 52 | } 53 | 54 | /* 55 | * out/gd memory dimension step_size type1 seed iters regularization 56 | * 57 | */ 58 | int main(int argc, char **argv) { 59 | aa_int type1 = TYPE1, n = DIM, iters = ITERS, memory = MEM, seed = SEED; 60 | aa_int i, one = 1; 61 | aa_int verbosity = VERBOSITY; 62 | aa_float neg_step_size = -STEPSIZE; 63 | aa_float regularization = REGULARIZATION; 64 | aa_float relaxation = RELAXATION; 65 | aa_float safeguard_tolerance = SAFEGUARD_TOLERANCE; 66 | aa_float max_aa_norm = MAX_AA_NORM; 67 | aa_float err = 0; 68 | aa_float *x, *xprev, *Qhalf, *Q, zerof = 0.0, onef = 1.0; 69 | _timer aa_timer; 70 | aa_float aa_time = 0; 71 | 72 | printf("Usage: 'out/gd memory type1 dimension step_size seed iters " 73 | "regularization relaxation safeguard_tolerance max_aa_norm'\n"); 74 | 75 | switch (argc - 1) { 76 | case 10: 77 | max_aa_norm = atof(argv[10]); 78 | case 9: 79 | safeguard_tolerance = atof(argv[9]); 80 | case 8: 81 | relaxation = atof(argv[8]); 82 | case 7: 83 | regularization = atof(argv[7]); 84 | case 6: 85 | iters = atoi(argv[6]); 86 | case 5: 87 | seed = atoi(argv[5]); 88 | case 4: 89 | neg_step_size = -atof(argv[4]); 90 | case 3: 91 | n = atoi(argv[3]); 92 | case 2: 93 | type1 = atoi(argv[2]); 94 | case 1: 95 | memory = atoi(argv[1]); 96 | break; 97 | default: 98 | printf("Running default parameters.\n"); 99 | } 100 | 101 | x = (aa_float *)malloc(sizeof(aa_float) * n); 102 | xprev = (aa_float *)malloc(sizeof(aa_float) * n); 103 | Qhalf = (aa_float *)malloc(sizeof(aa_float) * n * n); 104 | Q = (aa_float *)malloc(sizeof(aa_float) * n * n); 105 | 106 | srand(seed); 107 | 108 | /* generate random data */ 109 | for (i = 0; i < n; i++) { 110 | x[i] = rand_float(); 111 | } 112 | for (i = 0; i < n * n; i++) { 113 | Qhalf[i] = rand_float(); 114 | } 115 | 116 | BLAS(gemm) 117 | ("Trans", "No", &n, &n, &n, &onef, Qhalf, &n, Qhalf, &n, &zerof, Q, &n); 118 | 119 | /* add small amount regularization */ 120 | for (i = 0; i < n; i++) { 121 | Q[i + i * n] += 1e-6; 122 | } 123 | 124 | AaWork *a = aa_init(n, memory, type1, regularization, relaxation, 125 | safeguard_tolerance, max_aa_norm, verbosity); 126 | for (i = 0; i < iters; i++) { 127 | if (i > 0) { 128 | _tic(&aa_timer); 129 | aa_apply(x, xprev, a); 130 | aa_time += _tocq(&aa_timer); 131 | } 132 | 133 | memcpy(xprev, x, sizeof(aa_float) * n); 134 | /* x = x - step_size * Q * xprev */ 135 | BLAS(gemv) 136 | ("No", &n, &n, &neg_step_size, Q, &n, xprev, &one, &onef, x, &one); 137 | 138 | _tic(&aa_timer); 139 | aa_safeguard(x, xprev, a); 140 | aa_time += _tocq(&aa_timer); 141 | 142 | err = BLAS(nrm2)(&n, x, &one); 143 | if (i % PRINT_INTERVAL == 0) { 144 | printf("Iter: %i, Err %.4e\n", i, err); 145 | } 146 | } 147 | printf("Iter: %i, Err %.4e\n", i, err); 148 | printf("AA time: %.4f seconds\n", aa_time / 1e3); 149 | aa_finish(a); 150 | free(Q); 151 | free(Qhalf); 152 | free(x); 153 | free(xprev); 154 | return 0; 155 | } 156 | -------------------------------------------------------------------------------- /test/run_tests.c: -------------------------------------------------------------------------------- 1 | /* Gradient descent (GD) on convex quadratic */ 2 | #include "aa.h" 3 | #include "aa_blas.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "minunit.h" 10 | 11 | /* default parameters */ 12 | #define SEED (1234) 13 | #define DIM (100) 14 | #define MEM (5) 15 | #define TYPE1_REGULARIZATION (1e-3) 16 | #define TYPE2_REGULARIZATION (0) 17 | #define SAFEGUARD_TOLERANCE (2.0) 18 | #define MAX_AA_NORM (1e10) 19 | #define ITERS (10000) 20 | #define STEPSIZE (0.01) 21 | #define PRINT_INTERVAL (500) 22 | #define VERBOSITY (1) 23 | 24 | int tests_run = 0; 25 | 26 | /* duplicate these with underscore prefix */ 27 | typedef struct _timer { 28 | struct timespec tic; 29 | struct timespec toc; 30 | } _timer; 31 | 32 | void _tic(_timer *t) { 33 | clock_gettime(CLOCK_MONOTONIC, &t->tic); 34 | } 35 | 36 | aa_float _tocq(_timer *t) { 37 | struct timespec temp; 38 | 39 | clock_gettime(CLOCK_MONOTONIC, &t->toc); 40 | 41 | if ((t->toc.tv_nsec - t->tic.tv_nsec) < 0) { 42 | temp.tv_sec = t->toc.tv_sec - t->tic.tv_sec - 1; 43 | temp.tv_nsec = 1e9 + t->toc.tv_nsec - t->tic.tv_nsec; 44 | } else { 45 | temp.tv_sec = t->toc.tv_sec - t->tic.tv_sec; 46 | temp.tv_nsec = t->toc.tv_nsec - t->tic.tv_nsec; 47 | } 48 | return (aa_float)temp.tv_sec * 1e3 + (aa_float)temp.tv_nsec / 1e6; 49 | } 50 | 51 | /* uniform random number in [-1,1] */ 52 | static aa_float rand_float(void) { 53 | return 2 * (((aa_float)rand()) / RAND_MAX) - 1; 54 | } 55 | 56 | static const char *gd(aa_int type1, aa_float relaxation) { 57 | aa_int n = DIM, iters = ITERS, memory = MEM, seed = SEED; 58 | aa_int i, one = 1; 59 | aa_int verbosity = VERBOSITY; 60 | aa_float neg_step_size = -STEPSIZE; 61 | aa_float safeguard_tolerance = SAFEGUARD_TOLERANCE; 62 | aa_float max_aa_norm = MAX_AA_NORM; 63 | aa_float err = 0; 64 | aa_float regularization; 65 | aa_float *x, *xprev, *Qhalf, *Q, zerof = 0.0, onef = 1.0; 66 | _timer aa_timer; 67 | aa_float aa_time = 0; 68 | x = (aa_float *)malloc(sizeof(aa_float) * n); 69 | xprev = (aa_float *)malloc(sizeof(aa_float) * n); 70 | Qhalf = (aa_float *)malloc(sizeof(aa_float) * n * n); 71 | Q = (aa_float *)malloc(sizeof(aa_float) * n * n); 72 | 73 | srand(seed); 74 | 75 | if (type1) { 76 | regularization = TYPE1_REGULARIZATION; 77 | } else { 78 | regularization = TYPE2_REGULARIZATION; 79 | } 80 | 81 | /* generate random data */ 82 | for (i = 0; i < n; i++) { 83 | x[i] = rand_float(); 84 | } 85 | for (i = 0; i < n * n; i++) { 86 | Qhalf[i] = rand_float(); 87 | } 88 | 89 | BLAS(gemm) 90 | ("Trans", "No", &n, &n, &n, &onef, Qhalf, &n, Qhalf, &n, &zerof, Q, &n); 91 | 92 | /* add small amount regularization */ 93 | for (i = 0; i < n; i++) { 94 | Q[i + i * n] += 1e-2; 95 | } 96 | 97 | AaWork *a = aa_init(n, memory, type1, regularization, relaxation, 98 | safeguard_tolerance, max_aa_norm, verbosity); 99 | for (i = 0; i < iters; i++) { 100 | if (i > 0) { 101 | _tic(&aa_timer); 102 | aa_apply(x, xprev, a); 103 | aa_time += _tocq(&aa_timer); 104 | } 105 | 106 | memcpy(xprev, x, sizeof(aa_float) * n); 107 | /* x = x - step_size * Q * xprev */ 108 | BLAS(gemv) 109 | ("No", &n, &n, &neg_step_size, Q, &n, xprev, &one, &onef, x, &one); 110 | 111 | _tic(&aa_timer); 112 | aa_safeguard(x, xprev, a); 113 | aa_time += _tocq(&aa_timer); 114 | 115 | err = BLAS(nrm2)(&n, x, &one); 116 | if (i % PRINT_INTERVAL == 0) { 117 | printf("Iter: %i, Err %.4e\n", i, err); 118 | } 119 | } 120 | printf("Iter: %i, Err %.4e\n", i, err); 121 | printf("AA time: %.4f seconds\n", aa_time / 1e3); 122 | aa_finish(a); 123 | free(Q); 124 | free(Qhalf); 125 | free(x); 126 | free(xprev); 127 | 128 | mu_assert_less("Failed to produce small error", err, 1e-6); 129 | 130 | return 0; 131 | } 132 | 133 | static const char *gd_type1_relax1(void) { 134 | return gd(1, 1.0); 135 | } 136 | 137 | static const char *gd_type1_relaxl1(void) { 138 | return gd(1, 0.98); 139 | } 140 | 141 | static const char *gd_type2_relax1(void) { 142 | return gd(0, 1.0); 143 | } 144 | 145 | static const char *gd_type2_relaxl1(void) { 146 | return gd(0, 0.98); 147 | } 148 | 149 | static const char *all_tests(void) { 150 | printf("type 1, relaxation 1.0\n"); 151 | mu_run_test(gd_type1_relax1); 152 | printf("type 1, relaxation < 1.0\n"); 153 | mu_run_test(gd_type1_relaxl1); 154 | printf("type 2, relaxation 1.0\n"); 155 | mu_run_test(gd_type2_relax1); 156 | printf("type 2, relaxation < 1.0\n"); 157 | mu_run_test(gd_type2_relaxl1); 158 | return 0; 159 | } 160 | 161 | int main(void) { 162 | const char *result = all_tests(); 163 | if (result != 0) { 164 | printf("%s\n", result); 165 | } else { 166 | printf("ALL TESTS PASSED\n"); 167 | } 168 | printf("Tests run: %d\n", tests_run); 169 | 170 | return result != 0; 171 | } 172 | -------------------------------------------------------------------------------- /src/aa.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Anderson acceleration. 3 | * 4 | * x: input iterate 5 | * x_prev: previous input iterate 6 | * f: f(x) output of map f applied to x 7 | * g: x - f (error) 8 | * g_prev: previous error 9 | * s: x - x_prev 10 | * y: g - g_prev 11 | * d: s - y = f - f_prev 12 | * 13 | * capital letters are the variables stacked columnwise 14 | * idx tracks current index where latest quantities written 15 | * idx cycles from left to right columns in matrix 16 | * 17 | * Type-I: 18 | * return f = f - (S - Y) * ( S'Y + r I)^{-1} ( S'g ) 19 | * 20 | * Type-II: 21 | * return f = f - (S - Y) * ( Y'Y + r I)^{-1} ( Y'g ) 22 | * 23 | */ 24 | 25 | #include "aa.h" 26 | #include "aa_blas.h" 27 | 28 | #define MAX(a, b) (((a) > (b)) ? (a) : (b)) 29 | #define MIN(a, b) (((a) < (b)) ? (a) : (b)) 30 | #define FILL_MEMORY_BEFORE_SOLVE (1) 31 | 32 | #if PROFILING > 0 33 | 34 | #define TIME_TIC \ 35 | timer __t; \ 36 | tic(&__t); 37 | #define TIME_TOC toc(__func__, &__t); 38 | 39 | #include 40 | typedef struct timer { 41 | struct timespec tic; 42 | struct timespec toc; 43 | } timer; 44 | 45 | void tic(timer *t) { 46 | clock_gettime(CLOCK_MONOTONIC, &t->tic); 47 | } 48 | 49 | aa_float tocq(timer *t) { 50 | struct timespec temp; 51 | 52 | clock_gettime(CLOCK_MONOTONIC, &t->toc); 53 | 54 | if ((t->toc.tv_nsec - t->tic.tv_nsec) < 0) { 55 | temp.tv_sec = t->toc.tv_sec - t->tic.tv_sec - 1; 56 | temp.tv_nsec = 1e9 + t->toc.tv_nsec - t->tic.tv_nsec; 57 | } else { 58 | temp.tv_sec = t->toc.tv_sec - t->tic.tv_sec; 59 | temp.tv_nsec = t->toc.tv_nsec - t->tic.tv_nsec; 60 | } 61 | return (aa_float)temp.tv_sec * 1e3 + (aa_float)temp.tv_nsec / 1e6; 62 | } 63 | 64 | aa_float toc(const char *str, timer *t) { 65 | aa_float time = tocq(t); 66 | printf("%s - time: %8.4f milli-seconds.\n", str, time); 67 | return time; 68 | } 69 | 70 | #else 71 | 72 | #define TIME_TIC 73 | #define TIME_TOC 74 | 75 | #endif 76 | 77 | /* This file uses Anderson acceleration to improve the convergence of 78 | * a fixed point mapping. 79 | * At each iteration we need to solve a (small) linear system, we 80 | * do this using LAPACK ?gesv. 81 | */ 82 | 83 | /* contains the necessary parameters to perform aa at each step */ 84 | struct ACCEL_WORK { 85 | aa_int type1; /* bool, if true type 1 aa otherwise type 2 */ 86 | aa_int mem; /* aa memory */ 87 | aa_int dim; /* variable dimension */ 88 | aa_int iter; /* current iteration */ 89 | aa_int verbosity; /* verbosity level, 0 is no printing */ 90 | aa_int success; /* was the last AA step successful or not */ 91 | 92 | aa_float relaxation; /* relaxation x and f, beta in some papers */ 93 | aa_float regularization; /* regularization */ 94 | aa_float safeguard_factor; /* safeguard tolerance factor */ 95 | aa_float max_weight_norm; /* maximum norm of AA weights */ 96 | 97 | aa_float *x; /* x input to map*/ 98 | aa_float *f; /* f(x) output of map */ 99 | aa_float *g; /* x - f(x) */ 100 | aa_float norm_g; /* ||x - f(x)|| */ 101 | 102 | /* from previous iteration */ 103 | aa_float *g_prev; /* x_prev - f(x_prev) */ 104 | 105 | aa_float *y; /* g - g_prev */ 106 | aa_float *s; /* x - x_prev */ 107 | aa_float *d; /* f - f_prev */ 108 | 109 | aa_float *Y; /* matrix of stacked y values */ 110 | aa_float *S; /* matrix of stacked s values */ 111 | aa_float *D; /* matrix of stacked d values = (S-Y) */ 112 | aa_float *M; /* S'Y or Y'Y depending on type of aa */ 113 | 114 | /* workspace variables */ 115 | aa_float *work; /* scratch space */ 116 | blas_int *ipiv; /* permutation variable, not used after solve */ 117 | 118 | aa_float *x_work; /* workspace (= x) for when relaxation != 1.0 */ 119 | }; 120 | 121 | /* add regularization dependent on Y and S matrices */ 122 | static aa_float compute_regularization(AaWork *a, aa_int len) { 123 | /* typically type-I does better with higher regularization than type-II */ 124 | TIME_TIC 125 | aa_float r, nrm_m; 126 | blas_int btotal = (blas_int)(len * len), one = 1; 127 | nrm_m = BLAS(nrm2)(&btotal, a->M, &one); 128 | r = a->regularization * nrm_m; 129 | if (a->verbosity > 2) { 130 | printf("iter: %i, norm: M %.2e, r: %.2e\n", (int)a->iter, nrm_m, r); 131 | } 132 | TIME_TOC 133 | return r; 134 | } 135 | 136 | /* sets a->M to S'Y or Y'Y depending on type of aa used */ 137 | /* M is len x len after this */ 138 | static void set_m(AaWork *a, aa_int len) { 139 | TIME_TIC 140 | aa_int i; 141 | blas_int bdim = (blas_int)(a->dim); 142 | blas_int blen = (blas_int)len; 143 | aa_float onef = 1.0, zerof = 0.0, r; 144 | /* if len < mem this only uses len cols */ 145 | BLAS(gemm) 146 | ("Trans", "No", &blen, &blen, &bdim, &onef, a->type1 ? a->S : a->Y, &bdim, 147 | a->Y, &bdim, &zerof, a->M, &blen); 148 | if (a->regularization > 0) { 149 | r = compute_regularization(a, len); 150 | for (i = 0; i < len; ++i) { 151 | a->M[i + len * i] += r; 152 | } 153 | } 154 | TIME_TOC 155 | } 156 | 157 | /* initialize accel params, in particular x_prev, f_prev, g_prev */ 158 | static void init_accel_params(const aa_float *x, const aa_float *f, AaWork *a) { 159 | TIME_TIC 160 | blas_int bdim = (blas_int)a->dim; 161 | aa_float neg_onef = -1.0; 162 | blas_int one = 1; 163 | /* x_prev = x */ 164 | memcpy(a->x, x, sizeof(aa_float) * a->dim); 165 | /* f_prev = f */ 166 | memcpy(a->f, f, sizeof(aa_float) * a->dim); 167 | /* g_prev = x */ 168 | memcpy(a->g_prev, x, sizeof(aa_float) * a->dim); 169 | /* g_prev = x_prev - f_prev */ 170 | BLAS(axpy)(&bdim, &neg_onef, f, &one, a->g_prev, &one); 171 | TIME_TOC 172 | } 173 | 174 | /* updates the workspace parameters for aa for this iteration */ 175 | static void update_accel_params(const aa_float *x, const aa_float *f, AaWork *a, 176 | aa_int len) { 177 | /* at the start a->x = x_prev and a->f = f_prev */ 178 | TIME_TIC 179 | aa_int idx = (a->iter - 1) % a->mem; 180 | blas_int one = 1; 181 | blas_int bdim = (blas_int)a->dim; 182 | aa_float neg_onef = -1.0; 183 | 184 | /* g = x */ 185 | memcpy(a->g, x, sizeof(aa_float) * a->dim); 186 | /* s = x */ 187 | memcpy(a->s, x, sizeof(aa_float) * a->dim); 188 | /* d = f */ 189 | memcpy(a->d, f, sizeof(aa_float) * a->dim); 190 | /* g = x - f */ 191 | BLAS(axpy)(&bdim, &neg_onef, f, &one, a->g, &one); 192 | /* s = x - x_prev */ 193 | BLAS(axpy)(&bdim, &neg_onef, a->x, &one, a->s, &one); 194 | /* d = f - f_prev */ 195 | BLAS(axpy)(&bdim, &neg_onef, a->f, &one, a->d, &one); 196 | 197 | /* g, s, d correct here */ 198 | 199 | /* y = g */ 200 | memcpy(a->y, a->g, sizeof(aa_float) * a->dim); 201 | /* y = g - g_prev */ 202 | BLAS(axpy)(&bdim, &neg_onef, a->g_prev, &one, a->y, &one); 203 | 204 | /* y correct here */ 205 | 206 | /* copy y into idx col of Y */ 207 | memcpy(&(a->Y[idx * a->dim]), a->y, sizeof(aa_float) * a->dim); 208 | /* copy s into idx col of S */ 209 | memcpy(&(a->S[idx * a->dim]), a->s, sizeof(aa_float) * a->dim); 210 | /* copy d into idx col of D */ 211 | memcpy(&(a->D[idx * a->dim]), a->d, sizeof(aa_float) * a->dim); 212 | 213 | /* Y, S, D correct here */ 214 | 215 | /* set a->f and a->x for next iter (x_prev and f_prev) */ 216 | memcpy(a->f, f, sizeof(aa_float) * a->dim); 217 | memcpy(a->x, x, sizeof(aa_float) * a->dim); 218 | 219 | /* workspace for when relaxation != 1.0 */ 220 | if (a->x_work) { 221 | memcpy(a->x_work, x, sizeof(aa_float) * a->dim); 222 | } 223 | 224 | /* x, f correct here */ 225 | 226 | memcpy(a->g_prev, a->g, sizeof(aa_float) * a->dim); 227 | /* g_prev set for next iter here */ 228 | 229 | /* compute ||g|| = ||f - x|| */ 230 | a->norm_g = BLAS(nrm2)(&bdim, a->g, &one); 231 | 232 | TIME_TOC 233 | } 234 | 235 | /* f = (1-relaxation) * \sum_i a_i x_i + relaxation * \sum_i a_i f_i */ 236 | static void relax(aa_float *f, AaWork *a, aa_int len) { 237 | TIME_TIC 238 | /* x_work = x initially */ 239 | blas_int bdim = (blas_int)(a->dim), one = 1, blen = (blas_int)len; 240 | aa_float onef = 1.0, neg_onef = -1.0; 241 | aa_float one_m_relaxation = 1. - a->relaxation; 242 | /* x_work = x - S * work */ 243 | BLAS(gemv) 244 | ("NoTrans", &bdim, &blen, &neg_onef, a->S, &bdim, a->work, &one, &onef, 245 | a->x_work, &one); 246 | /* f = relaxation * f */ 247 | BLAS(scal)(&bdim, &a->relaxation, f, &one); 248 | /* f += (1 - relaxation) * x_work */ 249 | BLAS(axpy)(&bdim, &one_m_relaxation, a->x_work, &one, f, &one); 250 | TIME_TOC 251 | } 252 | 253 | /* solves the system of equations to perform the AA update 254 | * at the end f contains the next iterate to be returned 255 | */ 256 | static aa_float solve(aa_float *f, AaWork *a, aa_int len) { 257 | TIME_TIC 258 | blas_int info = -1, bdim = (blas_int)(a->dim), one = 1, blen = (blas_int)len; 259 | aa_float onef = 1.0, zerof = 0.0, neg_onef = -1.0, aa_norm; 260 | 261 | /* work = S'g or Y'g */ 262 | BLAS(gemv) 263 | ("Trans", &bdim, &blen, &onef, a->type1 ? a->S : a->Y, &bdim, a->g, &one, 264 | &zerof, a->work, &one); 265 | 266 | /* work = M \ work, where update_accel_params has set M = S'Y or M = Y'Y */ 267 | BLAS(gesv)(&blen, &one, a->M, &blen, a->ipiv, a->work, &blen, &info); 268 | aa_norm = BLAS(nrm2)(&blen, a->work, &one); 269 | if (a->verbosity > 1) { 270 | printf("AA type %i, iter: %i, len %i, info: %i, aa_norm %.2e\n", 271 | a->type1 ? 1 : 2, (int)a->iter, (int)len, (int)info, aa_norm); 272 | } 273 | 274 | /* info < 0 input error, input > 0 matrix is singular */ 275 | if (info != 0 || aa_norm >= a->max_weight_norm) { 276 | if (a->verbosity > 0) { 277 | printf("Error in AA type %i, iter: %i, len %i, info: %i, aa_norm %.2e\n", 278 | a->type1 ? 1 : 2, (int)a->iter, (int)len, (int)info, aa_norm); 279 | } 280 | a->success = 0; 281 | /* reset aa for stability */ 282 | aa_reset(a); 283 | TIME_TOC 284 | return -aa_norm; 285 | } 286 | 287 | /* here work = gamma, ie, the correct AA shifted weights */ 288 | /* if solve was successful compute new point */ 289 | 290 | /* first set f -= D * work */ 291 | BLAS(gemv) 292 | ("NoTrans", &bdim, &blen, &neg_onef, a->D, &bdim, a->work, &one, &onef, f, 293 | &one); 294 | 295 | /* if relaxation is not 1 then need to incorporate */ 296 | if (a->relaxation != 1.0) { 297 | relax(f, a, len); 298 | } 299 | 300 | a->success = 1; /* this should be the only place we set success = 1 */ 301 | TIME_TOC 302 | return aa_norm; 303 | } 304 | 305 | /* 306 | * API functions below this line, see aa.h for descriptions. 307 | */ 308 | AaWork *aa_init(aa_int dim, aa_int mem, aa_int type1, aa_float regularization, 309 | aa_float relaxation, aa_float safeguard_factor, 310 | aa_float max_weight_norm, aa_int verbosity) { 311 | TIME_TIC 312 | AaWork *a = (AaWork *)calloc(1, sizeof(AaWork)); 313 | if (!a) { 314 | printf("Failed to allocate memory for AA.\n"); 315 | return (AaWork *)0; 316 | } 317 | a->type1 = type1; 318 | a->iter = 0; 319 | a->dim = dim; 320 | a->mem = MIN(mem, dim); /* for rank stability */ 321 | a->regularization = regularization; 322 | a->relaxation = relaxation; 323 | a->safeguard_factor = safeguard_factor; 324 | a->max_weight_norm = max_weight_norm; 325 | a->success = 0; 326 | a->verbosity = verbosity; 327 | if (a->mem <= 0) { 328 | return a; 329 | } 330 | 331 | a->x = (aa_float *)calloc(a->dim, sizeof(aa_float)); 332 | a->f = (aa_float *)calloc(a->dim, sizeof(aa_float)); 333 | a->g = (aa_float *)calloc(a->dim, sizeof(aa_float)); 334 | 335 | a->g_prev = (aa_float *)calloc(a->dim, sizeof(aa_float)); 336 | 337 | a->y = (aa_float *)calloc(a->dim, sizeof(aa_float)); 338 | a->s = (aa_float *)calloc(a->dim, sizeof(aa_float)); 339 | a->d = (aa_float *)calloc(a->dim, sizeof(aa_float)); 340 | 341 | a->Y = (aa_float *)calloc(a->dim * a->mem, sizeof(aa_float)); 342 | a->S = (aa_float *)calloc(a->dim * a->mem, sizeof(aa_float)); 343 | a->D = (aa_float *)calloc(a->dim * a->mem, sizeof(aa_float)); 344 | 345 | a->M = (aa_float *)calloc(a->mem * a->mem, sizeof(aa_float)); 346 | a->work = (aa_float *)calloc(MAX(a->mem, a->dim), sizeof(aa_float)); 347 | a->ipiv = (blas_int *)calloc(a->mem, sizeof(blas_int)); 348 | 349 | if (relaxation != 1.0) { 350 | a->x_work = (aa_float *)calloc(a->dim, sizeof(aa_float)); 351 | } else { 352 | a->x_work = 0; 353 | } 354 | TIME_TOC 355 | return a; 356 | } 357 | 358 | aa_float aa_apply(aa_float *f, const aa_float *x, AaWork *a) { 359 | TIME_TIC 360 | aa_float aa_norm = 0; 361 | aa_int len = MIN(a->iter, a->mem); 362 | a->success = 0; /* if we make an AA step we set this to 1 later */ 363 | if (a->mem <= 0) { 364 | TIME_TOC 365 | return aa_norm; /* 0 */ 366 | } 367 | if (a->iter == 0) { 368 | /* if first iteration then seed params for next iter */ 369 | init_accel_params(x, f, a); 370 | a->iter++; 371 | TIME_TOC 372 | return aa_norm; /* 0 */ 373 | } 374 | /* set various accel quantities */ 375 | update_accel_params(x, f, a, len); 376 | 377 | /* only perform solve steps when the memory is full */ 378 | if (!FILL_MEMORY_BEFORE_SOLVE || a->iter >= a->mem) { 379 | /* set M = S'Y or Y'Y depending on type of aa used */ 380 | set_m(a, len); 381 | /* solve linear system, new point overwrites f if successful */ 382 | aa_norm = solve(f, a, len); 383 | } 384 | a->iter++; 385 | TIME_TOC 386 | return aa_norm; 387 | } 388 | 389 | aa_int aa_safeguard(aa_float *f_new, aa_float *x_new, AaWork *a) { 390 | TIME_TIC 391 | blas_int bdim = (blas_int)a->dim; 392 | blas_int one = 1; 393 | aa_float neg_onef = -1.0; 394 | aa_float norm_diff; 395 | if (!a->success) { 396 | /* last AA update was not successful, no need for safeguarding */ 397 | TIME_TOC 398 | return 0; 399 | } 400 | 401 | /* reset success indicator in case safeguarding called multiple times */ 402 | a->success = 0; 403 | 404 | /* work = x_new */ 405 | memcpy(a->work, x_new, a->dim * sizeof(aa_float)); 406 | /* work = x_new - f_new */ 407 | BLAS(axpy)(&bdim, &neg_onef, f_new, &one, a->work, &one); 408 | /* norm_diff = || f_new - x_new || */ 409 | norm_diff = BLAS(nrm2)(&bdim, a->work, &one); 410 | /* g = f - x */ 411 | if (norm_diff > a->safeguard_factor * a->norm_g) { 412 | /* in this case we reject the AA step and reset */ 413 | memcpy(f_new, a->f, a->dim * sizeof(aa_float)); 414 | memcpy(x_new, a->x, a->dim * sizeof(aa_float)); 415 | if (a->verbosity > 0) { 416 | printf("AA rejection, iter: %i, norm_diff %.4e, prev_norm_diff %.4e\n", 417 | (int)a->iter, norm_diff, a->norm_g); 418 | } 419 | aa_reset(a); 420 | TIME_TOC 421 | return -1; 422 | } 423 | TIME_TOC 424 | return 0; 425 | } 426 | 427 | void aa_finish(AaWork *a) { 428 | if (a) { 429 | free(a->x); 430 | free(a->f); 431 | free(a->g); 432 | free(a->g_prev); 433 | free(a->y); 434 | free(a->s); 435 | free(a->d); 436 | free(a->Y); 437 | free(a->S); 438 | free(a->D); 439 | free(a->M); 440 | free(a->work); 441 | free(a->ipiv); 442 | if (a->x_work) { 443 | free(a->x_work); 444 | } 445 | free(a); 446 | } 447 | } 448 | 449 | void aa_reset(AaWork *a) { 450 | /* to reset we simply set a->iter = 0 */ 451 | if (a->verbosity > 0) { 452 | printf("AA reset.\n"); 453 | } 454 | a->iter = 0; 455 | } 456 | -------------------------------------------------------------------------------- /python/aa.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "import aa as aa\n", 11 | "import matplotlib.pyplot as plt\n", 12 | "import scipy.linalg as la\n", 13 | "plt.style.use('classic')" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 2, 19 | "metadata": {}, 20 | "outputs": [ 21 | { 22 | "name": "stderr", 23 | "output_type": "stream", 24 | "text": [ 25 | "/Users/bodonoghue/miniconda2/envs/python37/lib/python3.7/site-packages/ipykernel_launcher.py:9: FutureWarning: `rcond` parameter will change to the default of machine precision times ``max(M, N)`` where M and N are the input matrix dimensions.\n", 26 | "To use the future default and silence this warning we advise to pass `rcond=None`, to keep using the old, explicitly pass `rcond=-1`.\n", 27 | " if __name__ == '__main__':\n" 28 | ] 29 | }, 30 | { 31 | "data": { 32 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAGqCAYAAAAREa7qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAxOAAAMTgF/d4wjAAB3GklEQVR4nO3dd3xN9x/H8ddNIkPIECNiEysIEnsVRYzYe1NtzdrVotWiYhWpRinlZ68itGnNUnsrqhSxasROcpOIrJvfH99KmwpCbnLuTT7PxyOP8r3nnvN2eskn3/Mduj179iQihBBCCGGGLLQOIIQQQgjxpqSQEUIIIYTZkkJGCCGEEGZLChkhhBBCmC0pZIQQQghhtqSQEUIIIYTZkkJGCCGEEGZLChkhhBBCmC0rrQP818WLF/n666+xtLQkISGBQYMG4eHhoXUsIYQQQpggnamt7Pvo0SNsbW2xt7fn2rVrzJgxg/nz52sdSwghhBAmyOg9Mvv27WPz5s1cunSJqKgodu3ahaWlZbJjVq9ezaZNm4iMjMTb25tRo0aRK1cuAFxcXJKOs7a2xsJCnn4JIYQQImVGrxJiYmLw8vKia9euKb6+detWVqxYwdChQwkICCAqKoqJEyc+d1xcXByzZ8+mV69exo4ohBBCiEzC6D0yjRs3BuD06dMpvh4YGEj79u2pV68eAGPGjKF79+4EBwfj7u4OQEJCApMnT6ZBgwZUr17d2BGFEEIIkUlk6HOb2NhYrly5QuXKlZPa3NzccHV15fz58wAYDAb8/PwoX748vr6+GRlPCCGEEGYmQ2ct6fV6DAYDzs7OydqdnJwICwsD4Ndff+XgwYM8evSIQ4cOYW9vz5QpU5IdbzAYePToEXZ2duh0uoyKL4QQQog0SExMJDo6GhcXF6ONgc3QQiYx8dUTpBo2bEjDhg1fesyjR4/o1KmTsWIJIYQQIgOtX7+ePHnyGOVcGVrIODo6YmFhQWhoaLL2sLAwnJycUn0eOzs7AG7evImDg4MxI2ZJ48aNw8/PT+sYmYLcS+OQ+2g8ci+NR+5l2un1egoVKpT0fdwYMrSQsba2pkSJEpw+fRpvb28AQkJCuHv37mstevfscZKDg4MUMkZgbW0t99FI5F4ah9xH45F7aTxyL43HmMNCjF7I6PV67t+/z+3btwEIDg7G0tKSAgUKYGdnR5s2bQgICKBUqVLkz5+fb775Bk9Pz6QZS0IIIYQQqWX0QubQoUNMnz496fcDBgwAYM6cOVSqVInmzZsTGhqKv79/0oJ4o0ePfqNr3bl2G4eKUh2nlY+Pj9YRMg25l8Yh99F45F4aj9xL02RyWxSkRlRUFL6+vix4fxj9v/XXOo4QQgghUkGv1+Po6EhQUBD29vZGOadZr/9vffBnrSMIIYQQQkMmt/v166h64wqGBAMWlmZdjwkhhFl5+vQpsbGxWscQJsra2hpbW9sMu55ZFzJ5og3s+/5n6neRFYCFECIjPH36lGLFinH37l2towgT5erqyrVr1zKsmDHrQuZIARceLf5aChkhhMggsbGx3L17V9bxEil6tk5MbGysFDKpEdKoM002LiQ+Lh6rbGb9RxFCCLMi63gJU2HWg0vaTPoUu3gD6z+RlRaFEEKIrMisC5nsObOz3as2Lhu+0TqKEEIIITRg1oUMQKXJs6h/4x77NmzVOooQQgghMpj5FzJvVWVz+bJEjH1X6yhCCCGEyGBmX8gAlJu/mrdu3GHz7IVaRxFCCCFem06nY+nSpW/03vr169OnTx+j5jEnmaKQKV+zEhtq1sNt5khin8oiTUIIIURWkSkKGQDftRvJFR3Dis5dtY4ihBAii4iOjtY6QpaXaQqZ3Plz89vI6XTatonj2w9oHUcIIYQZWbNmDWXKlMHW1pYKFSqwZcuW5x7ZLF26FJ1Ox759+2jfvj0ODg40b94cgP3799OmTRsKFCiAnZ0dZcqUYcKECcTExCS7jsFgYMKECeTPn5/s2bPTsGFDLly4kOqc69evfy5nSq5du0b37t3JkycPNjY2VKpUicDAwGTHXLp0ibZt25I3b15sbW0pXLgwHTt2JD4+PumYBw8eMGjQIAoVKoSNjQ2FChWiZ8+ez/25tJSpVpHrOGEk6zZ9R6F3WhB96R529hm314MQQojkEhMhIiL9zp8zJ+h0aT/Prl276N69O61atWLWrFk8fPiQ4cOHExMTQ9GiRZ87vkePHvTo0YNBgwZhMBgAVThUq1aNfv364eDgwOXLl5k6dSpXrlxh1apVSe+dOHEiX3zxBSNHjqRJkyacOHGCli1bpirnL7/8QpcuXWjRogWzZs3iwYMHfPDBB8TFxSXLefPmTapXr07evHmZM2cOefLkYd26dbRv357NmzfTqlUrAFq0aIGzszPz588nd+7c3L59m59//jnpzxQWFkatWrUICwtj/PjxVKhQgfv377NlyxZiY2OxsbF5wztuXJmqkAF4++dfeVihEKubNqXf/l81TiOEEFlXRAQ4Oqbf+cPDwRiLC3/22Wd4eHgQGBiI7u/KqEKFCnh7e6d4fKdOnfDzS74Qa69evZL9vnbt2pQuXZr69eszd+5cXFxcCAsLY/bs2bz//vt8+eWXADRp0gRLS0s+/vjjVOUsU6YMW7ZswcJCPVApW7YsNWrUSHbc559/TmJiInv37sXFxQUAHx8fbt68yaeffkqrVq14+PAhwcHBbNmyJamwAejWrVvSr+fMmcPVq1c5deoUFStWTGrv2tW0hnBkmkdLz+R2y8udWUvofXAvm9aEaB1HCCGyrJw5VbGRXl85c6Y9Y0JCAsePH6d9+/ZJRQyAl5cXxYoVS/E9rVu3fq4tIiKCsWPH4u7ujq2tLdmyZaNevXoYDAYuX74MwNmzZ4mMjKRTp07J3tulS5dU5Tx27BgdOnRIKmIAqlev/lyv0bZt22jevDmOjo7Ex8cnffn4+HD27Fn0ej0uLi4UL16cjz/+mMWLF3PlypXnrrljxw6qVauWrIgxRZmukAFo0KsrT7PB9A9P8ddfWqcRQoisSadTPSbp9WWMx0oPHz4kLi6OvHnzPvdavnz5UnyPq6vrc219+/ZlyZIljBgxgu3bt3P8+HE2bdoEqB3DAUJCQlI874uuk1LOlI79b9v9+/dZvnw52bJlS/b14YcfAvDo0SN0Oh07d+7E29ubDz/8EHd3d0qUKMGCBQuSXbNAgQKvzKa1TPdoCUBnYcGdXNlo5nWO9u1bsG8f2NlpnUoIIYSpyZ07N9myZeP+/fvPvXbv3j1Kly79XLvuPxXU06dP2bx5M4sXL6Z3795J7eHh4cmOy58/f9J5y5Url+w6qc2Z0rH37t2jTJkySb93cXGhbt26fPTRRymey83NDYDixYuzYsUKEhMTOXv2LP7+/gwcOJAiRYrQrFmzpHEzpi5T9sgAPM5tj0+FS9jYQJ8+8PfYJSGEECKJpaUlVatWZePGjSQmJia1nzp1imvXrqXqHLGxsSQkJDw3+HXRokXJfu/p6Ym9vT3r169P1r527dpU59ywYUPSYFyAo0ePcv369WTHNm3alLNnz1KuXDmqVKny3Nd/c+p0OipWrMhXX30FwLlz5wA1fufYsWOcOXPmlfm0lCl7ZAAiXHOR89Z1AgOhWjWYOFF9CSGEEP82ceJEGjduTNu2bXn//fd5+PAhn3/+Oa6ursnGo7yIg4MDtWrVShqw6+joyJo1azhx4kSy45ycnBgxYgRTpkwhZ86cNGnShOPHj7N48eJU52zSpAlt2rShf//+PHjwgM8+++y5R12TJk2iWrVq1KtXjyFDhlC0aFFCQ0M5d+4cN27cYNGiRZw9e5Zhw4bRuXNn3N3dSUhIYOnSpVhZWdGwYUMARowYwerVq2nUqBGffPIJFSpU4OHDh2zZsoUFCxaQ0xiDlIwg0/bIxLrlw+rWHfLkgaAg8PeHNWu0TiWEEMLUNGrUiFWrVnHhwgXatm3L9OnTmTVrFq6urjimctrV6tWrqVChAv3796dHjx7odDrWrVv33HGff/4548aNY8WKFbRq1YodO3bw448/vlbOixcv0q5dO2bOnIm/v/9zj78KFy7MiRMnqFixIuPGjaNx48YMHDiQvXv30qBBA0CN8ylcuDCzZ8+mVatWdO3alTt37hAUFJQ0W8vJyYmDBw/Stm1bpk2bRtOmTRk1ahRWVlZYW1unKnNG0O3Zsyfx1YeZlqioKHx9fQkPD8fhBXPvgj7tQomNuyl7Xj333LYNOnRQRU39+hkYVgghMhG9Xo+jo+NL//3NDP766y9KlSrFokWL6Nmzp9ZxzMarPh/PXg8KCsLe3t4o18y0PTLWxUrgdF+f9PumTWHuXGjdGn77TcNgQgghTEp0dDQDBw5k48aN7N27lyVLltC4cWNcXV1p166d1vHEK2TaMTI53D3IExoDCQlgaQnAO+/Aw4eqqDlwAEqW1DikEEIIzVlaWhISEsLgwYN59OgRDg4OvP3228yYMcNovQYi/WTaQiZXSU8sEiHxzh10hQoltY8ZAw8eQJMmcPAg/D0LTQghRBZlbW3N5s2btY4h3lCmfbTklqsId3JAVPDzm3HNmKHGyfj4QGhoxmcTQgghhHFk2kImp3VObjlbEH759+de0+lg0SIoVgx8fSEyUoOAQgghhEizTFvI6HQ6HuXOTvSVP1N83coK1q0DGxto2RKePMnggEIIIYRIs0xbyADo8zmRcP3FKzPa2cGPP6pVf1u1gujoDAwnhBBCiDTL1IXMk4Ku2Aa/fIlpe3u1tszTp9C2rfqvEEIIIcxDpi5k7teqSIGz19V+7y+RMyf8/LM6rEMHiInJmHxCCCGESJtMXcjYlPbgjqs9bN/+ymMdHNTqv/fuQadOUswIIYQQ5iBTFzK1CtXi+5JxGDYHpup4R0fYsQPu3IF27eQxkxBCZDXvvvsuOp2OMWPGvPS4mzdvYmlpia2tLaGvsY7H559/jk6nIz4+Pq1Rxd8ydSFT1a0qO8rZYPgpCOLiUvUeZ2fYtUutLyOzmYQQIuuIjo7m+++/B2DlypUkJCS88Njly5djMBiIiYlh7dq1GRVRpCBTFzKWFpbkb9CKJ1aJsH9/qt/n6KieRsXFQbNmEBGRjiGFEEKYhMDAQPR6Pc2bNyckJISdO3e+8Njly5dTvnx5ChcuzLJlyzIwpfgvkytkdu7cyeDBgxk8eDAnT55M8/mal/ZlV2lr2Lr1td73bACwjY3aziAsLM1RhBBCmLBly5bh7OzM0qVLsbOze2GBcvjwYS5dukSvXr3o3r07R48e5c8/U16zTKQ/kypkIiMjWb16NbNnz8bPz4+AgICXdu2lRpMSTVhfKJzYn3987fdmzw4//AC5ckGjRvD4cZqiCCFElpKYmIg+Rp9uX4mJiUbLeufOHXbt2kXnzp3JkycPrVq1YvPmzYSnMOt12bJlWFhY0L17d3r37g2oHhqhDaNuGrlv3z42b97MpUuXiIqKYteuXVj+vfP0M6tXr2bTpk1ERkbi7e3NqFGjyJUrFwAXLlygfPny2NjYYGNjQ758+bh16xZFihR540xOtk5E1quB1cYjcOsWFCz4Wu+3tYXAQOjSBRo0gJ07IW/eN44jhBBZRkRsBI7THNPt/OEfh+Ng42CUc61YsQKDwUCvXr0A6NWrF+vWrWP9+vW89957ScfFxMSwbt06GjVqhNvfuw5XrVqVFStW8MUXX2BhYVL9A1mCUQuZmJgYvLy88Pb25rvvvnvu9a1bt7JixQrGjh2Lm5sbAQEBTJw4ka+++gqA8PBwcubMmXR8jhw5UqyGX1ctzxZcKvEnZbZvh379Xvv91tZqO4NevaBePVXM/GtDbSGEECnIaZ2T8I/T/m/4y85vLMuXL6dkyZLUrFkTAB8fH1xdXVm2bFmyQmbLli2EhYUlFTygip4PPviA3bt306hRI4DnZiVZWRn12634F6OWjo0bN6ZHjx6UK1cuxdcDAwNp37499erVw93dnTFjxnD27FmCg4MBcHBwIOJfI2sjIyNxcEh7td2kRBM2Fo7CsG3bG58jWzZYuRLeegtq14aLF9McSwghMjWdToeDjUO6fel0OqPkPH78OOfPn6ddu3aEhYURFhZGREQEbdq04eDBg0nfo0A9VsqePTsNGjRIOrZZs2ZYWVkljam5fv062bJlS/Z1/fp1o2QVz8uwPrDY2FiuXLlC5cqVk9rc3NxwdXXl/PnzAHh4eHDu3DliY2PR6/Xcu3ePQkbo+qjsWplfy9hi2LEd0jB339ISFiyAHj2gbl04dSrN0YQQQmjsWQEyffp0nJ2dk74WLFgA/DP+5d69e+zYsYMnT55QoECBpOPc3d2Jj49n06ZNRERE4ObmxvHjx5N9PXsMJYwvw/q69Ho9BoMBZ2fnZO1OTk6E/T0lKEeOHHTp0oXhw4ej0+kYPHjwc2Ns3oSlhSW56/oQsyIIq2PHoFatNz6XTgd+fmoAcIMGajDwW2+lOaIQQggNxMbGsnbtWmrVqsWUKVOee33EiBEsX76ciRMnsnLlSuLj41m4cCElS5ZMdtyZM2cYPnw4GzZsoG/fvlSpUiWj/ghZXoYVMqkdXe7j44OPj0+qjh03bhzW1tapel8j9yYcLLObJtu2pamQeWb0aLV4XosWsGaNWjxPCCGEeQkKCuLRo0cMGjSI+vXrP/d6//79GThwIHv37mX58uW4u7snGzPzTN26dZkxYwbLli2jb9++GZDcfGzfvp3tf28VFBsba/TzZ1gh4+joiIWFxXNLOYeFheHk5PRG5/Tz80v1GJomJZrwSYH3aPBzENkmTXqj6/1Xv37g5KRmNH37rXrkJIQQwnwsW7YMR0dH2rVrl+LrXbt2ZeTIkcyZM4ezZ8/i5+eX4nGWlpb06dOHqVOncu3aNYoVK5aesc3Kvzsa9Ho98+bNM+r5M2yMjLW1NSVKlOD06dNJbSEhIdy9excPD490v34hx0Lcr10Jy99Ow4MHRjtv+/awZQsMGgRff2200wohhMgAz2Yh2dnZpfi6o6MjT548YcuWLSQmJjJ27NgXnmvKlCkYDIaXFjGff/45iYmJMovJiIxayOj1eoKDg7l9+zYAwcHBBAcHEx0dDUCbNm3YuHEj+/fvJzg4mJkzZ+Lp6Ym7u7sxY7zQ2zW6caVQDjV/2ogaNVKn/Pxz9WXENZqEEEII8RJGLQkPHTrE9OnTk34/YMAAAObMmUOlSpVo3rw5oaGh+Pv7Jy2IN3r0aGNGeKn2ZduzrvAYRvy4GZtu3Yx67urVYd8+8PGBkBCYNw+k4BZCCCHSl1G/1TZt2pSmTZu+9Jju3bvTvXt3Y1421Yo5F+NyzVIkLtwKCQlqPrURlSsHhw9D06bqkdOaNWqbAyGEEEKkjyy3lnLp5r2JiY+BI0fS5fyFCsGBAxAaqh45PXqULpcRQgghBFmwkGlRthVB7gbitwSm2zWcnWHHDsifH+rUgRs30u1SQgghRJaW5QoZjzweHKjoTPTmDel6HVtbWL8eGjaEmjXhzJl0vZwQQgiRJWW5Qkan02HbrCV2127CtWvpei1LSwgIgCFD1GaTu3en6+WEEEKILCfLFTIADSu15WhxGwgKSvdr6XQwbhx89ZVa/Xft2nS/pBBCCJFlZM1CplhDNrnHEhW4PsOu2acPbNwI778Pc+Zk2GWFEEKITC1LFjL21vaEvl0bmwOHISIiw67btCn88gtMnQqjRoHBkGGXFkIIITKlLFnIANSu34ububMZfZXfV6laFQ4dUrtmd+oEfy96LIQQwgS8++676HQ6xowZ89Ljbt68iaWlJba2ts/tIfgyn3/+OTqdjvj4+KS2+vXrU6dOnTfOnNVl2UKmdZnWbCweQ+TGjB+04u6uFs67e1fNajLi1k9CCCHeUHR0NN9//z0AK1euJCEh4YXHLl++HIPBQExMDGtl8KOmsmwhkzt7bm7X98Ji61b4V2WcYdfPDbt2QeHCUKMGXLqU4RGEEEL8S2BgIHq9nubNmxMSEsLOl/TYL1++nPLly1O4cGGWLVuWgSnFf2XZQgagbMt+RBAL27Zpcn1bW7WNQceOaq2Z/fs1iSGEEAJYtmwZzs7OLF26FDs7uxcWKIcPH+bSpUv06tWL7t27c/ToUf78888MTiueydKFTOty7VhcIY7oRfM1y2BhAdOmgZ+fGgwsPZRCiEwjMRH0+vT7Skw0WtQ7d+6wa9cuOnfuTJ48eWjVqhWbN28mPDz8uWOXLVuGhYUF3bt3p3fv3oDqoRHayNKFTL4c+fijRVWst26Hhw81zdK/P2zYoKZnT5tm1L+fQgihjYgIcHRMvy8jzjpdsWIFBoOBXr16AdCrVy+ePn3K+vXJl+mIiYlh3bp1NGrUCDc3N0qXLk3VqlWT3i8yXpYuZABq1e/FH0XtYfVqraPQrBns2wdff60Km7g4rRMJIUQa5MwJ4eHp95Uzp9GiLl++nJIlS1KzZk0AfHx8cHV1fe7x0pYtWwgLC0sqeEAVPbdu3WL3v5Zvj4+PT/Yl0k+WL2Talm3LXI8I4pYs0joKAJUqwdGj6qtlS9V7KoQQZkmnAweH9PvS6YwS8/jx45w/f5527doRFhZGWFgYERERtGnThoMHDxIcHJx07LJly8iePTsNGjRIOrZZs2ZYWVklFT3Xr18nW7Zsyb6uX79ulKzieVm+kHHL6cZfTarDxUsms7NjwYJq4G9iItStC7duaZ1ICCEyr2cFyPTp03F2dk76WrBgAfDP+Jd79+6xY8cOnjx5QoECBZKOc3d3Jz4+nk2bNhEREYGbmxvHjx9P9uXm5qbZny+zs9I6gCnw9e7Kbu9r+Pzvf+Dvr3UcQP2wERQEgwZB9erw00+qt0YIIYTxxMbGsnbtWmrVqsWUKVOee33EiBEsX76ciRMnsnLlSuLj41m4cCElS5ZMdtyZM2cYPnw4GzZsoG/fvlSpUiWj/ghZnhQyQAePDvQsNYLGK1dgMWMGWFtrHQmAbNlg4UKYPl31zKxaBa1aaZ1KCCEyj6CgIB49esSgQYOoX7/+c6/379+fgQMHsnfvXpYvX467uzvvvffec8fVrVuXGTNmsGzZMvr27ZsBycUzWf7REqjHSwl1ahNpDWzfrnWcZHQ6+PhjWLoUunWDL7+UGU1CCGEsy5Ytw9HRkXbt2qX4eteuXbGzs2POnDmcPXuWd955J8XjLC0t6dOnD/v27ePatWvpGVn8h27Pnj1m920xKioKX19fwsPDcXBwMMo55x2bR+5Pp9K5oA8sXmyUcxrbiROqR6ZFC5g3z2Q6joQQWYher8fR0dGo//6KzONVn49nrwcFBWFvb2+Ua0qPzN/ae7RnQcG7JGzZDC/ZX0NLVarAsWOqoGnaFF5jnzIhhBAiU5JC5m+uOVyxrF2HmIQYtT21iXo2oylnTrVH0+XLWicSQgghtCOFzL909OzK7nLZYcsWraO8VI4csGkTtG6tipm9e7VOJIQQQmhDCpl/aVe2HYuLPCJ+w3ow8aWmLS1hxgz11bw5/O9/WicSQgghMp4UMv+Sxz4PsW83IEYfCi/Zvt2U9Oun1psZNQo++sjk6y8hhBDCqKSQ+Y92lbqyppaD2vDITDRoAEeOqMdN7dtDVJTWiYQQQoiMIQvi/Ue7su2oUm4I7/jvwCI4GNzdtY6UKqVKqWKmfXuoVw9++AEKFNA6lRAis9LLRnAiBVp8LqSQ+Q9nO2fq1erCqSMHqDJvHsyZo3WkVHNxgR07YOBAqFYNfvwRvLy0TiWEyEysra1xdXWlUKFCWkcRJsrV1RXrDFzoTAqZFAzwHsDHZdezc8kSdJMnq2lCZsLaGr77Tq0A/NZbsGIFtGmjdSohRGZha2vLtWvXiI2N1TqKMFHW1tbY2tpm2PWkkElBtQLVeFSpJA/3hpJn+XK1c6MZ0engww+hZEno2RPGjVPbHBhpx3shRBZna2ubod+ohHgZGeybAp1Ox8Cqg/i6OiQGBJjt5kZt2sC+fTB/viponj7VOpEQQghhXFLIvEDX8l2ZX/wx8fdCYPdureO8scqV1bYGV66oR00hIVonEkIIIYxHCpkXyGmTk47ePdlVrxAsWqR1nDRxdYU9e6BMGahaFU6e1DqREEIIYRxSyLxEf+/+fFT0EomBgfDggdZx0sTWFpYuhWHDVM/M999rnUgIIYRIOylkXqKia0Wyl6vEnQpFYflyreOk2bNBwGvXqhWBP/9cVgIWQghh3kyqkLl48SJDhgxh2LBhDBkyhPPnz2sdid4Ve/Otl0HNaTbTQb//5eurNvhevhw6d5aVgIUQQpgvkypkcufOzfTp0/nqq68YNWoUX5vANgGdy3fma9cbGK5fgwsXtI5jNOXLq0HA9+9D3bpw86bWiYQQQojXl+ZCZt++fYwcORJfX18aNGhAQkLCc8esXr2aDh060LRpU8aPH8/jx49TPJeLiwv29vaAWlDHwkL7OiuXXS7eLt+KP72KwObNWscxqty51d6Y3t5qEPCRI1onEkIIIV5PmiuFmJgYvLy86Nq1a4qvb926lRUrVjB06FACAgKIiopi4sSJLz1nXFwcs2fPplevXmmNZxS9K/ZmcZGHJGayQgbUSsALF6pF895+W60ELIQQQpiLNK/s27hxYwBOnz6d4uuBgYG0b9+eevXqATBmzBi6d+9OcHAw7u7ubNmyhT179uDg4MCkSZNISEhg8uTJNGjQgOrVq6c1nlE0LtGYwcWf8uX3v8GtW1CwoNaRjEqng6FD1fTszp3hjz/Azw9MoENMCCGEeKl0/VYVGxvLlStXqFy5clKbm5sbrq6uSQN5W7dujb+/P5MmTcJgMODn50f58uXx9fVNz2ivxdbKlqqeTblRvmCme7z0b02aqMdLmzapVYEjIrROJIQQQrxcuhYyer0eg8GAs7NzsnYnJyfCwsKeO/7XX3/l4MGDHDp0iOHDhzN+/Pj0jPdaWpZqyfdlDJl+AZbSpeHoUYiOhlq14No1rRMJIYQQL5aum0YmvuZ05YYNG9KwYcNUHz9u3LikrcJ9fHzw8fF5reu9jhYlW1CpYD9Gb7iD7vZtKFAg3a6lNWdn2LoVRo6EatVg40b4+8mgEEII8Vq2b9/O9u3bAdJl1/R0LWQcHR2xsLAgNDQ0WXtYWBhOTk5pPr+fnx8ODg5pPk9q5LHPQ9GyNbjt9ZiC338Pw4dnyHW1YmUFc+dCuXLQrJn6db9+WqcSQghhbv7d0aDX65k3b55Rz5+uj5asra0pUaJEsoHAISEh3L17Fw8Pj/S8dLpoX7Y96zwS1dK4WUT//vDTTzBmjNreID5e60RCCCHEP9JcyOj1eoKDg7l9+zYAwcHBBAcHEx0dDUCbNm3YuHEj+/fvJzg4mJkzZ+Lp6Ym7u3taL53hulfozkzXKySeOgVXr2odJ8PUrw/Hj8Mvv0DTpvDokdaJhBBCCCXNj5YOHTrE9OnTk34/YMAAAObMmUOlSpVo3rw5oaGh+Pv7ExkZibe3N6NHj07rZTWRL0c+qno25WKNK5RZsgS++ELrSBmmeHE4fBh69lTjZn74QT12EkIIIbSk27Nnj9ltIBQVFYWvry/h4eEZNkbmmQ3nN/Dz3A9Y/IMO3V9/qcEkWYjBABMnwpw5avG81q21TiSEEMJc6PV6HB0dCQoKSlrJP61kybPX1LJUS34sGkOMzqAGj2QxFhaqkFmyBHr0UJ1SmWQvTSGEEGZICpnXZGNlQx+vfmypkxu+/VbrOJrp0AEOHFCbgssO2kIIIbQihcwbGFh1IGOKXsLw6x44c0brOJqpWFENAr5/H2rXhhs3tE4khBAiq5FC5g0Udy5O+YqNOd7UM0sN+E1JnjxqB+1ataBKFdi3T+tEQgghshIpZN7QkKpDGFjuGolBQXDunNZxNJUtG3zzjarpmjWDBQu0TiSEECKrkELmDfm4+6DP48CVlnVg5kyt45iE/v3V1gaffgoDB0I6rEQthBBCJCOFzBuy0FkwsMpAPq8UCuvWwb17WkcyCfXqwYkTas2Zxo3hwQOtEwkhhMjMpJBJg76V+7Ip8TwRtbxh4UKt45iMIkXg4EHIlw+qVs3S46GFEEKkMylk0iCXXS66VejGsnqOMH++PEv5F3t71VH13ntQpw5s2KB1IiGEEJmRFDJpNKjqIMZa7SXBPrt8t/4PnQ7Gj4dVq+Cdd2DCBLUysBBCCGEsUsikkVd+L8rn9+RAq0owd67WcUxSq1ZqzMzq1dCuHUREaJ1ICCFEZiGFjBEMrjqYUfnPknj+PBw9qnUck1SuHBw7plYArlkTrlzROpEQQojMQAoZI+jg0YEbiaHcat8IvvpK6zgmK1cuNT27cWO1g/Yvv2idSAghhLmTQsYIbK1sebfyu/h5P4FNm+DSJa0jmSwrK7Vz9pdfqkdOc+fKppNCCCHenBQyRjKgygCWhO0hskt7mDRJ6zgmr29f2LULpk6Fd9+FmBitEwkhhDBHUsgYSRGnIviU8OFbHxfYuBHOn9c6ksmrWVNtOnn2LDRsCHfvap1ICCGEuZFCxog+qPYB026tIa5bF/X8RLxSwYJqo8lixdSmkydOaJ1ICCGEOZFCxogaFW9EEccirK/nouYah4drHcks2NnBihUwfDjUr69+LYQQQqSGFDJGpNPpGF93PGMercFQrhysXKl1JLOh08Ho0eqp3LBhMHIkxMdrnUoIIYSpk0LGyFqXaY2zrTP7m5aFb7+VKTmvycdHrTezYwc0bQqPHmmdSAghhCmTQsbILHQWDKs+jHF5zsCNG+q7sngt7u5qJWAHB7Xp5NmzWicSQghhqqSQSQddynfhbOQVHrZoAMuXax3HLOXMqbau6tsXateWbayEEEKkTAqZdJDTJifdyndjiWcCrF0ri6S8IQsL+PRTtelkv35qA0rZdFIIIcS/SSGTTt73fp/J8bsxODjAzz9rHcestWoFR47A99+rX8tkMCGEEM9IIZNOvN28KZW3DL81Lg9Ll2odx+yVLauGGxkMUL06XLyodSIhhBCmQAqZdPS+1/tMKH4Dtm1TA39Fmjg5wY8/Qtu2qpgJCtI6kRBCCK1JIZOOulboyl7DVUIb1YZvvtE6TqZgaan2Z1q4ELp2hSlTZIa7EEJkZVLIpCMHGwe6lu/K/+rkgEWL4MkTrSNlGp06wYED6rZ26gSRkVonEkIIoQUpZNLZ+97v81nibhIKuKkZTMJoKlZUezM9egS1asHVq1onEkIIkdGkkElnVdyqUNKlFMd8ysOyZVrHyXRy54bt29Xu2VWrwq5dWicSQgiRkaSQSWc6nU71yuT/Uy1Xe/261pEynWzZwN8fZs2C1q1h9mwZNyOEEFmFFDIZoFuFbhyMuUz4WzVkI8l01KcP7NmjCplevSA6WutEQggh0psUMhnAwcaBTuU6EVg1h9qyQLoL0k21amrczNWrULcu3LypdSIhhBDpSQqZDNKtfDc+dzhF4t27cOqU1nEyNVdX2L0bvLygShXYv1/rREIIIdKLFDIZpEGxBkRbJXKvQTW11r5IVzY2aq2ZiROhaVNYsEDrREIIIdKDSRYy4eHhtGzZkp9++knrKEZjZWFFJ49ObCynU4WMPF7KEAMGqIWVP/sM+veH2FitEwkhhDAmkyxkli1bhqenp9YxjK5rha5MyX6CxPv34fRpreNkGXXrqnEzJ09CgwZw967WiYQQQhhLmguZffv2MXLkSHx9fWnQoAEJCQnPHbN69Wo6dOhA06ZNGT9+PI8fP37h+a5fv050dDQlS5ZMazSTU7NgTaxzOHL7LS95vJTBChVSY2WKF1fjZo4f1zqREEIIY0hzIRMTE4OXlxddu3ZN8fWtW7eyYsUKhg4dSkBAAFFRUUycOPGF51u8eDF9+vRJayyTpNPp6FK+C2sqoHbEluccGcrOTk0aGzkS6teX9QmFECIzsErrCRo3bgzA6Rc8KgkMDKR9+/bUq1cPgDFjxtC9e3eCg4Nxd3dny5Yt7NmzBwcHB3x9fSlYsCD58uVLayyT1bV8V2ofmcuoHAWxWLMGevfWOlKWotOpQsbTEzp3ht9+gy+/BKs0/00QQgihhXQdIxMbG8uVK1eoXLlyUpubmxuurq6cP38egNatW+Pv78+kSZO4dOkSFy5cYMyYMezcuZMNGzZwKpNNVfbM50lh56Kc7PqW+g4qg3410aiRery0ezf4+MDDh1onEkII8SbStZDR6/UYDAacnZ2TtTs5OREWFvbc8T169MDf358ZM2bQuHFjOnTogJeXV3pGzHA6nY5uFbrhV+wm3LunNgoSmiheHA4dgly51D5NZ85onUgIIcTrStcO9cQ09DakZpzMuHHjsLa2BsDHxwcfH583vl5G6lupL5P3TebROwNw+fJLtdCJ0ESOHLB+Pfj5QZ068N136pGTEEII49i+fTvb//6hPTYdxoamayHj6OiIhYUFoaGhydrDwsJwcnJK8/n9/PxwcHBI83kyWgGHAnT06MjMmCimzT2kBmr86/GbyFg6HYwfDxUrQo8eamb8F1+ApaXWyYQQwvz9u6NBr9czb948o54/XR8tWVtbU6JEiWQDgUNCQrh79y4eHh7peWmTN6LGCAKurSWmVze1bbPQnK8vHDkCgYHQsiWk8PRTCCGEiUlzIaPX6wkODub27dsABAcHExwcTPTfWw+3adOGjRs3sn//foKDg5k5cyaenp64u7un9dJmzdvNm8r5K7O8gQts2KB2ORSaK1MGjh5VvTFVq8Iff2idSAghxMuk+dHSoUOHmD59etLvBwwYAMCcOXOoVKkSzZs3JzQ0FH9/fyIjI/H29mb06NFpvWymMKLGCIZvG06/Th2xmDwZ/vc/rSMJwNERtmyBzz+HmjXVkj/t2mmdSgghREp0e/bsMbv5v1FRUfj6+hIeHm6WY2SeSTAk4P61O/NKjaB52zFw9iyUKqV1LPEvmzdDr14wbJjagNLCJDf1EEII86DX63F0dCQoKAh7e3ujnFP+WdaQpYUlQ6sNZfKdNWqU6aRJWkcS/9GmjRo3s26djJsRQghTJIWMxvp59eOP+3/wW78WaqzM3wsFCtPh4QHHjqnZTdWqyf8iIYQwJVLIaMzBxoF3Kr/D9NvroG9f9fxCmBwnJ/jhB7XGTM2a6pGTEEII7UkhYwKGVh9K4J+B3Pmgr/puefas1pFECiws4NmY7F69YMIEMBi0TiWEEFmbFDImoLhzcVqUbIH/7Q3wzjvwr1lgwvS0a6fGzaxZA61bQ3i41omEECLrkkLGRIyuNZr5J+YTNrifGitz7ZrWkcRLeHioTScNBjVu5sIFrRMJIUTWJIWMiahVqBbVC1Rnzr0t6kd+We3X5D0bN9OxI9SoodaeEUIIkbGkkDEhn9b7lK+OfkXE8EGwZAk8eKB1JPEKlpZqX6YlS6BnT/jsMxk3I4QQGUkKGRPyVtG38MznSUDsAahXD77+WutIIpXat4dDh2DVKrX2jIybEUKIjCGFjIkZU3sMc4/NJXb0CAgIgMhIrSOJVCpfXo2biYuD6tXhzz+1TiSEEJmfFDImpnnJ5jjbOrPC+Sa4u8N332kdSbwGZ2cIClLDnKpXV2NohBBCpB8pZEyMhc6C0bVGM+vIbBLHjIHZsyE2VutY4jVYWoKfHyxeDN27q80nZdyMEEKkDylkTFD3Ct15+OQhuys6gJ2dWrBEmJ0OHeDwYVi5Etq2Bb1e60RCCJH5SCFjgmysbOhXuR/f/PYtfPghzJghP9KbqWfjZmJi1KOmixe1TiSEEJmLFDImqn+V/vx48UdutW4AoaHw889aRxJvyNkZfvpJzWaqXh1+/FHrREIIkXlIIWOiijoVxcfdh29/XwqDB8NXX2kdSaSBpSVMnQqLFkG3bjBpknSyCSGEMUghY8JG1hhJwPEAwnt1hgMH4I8/tI4k0qhjR7XezLJlamaTjJsRQoi0kULGhDUo1oBKrpWYc2Wl+jFeemUyhQoV1LiZ6GgZNyOEEGklhYyJm1h/InOOzCH8/d5q+ov8CJ8p5Mqlhj21bq2KmaAgrRMJIYR5kkLGxNUrUg+v/F7Miz0AZcvC999rHUkYiaUlTJsG334LXbvC5MkybkYIIV6XFDJmYESNEXxz/BsSevZQgytEptK5Mxw8CP/7n9qzKSJC60RCCGE+pJAxAy1KtsDa0pofvXLAkSNw9arWkYSReXrCiRMQFaUeNV26pHUiIYQwD1LImAFLC0s+qPYBMy7/D5o1g+XLtY4k0sGzcTO+vlCtmlp7RgghxMtJIWMm+lTqw8mQk9xu20gVMjKYIlOyslILOS9YAF26wBdfyP9qIYR4GSlkzISznTPNSzZnids9CA9XgypEptWli1o6aPFitWeTjJsRQoiUSSFjRrqV78bKS9+T2KWLDPrNAipWVONmIiKgRg24fFnrREIIYXqkkDEjvqV8uRNxh4stqsP69fDkidaRRDpzcYGtW6F5czVuRrbcEkKI5KSQMSN22exoW6YtiyxOQ8GCEBiodSSRAaysYOZM+OYbNVXbzw8SE7VOJYQQpkEKGTPTq2IvVp5bRUKvnrB0qdZxRAbq2hX271cbT8q4GSGEUKSQMTMNijbAxtKGX2q7wd698NdfWkcSGahSJbVPU3i4GjcTHKx1IiGE0JYUMmbG0sKS3hV7s+DOFmjSRAb9ZkG5c8O2bWpJoapV1RgaIYTIqqSQMUN9KvXhp8s/Ed61nVrXPiFB60gig1lZwZdfwrx50LEjTJ0q42aEEFmTFDJmqESuEtQoWINlRUIhJkamsmRh3bqpcTPffqsKGhk3I4TIaqSQMVN9K/Vl8bnlJA4YAF99pXUcoaHKldV6M2Fhst6MECLrkULGTHXw6MCVx1f4vW0ttcrvuXNaRxIaejZupkULNW4mKEjrREIIkTFMrpAJDg7mo48+YuTIkfj5+Wkdx2TlsM5Bp3KdWPTXZvV84csvtY4kNPZsn6Zvv1VTtSdOlH2ahBCZn5XWAf4tPj6eefPmMXnyZHLkyKF1HJP3TuV3aLWmFTNG78WuclUYNw5KldI6ltBY587g4QFt2sDJk7BiBTg6ap1KCCHSR5p7ZPbt28fIkSPx9fWlQYMGJKQwg2b16tV06NCBpk2bMn78eB4/fpziuf744w+yZ8/OtGnTGD58OAcOHEhrvEytdqHaFHIsxMonR6BnT/j8c60jCRNRoYIaNxMfr7Y2OH9e60RCCJE+0lzIxMTE4OXlRdeuXVN8fevWraxYsYKhQ4cSEBBAVFQUEydOTPHYR48ecfnyZT766CMmT57MwoULCQ8PT2vETEun0zG02lDmHptL4iefqC0Lfv9d61jCRDg7w48/qtlMNWrApk1aJxJCCONLcyHTuHFjevToQbly5VJ8PTAwkPbt21OvXj3c3d0ZM2YMZ8+eJfjvJUm3bNnC8OHDmTBhAjly5KBs2bLkzJmTnDlzUqJECW7dupXWiJlatwrduBNxhz2GK/Dee/DZZ1pHEibE0hK++EKtm9i3r3r6KMsOCSEyk3Qd7BsbG8uVK1eoXLlyUpubmxuurq6c/7uvu3Xr1vj7+zNp0iQ8PDy4c+cOcXFxxMXFcePGDfLnz5+eEc2eXTY7+nv3Z9bhWTB2LGzfrgZGCPEvbdvCkSOqV6ZFC3jB010hhDA76VrI6PV6DAYDzs7OydqdnJwICwt77vgcOXLQtWtXRowYwdChQ2nfvj25cuVKz4iZwrDqw9hzbQ+ndfdg8GD45BOtIwkTVLYsHDsGtrZqivbZs1onEkKItEvXWUuJb7BmesOGDWnYsGGqjh03bhzW1tYA+Pj44OPj89rXywzy5cjHO5XfYdqBaawdEwDFisGhQ1CrltbRhIlxcFC9MlOmQO3aaiftLl20TiWEyMy2b9/O9u3bAfWkxtjStZBxdHTEwsKC0NDQZO1hYWE4OTml+fx+fn44ODik+TyZwYe1PqR0QGkuNZhEqWHD4NNP4ZdftI4lTJCFhfp4eHlB9+5qN+3p09U6NEIIYWz/7mjQ6/XMmzfPqOdP10dL1tbWlChRgtOnTye1hYSEcPfuXTw8PNLz0llOEacidCnfhRkHZ8CoUWqczK+/ah1LmLAWLVQRs307+PjAgwdaJxJCiNeX5kJGr9cTHBzM7du3AbUyb3BwMNHR0QC0adOGjRs3sn//foKDg5k5cyaenp64u7un9dLiPz6q/RErz67klmUUjBypfuyWLZHFS5QsqQYB58oFVarIOHEhhPlJc2fyoUOHmD59etLvBwwYAMCcOXOoVKkSzZs3JzQ0FH9/fyIjI/H29mb06NFpvaxIQdk8ZfEt5cusQ7OYM3yi2kxy505o0kTraMKE5cgB69fDzJnw1lswbx707q11KiGESB3dnj17zO5H9qioKHx9fQkPD5cxMv9x8s5J6i2tx/Vh18nz9WK1SN6RI6DTaR1NmIGdO9Xg327dYPZsyJZN60RCiMxEr9fj6OhIUFAQ9vb2RjmnyW0aKdLG282bOoXrMPfoXPjgA7h2TQ2CECIVGjdWWxscOAANG8Ldu1onEkKIl5NCJhMaV2ccAccD0FslwNChakqKEKlUrBgcPAhFioC3t+rQE0IIUyWFTCZUr0g9PPJ4MP/4fLVA3okTaiU0IVIpe3a1a/aYMfD227BwodaJhBAiZVLIZEI6nY5P6n7CrMOziLK3hvffl14Z8dp0Ohg2DH76SS0W/f77EBOjdSohhEhOCplMqql7U4o6FWX+ifkwYoT6bnTxotaxhBmqX19Ny/7tNzWr6e+VFoQQwiRIIZNJ6XQ6PnvrM2YemsmTfLnUNJSZM7WOJcxUoUKwfz94eKhxM/v3a51ICCEUKWQyseYlm+OW042lp5fChx/CqlVw547WsYSZsrWFxYvhs8+gaVP4+mtZb1EIoT0pZDIxnU7HqJqjmHNkDoYypdV3n7lztY4lzJhOBwMHwo4d4OcHffrA34t4CyGEJqSQyeQ6levEk7gnBF0KUiM3v/tOvvOINKtdW42bCQ6GOnXgxg2tEwkhsiopZDI5a0trPqj2AbMPz1YjNV1dYe1arWOJTMDNDfbsgerV1biZ3bu1TiSEyIqkkMkC+nv358SdE5wMOQVDhsjgBmE01tbwzTcwYwa0bAmzZslHSwiRsaSQyQKc7ZzpW6kvs4/Mhh494MoVOHpU61giE3nnHdU74++vJshFRWmdSAiRVUghk0UMqzGMDec3cDMhVBUzixZpHUlkMtWqqXEzISFQs6aql4UQIr1JIZNFuOdyp3nJ5gQcC4D33lPjZPR6rWOJTCZvXrWDdsOGULUqbNumdSIhRGYnhUwWMqrmKL49+S0RZUuolc1Wr9Y6ksiEsmVTj5jmzoUOHdQ0bRk3I4RIL1LIZCG1C9WmlEsp/nf6f2rjHHm8JNJRjx5qBeCFC6F9e4iI0DqRECIzkkImC9HpdIysORL/I/4kdOqo9l46eVLrWCITq1xZfcQiItQ0bdnuSwhhbFLIZDEdPDoQmxDLz3f3Q9eu0isj0p2Lixor06qVGhD8ww9aJxJCZCZSyGQxVhZWvOf1Ht+e/FY9Xlq1CiIjtY4lMjlLS5g2TdXN3bur/ZoMBq1TCSEyAylksqB+Xv3YcWUHf5XMCyVKwLp1WkcSWUSnTnD4sBpn3rIlhIZqnUgIYe6kkMmCCjoUpFnJZnz322IZ9CsyXPnycPy46qWpUgXOnNE6kRDCnEkhk0W95/UeS08vJbFbNzh7Vn0JkUGcnGDzZrV7du3a6gmnEEK8CSlksqgmJZoQGRvJkYgLqr9femVEBrOwgE8/he+/hw8+gKFDIS5O61RCCHMjhUwWZW1pTdsybVn3xzr1eGnFCtkgR2iiWTP1qGnvXrUicEiI1omEEOZECpksrHP5znx//nsMNapD6dJqV2whNFCihBoEXLQoeHvDoUNaJxJCmAspZLKwBkUbEBMfw8Gbh9Q68tOnQ1iY1rFEFpU9OyxfDuPGQePGMG+ebG0ghHg1KWSysGyW2ehcrjOLf1sMb78NXl7w5ZdaxxJZmE4HQ4bA9u3wxRdqMHB0tNaphBCmTAqZLG5ItSGsPbeWe5H3YMoUtdvf/ftaxxJZXJ06cOoUXLkCtWrBtWtaJxJCmCopZLK4snnK8lbRt1h4ciHUqKF6ZqZO1TqWEOTPD7t3q6KmShXVSyOEEP8lhYxgaLWhfHPiG2LiY2DyZFiwAP76S+tYQmBtrcag+/urHbSnTJGtDYQQyUkhI2hWshm57HKx5Lcl4OkJbdqogb9CmIiePWH/fvjuO2jXDsLDtU4khDAVUsgILHQWTKw/kSn7p/A0/imMHQtLlsDdu1pHEyJJ5cpw8iQ8fap20T5/XutEQghTIIWMAKBd2Xa4ZHdRY2U8PaFRI9WfL4QJyZULfvpJLUZdo4ZaFVgIkbVJISOAf3plph2YpsbKjBsH33wj68oIk2NpqYZyrVgB774LH34I8fFapxJCaEUKGZGkVelWONs5s/zMcqhZU60rM2+e1rGESFHr1mprg59/hiZN4MEDrRMJIbRgcoVMQEAAgwYNYuDAgQQGBmodJ0ux0FnwUe2PmHFoBgmGBDVWxt9f9mASJqtUKTh6FHLnVlsbHD+udSIhREYzqUImODiYy5cv88033xAQEMDGjRuJkm+iGapr+a7EJcSx8cJG9WNu4cJqqogQJipHDli3DoYNg/r15eMqRFaT5kJm3759jBw5El9fXxo0aEBCQsJzx6xevZoOHTrQtGlTxo8fz+PHj1M8V548ebC0tCQuLo7o6Gisra3Jli1bWiOK15DNMhvDqg/jq6NfqfXix49XU7FlnXhhwnQ6GDUKfvxRdSS+/z7ExGidSgiREdJcyMTExODl5UXXrl1TfH3r1q2sWLGCoUOHEhAQQFRUFBMnTkzxWAcHBwoXLkyPHj3o1asXLVu2xNraOq0RxWvqW7kvZ+6e4cSdE9C2rVpi9ZtvtI4lxCs1bKi2Njh9GurWhZs3tU4khEhvaS5kGjduTI8ePShXrlyKrwcGBtK+fXvq1auHu7s7Y8aM4ezZswQHBwOwZcsWhg8fzoQJEzhx4gR37txh9erVrF69mq1btxISEpLWiOI1Odk60btib74+9rX6UfeLL2DaNIiI0DqaEK9UqBDs2weVKqnx6rt3a51ICJGe0nWMTGxsLFeuXKFy5cpJbW5ubri6unL+79WsWrdujb+/P5MmTSIxMZGcOXNiaWmJjY0NNjY2PHnyJD0jihf4oPoHrDu3jpCIEGjaVI2qlF4ZYSZsbWHhQrVtWMuWalP3xEStUwkh0kO6FjJ6vR6DwYCzs3OydicnJ8JSWJ+kSpUq2NraMmTIEAYNGkSFChUoUaJEekYUL1Amdxmaujdl9uHZqlfmo49g7lyIjdU6mhCp9u67sGcPfPUVdO4snYpCZEZW6XnyxNf8EcjCwoIPP/ww1cePGzcuaQyNj48PPj4+r3U98XLj6o6j4bKGfFznY1x8fdXKY2vXQq9eWkcTItWqVVNbG3TpolYD3rQJSpfWOpUQWcf27dvZ/vf29bHp8MNwuhYyjo6OWFhYEBoamqw9LCwMJyenNJ/fz88PBweHNJ9HpKxagWrULFSTuUfnMrHBRBg5EmbNUjv46XRaxxMi1fLmhR071ILV1arBsmVqb1QhRPr7d0eDXq9nnpEXWk3XR0vW1taUKFGC06dPJ7WFhIRw9+5dPDw80vPSwkjG1RnH3GNziYiJUD0xt26pkZRCmBkrK5gxQ60z06sXfPIJpLBahBDCzKS5kNHr9QQHB3P79m1ALWoXHBxM9N/rjrRp04aNGzeyf/9+goODmTlzJp6enri7u6f10iID1C9an7K5yzL/xHyws4N33pFBv8KsdewIR46oDSdbtIBHj7ROJIRIizQ/Wjp06BDTp09P+v2AAQMAmDNnDpUqVaJ58+aEhobi7+9PZGQk3t7ejB49Oq2XFRlEp9Mxvu54+v3Qjw+qfYDdgAHg4QEhIWp9GSHMkIcHHDsGvXtDlSpq3My/JlcKIcyIbs+ePWY3KTEqKgpfX1/Cw8NljEwGSExMpPK3lXnP6z0GVxsMzZurgQaff651NCHSxGBQSyT5+amORhnHLkT60uv1ODo6EhQUhL29vVHOaVJ7LQnTpNPpGFd3HDMOzSAuIU4N+p07F8LDtY4mRJpYWKgBwBs3wogRMHiwrDAghLmRQkakSvuy7bG1smXl2ZXw9tvg6QmzZ2sdSwij8PGBEyfU2Jm33lJj2oUQ5kEKGZEqlhaWfFz7Y6YdnIaBRJgyRRUyDx5oHU0IoyhWDA4eVONnZGsDIcyHFDIi1bp7diciJoJtwdugdm2oVQuMvB6AEFqytYXFi9WYmZYt1XRt2dpACNMmhYxINWtLa973fp+AYwGqYeRImD8fnj7VNpgQRvbuu7B3r6rT27cHvV7rREKIF5FCRryW973f55drv3Dl8RVo0gRcXNS2BUJkMlWqqK0NoqKgalX44w+tEwkhUiKFjHgtbjndaFOmDfOOz1PbFAwbBv7+0v8uMqXcueHnn6FTJ7VP05o1WicSQvyXFDLitY2qOYqFJxfy8MlDte/S9etquocQmZClJUyerIqYQYNU7S5TtIUwHVLIiNdWrUA16hSuw+zDsyF7dlXMLFyodSwh0pWvr5qivXcvNGwId+5onUgIAVLIiDc04a0JfH3sax5HP4b334d16yAsTOtYQqSrEiXg0CH1Xy8vVdQIIbQlhYx4I7UK1aKqW1UWnFgAFSpAxYqwYoXWsYRId9mzw9KlaoeO5s1h1iwZIiaElqSQEW9sWPVhzD8xX21bMGQIzJkDcXFaxxIi3el0MGCAWjTP318NBo6I0DqVEFmTFDLijfmW8sXKworNf26Gzp3VqMjVq7WOJUSGqV4dTp2C0FC1j+qFC1onEiLrkUJGvDFLC0sGVRnE18e+BisrGD9ebV0QH691NCEyTJ48sH07tG2rCpvvv9c6kRBZixQyIk36efXj2O1jXHhwAXr0gIQEGSsjshxLS7WtwYoV8N57MGqUPGUVIqNIISPSJJddLtqUacPS00tVr4yfH3zyCTx5onU0ITJc69Zw/Djs2KE2ib97V+tEQmR+UsiINOtbqS/Lzy4n3hCvRj0WLKgG/gqRBZUsqdaHLFxYTdE+cEDrREJkblLIiDRrVLwRVhZWaldsnQ5mzoTp09UISCGyIHt79Zhp/Hjw8YGvvpIp2kKkFylkRJpZWljSt1JftaYMQL166kfR+fO1DSaEhnQ6GDwYdu6EGTOga1eIjNQ6lRCZjxQywigGVR3EL9d+4fd7v6uGjz9WC2xER2uaSwit1aqlpmjfu6dmNV28qHUiITIXKWSEUbjmcKVPxT5MPzhdNfj4gJsbLFmibTAhTEC+fKpnpkULtd7Mpk1aJxIi85BCRhjN6Fqj+f7891wLvab61UePVoMDDAatowmhOSsr9YhpyRLo0wfGjJEll4QwBilkhNGUyFWCzuU6M3nfZNXQsSPo9WouqhACgPbt1RTtn36Cxo3VIychxJuTQkYY1Wdvfcbq31fz58M/wcYG+veHr7/WOpYQJqV0aTh6VD1y8vaGw4e1TiSE+ZJCRhhViVwl6FOpD5/9+plq6N8fdu2CS5e0DSaEicmRA9asgQ8/hEaNICBApmgL8SakkBFG90m9T9jy5xYuPryoBvx27gxffql1LCFMjk4Hw4apvZqmTIGePSEqSutUQpgXKWSE0RV0KEiX8l2YfXi2avjoI1i+HO7c0TaYECaqTh01Rfuvv6BmTbh8WetEQpgPKWREuhhdazTLzy7nXuQ9KFcOmjaVbQuEeIn8+eGXX9RjpqpVYfNmrRMJYR6kkBHponze8tQvWp95x+epho8/hgULZNsCIV4iWzaYPRsWLYJevVRnpkzRFuLlpJAR6WZ49eEsPLmQ2IRYqFEDqlRRIxqFEC/VsSMcOwZBQaqHRnbRFuLFpJAR6aZxicbktMnJxvMbVcPYsWqBPBnNKMQrlSmjpmgXKACVK8O+fVonEsI0SSEj0o2FzoJBVQYRcPzvXpjGjaFIEVi8WNtgQpiJHDlg5UqYMAGaNVOT/2SKthDJSSEj0lWfSn04ffc0p0JOqbmmY8eqf41jY7WOJoRZ0Olg4EDYs0etLdm+PYSHa51KCNMhhYxIV852zrxT6R1mHpqpGtq2BTs7WL1a22BCmJlq1eDkSXjyRA03O3tW60RCmAYpZES6G1VrFIEXAgl+HAyWlmq3vOnTZTNJIV5T7txqj6YePaBWLVi2TOtEQmhPk0LGYDAwdOhQWrduzeL/jJfYuXMngwcPZvDgwZw8eVKLeMLIijoVpWO5jnx56O/VfXv2hIgI2LJF22BCmCFLS/jsM9i4EUaNgvffh6dPtU4lhHY0KWQsLCz45JNPGDhwYLL2yMhIVq9ezezZs/Hz8yMgIICEhAQtIgoj+6j2Ryw7s4yQiBCwtlb/Ak+dKiMXhXhDPj5qNeAzZ6B2bbh2TetEQmgj1YXMvn37GDlyJL6+vjRo0CDFAmP16tV06NCBpk2bMn78eB4/fvzC8+XNm/e5tgsXLlC+fHlsbGxwdHQkX7583Lp1K7URhQkrn7c8jYs35qujX6mG996DK1dg925tgwlhxgoXVtOya9ZUu2j/9JPWiYTIeKkuZGJiYvDy8qJr164pvr5161ZWrFjB0KFDCQgIICoqiokTJ75WmPDwcHLmzJn0+xw5chAuw/MzjY/rfMw3x78h7GmYmlf6wQcwbZrWsYQwazY2ap3JgADo0gXGjwfpyBZZSaoLmcaNG9OjRw/KlSuX4uuBgYG0b9+eevXq4e7uzpgxYzh79izBwcEAbNmyheHDhzNhwoQXXsPBwYGIiIik30dGRuLg4JDaiMLE1SpUi0qulfjm+Deq4YMP4PBhOHFC22BCZALduqkF9DZtgiZN4P59rRMJkTGMMkYmNjaWK1euULly5aQ2Nzc3XF1dOX/+PACtW7fG39+fSZMmvfA8Hh4enDt3jtjYWPR6Pffu3aNQoULGiChMxLi645hzZA5RsVHg4qJGKk6dqnUsITIFDw+1tUHu3Go14IMHtU4kRPqzMsZJ9Ho9BoMBZ2fnZO1OTk6EhYWl+J4pU6Zw+fJlYmJiOH/+PDNnziRHjhx06dKF4cOHo9PpGDx4MJaWlsaIKEyETwkfijkVY8GJBYyqNQpGjgR3d/jzT7UmuxAiTXLmhLVr1eJ5TZrAlCkwbJhaWE+IzMgohUziG8w8GT9+fIrtPj4++Pj4pOoc48aNw9ra+rXfJ7Sj0+n4tN6nvPfjewyqOgi7ggXVdOzJk2HVKq3jCZEp6HQwdChUrao2oDx0SO0M8q8hiEJkmO3bt7N9+3ZAPcExNqMUMo6OjlhYWBAaGpqsPSwsDCcnJ2NcIkV+fn4yhsYM+ZbyJY99HtacW8M7ld+BTz9VvTFnzkDFilrHEyLTqFkTfvtNjZ+pWlWtPfOCYY5CpJt/dzTo9XrmzZtn1PMbZYyMtbU1JUqU4PTp00ltISEh3L17Fw8PD2NcQmQiOp2OQVUGseDEAtVQuLDaTGbsWG2DCZEJ5ckD27apnpkaNaTjU2Q+qS5k9Ho9wcHB3L59G4Dg4GCCg4OJjo4GoE2bNmzcuJH9+/cTHBzMzJkz8fT0xN3dPX2SC7PW3bM75x+c5+Sdv1dvHjdOjUw8cEDbYEJkQpaW6unt2rVqsuCgQRATo3UqIYxDt2fPnlQNcNm2bRvTp09/rn3OnDlUqlQJgFWrVrFp0yYiIyPx9vZm9OjR5MqVy6iBAaKiovD19SU8PFweLZmxAUEDiDfE812r71TD+PFqKvbfz1KFEMZ3/Tp06AAWFvD991CkiNaJRFai1+txdHQkKCgIe3t7o5wz1YWMKZFCJnM4e+8sNb6rwfXh18lrnxcePoSiRdVqv9WqaR1PiEzr6VMYPlwVMqtWQdOmWicSWUV6FDKy+7XQjGc+TxoUa4D/EX/VkDu36vOePFnTXEJkdra2sGABzJmjemc++0xWAxbmSwoZoamxdcYy7/g8wp/+vRXFqFHwyy9qNzwhRLrq1Ustrr1mDTRvrjpFhTA3UsgITdUpXIeK+Sr+s21Bvnxqtd8vvtA2mBBZRIUKcPy42v7My0ttcyCEOZFCRmhubJ2xzDkyhydxT1TDhx/C1q3w++/aBhMii3B0hA0b1LiZhg1h3jx4g3VOhdCEFDJCc03dm1LAoQBLfluiGgoUgHffVVOyhRAZQqdTO4Zs26a2NejeHSIjtU4lxKtJISM0p9PpGFtnLDMPzSQuIU41fvaZWlNmxw5twwmRxdStq4aohYSoyYN/7/srhMmSQkaYhPZl22NjacPq31erhty5VTEzYgTEx2sbTogsxtUVdu6Etm2henVYuVLrREK8mBQywiRYWljyUe2PmHZwGoZEg2ocNAiio2H9em3DCZEFWVmpR0zr1qnds/v3V+vPCGFqpJARJqNnxZ5Exkay+c/NqsHaWg38nTZNRh4KoZHmzdWjptOnoVYtuHpV60RCJCeFjDAZ1pbWjKo5Cr/9fiQ+K1z69oX79+Hnn7UNJ0QWVqQI7N8PdeqoKdqbN2udSIh/SCEjTMp7Xu9xPew6u67uUg22tmpO6MyZmuYSIquztoa5c2HRIrWQ3ujREBendSohpJARJsbe2p5h1Yfhd8Dvn8Z+/eDQIenTFsIEdOyo9nbdsQMaNIDbt7VOJLI6KWSEyRlSbQgn7pzg8M3DqiFPHvWgfvlybYMJIQAoVQqOHFH/rVRJzXASQitSyAiT42znzJCqQ/hkzyf/jJXp3VsVMgaDtuGEEABkzw5LlsD06Wqa9sSJsvGk0IYUMsIkfVTnI07fPc32K9tVQ4sWoNerRfKEECbjnXfg4EFYtUp1nD54oHUikdVIISNMkpOtE5/U/YQxO8eQYEhQIw179oQFC7SOJoT4j4oV1bgZBweoXFkVNkJkFClkhMkaVHUQoU9D/1lXZtAg2LhRrZ0uhDApDg5q7coxY6BxY5g9W5Z/EhlDChlhsmysbBhZYyTTD05XY2VKllRb8y5cqHU0IUQKdDoYOhR27wZ/f2jfHsLCtE4lMjspZIRJe9frXS4/vsy+G/tUwwcfqMdLsbHaBhNCvFCNGvDbb2pLA29v9Wsh0osUMsKk5bTJyaAqg5h56O8F8Zo2BScnWLFC01xCiJdzcYGgIDUYuE4d1ZEqj5pEepBCRpi8AVUGsP3Kdm7rb4OFBYwbB1Onyq7YQpg4CwsYP14VNJ9+qlZRiIrSOpXIbKSQESavkGMhGhZryIqzf/fCdO2q1pNZt07bYEKIVGnQQD1eunEDqleHCxe0TiQyEylkhFnoW6kv/zv9PzXo18oKxo6FKVNkgTwhzISbG/zyC/j6QrVqsGaN1olEZiGFjDALbcq04X7UfQ7f+nvbgl69ICICAgO1DSaESDUrK5g2TRUxgwerFRViYrROJcydFDLCLNha2dLTsyfzjs9TDTY2asGKL76QEYRCmBlfXzh1Co4fh9q14do1rRMJcyaFjDAbQ6sPZcP5DdwMv6ka3n0X7tyBn3/WNpgQ4rUVLap2HKlRA7y84McftU4kzJUUMsJsuOdyp0XJFsw9Olc12NnBqFHSKyOEmbKxgYAAmD8fundXnaxxcVqnEuZGChlhVkbVHMXCUwvRx+hVw8CBcPEi7NmjbTAhxBvr0kU9Ztq2Tc1wunVL60TCnEghI8xKrUK1KJu7LItPLVYNOXPC8OEwebKmuYQQaVO6NBw5ov5buTJs3651ImEupJARZkWn0zGq5ij8j/oTb/h7QbwPPoDTp2HHDk2zCSHSJnt2WLwYvvxS7dP06aeQkKB1KmHqpJARZqdt2bZY6CzYcH6DanB2hgkTYORIWe1XiEygd2/VO7NhAzRqJBvei5eTQkaYHSsLK0bUGMHMQzPVAnmgFqWIjYVFi7QNJ4QwivLl1biZAgXUo6bdu7VOJEyVFDLCLPWr3I8bYTfYdXWXarC2Vv3REyZAWJim2YQQxpEjh9ofdvJkaNlS/VceNYn/kkJGmCV7a3uGVh/K1ANT/2ls2RIqVlTTsYUQmYJOB++9BwcPwvLl0KwZ3L+vdSphSqSQEWZrSLUhHLt9jKO3jqoGnQ5mz4Z58yA4WNtwQgijqlQJTp5UQ+IqV4b9+7VOJEyFJoWMwWBg6NChtG7dmsWLFye1X7x4kSFDhjBs2DCGDBnC+fPntYgnzEQuu1z09+7PtIPT/mn09FT7MH34oXbBhBDpwsEB1q6F8eOhaVOYPl32jRWg27NnjyZLot6/f59Tp05x+/Zt+vXrB8CjR4+wtbXF3t6ea9euMWPGDObPn//ce6OiovD19SU8PBwHB4eMji5MyG39bUrMLcGp/qfwyOOhGu/fh5IlYfNmtbqWECLTOXECOnWCsmXVIycXF60TidTQ6/U4OjoSFBSEvb29Uc6Z6h6Zffv2MXLkSHx9fWnQoAEJKYy4Wr16NR06dKBp06aMHz+ex48fv/B8efPmfa7NxcUl6Q9mbW2NhYU8+RIvV8ChAD09ezLj4Ix/GvPmVT+yjRghP64JkUlVqaI2nrS2Vo+ajhzROpHQSqorhZiYGLy8vOjatWuKr2/dupUVK1YwdOhQAgICiIqKYuLEiW8UKi4ujtmzZ9OrV683er/IWsbUHsPac2v5K/yvfxqHDYMHD2QnOiEyMScn2LRJLSH19tswZ45su5YVpbqQady4MT169KBcuXIpvh4YGEj79u2pV68e7u7ujBkzhrNnzxL896DLLVu2MHz4cCZMmPDS6yQkJDB58mQaNGhA9erVX+OPIrKqki4laV2mNV8e+vKfRhsb1SMzbZr8yyZEJqbTqV1KfvlFFTLt2skKDFmNUZ7dxMbGcuXKFSpXrpzU5ubmhqura9KA3datW+Pv78+kSZNeeB6DwYCfnx/ly5fH19fXGNFEFvFx7Y/57tR3PIh68E/j++/DhQsyvUGILKBGDfjtN7V7tpeXGkMjsgajFDJ6vR6DwYCzs3OydicnJ8JeUBpPmTKFtWvXsmvXLkaNGoXBYODXX3/l4MGDHDp0iOHDhzN+/HhjxBNZQOX8lalXpB5fHf3qn0YHB7Xi7/Tp2gUTQmQYFxf44QcYMADeekutxCAdspmflTFOkvgGn5SUipSGDRvSsGHDVJ9j3LhxWFtbA+Dj44OPj89r5xCZx6f1PqXZqmYMrzGc3Nlzq8ahQ6FoUTh7Vk3NFkJkahYWMGYM1KwJXbrA3r2wcKEaTyO0sX37drb/vZ15bGys0c9vlELG0dERCwsLQkNDk7WHhYXhlI6fHj8/P5l+LZLULlybukXqMnX/VGb5zFKN+fJBnz4wYwasXKlpPiFExqlbVz1q6tNHLfi9ejXUrq11qqzp3x0Ner2eefPmGfX8Rnm0ZG1tTYkSJTh9+nRSW0hICHfv3sXDw8MYlxAiVaY0nML8E/O5GX7zn8bRo+H77+HaNe2CCSEyXN688NNPalZTkybw3XdaJxLpIdWFjF6vJzg4mNu3bwMQHBxMcHAw0dHRALRp04aNGzeyf/9+goODmTlzJp6enri7u6dPciFSUMm1Em3LtmXc7nH/NJYoAa1bw9dfaxdMCKEJnU6txhAUBB9/DL17Q3i41qmEMaV6Zd9t27YxPYVBk3PmzKFSpUoArFq1ik2bNhEZGYm3tzejR48mV65cRg0MsrKveLmb4TcpO68sO3vupGahmqrx4EFo3hxu3YKcObUNKITQxO3b8M478OefsGwZ1K+vdaKsJz1W9tVsi4K0kEJGvMrkvZP58dKPHHn3CBY6CzV1oWpV6NtXzWQSQmRJiYnwzTfw0Ueqp2byZDVAWGQMTbcoEMKcjK41mpv6mwRdClINz1bNmjtX5mMKkYXpdOpnmWPHYP16aNtWHjWZOylkRKZkl82OD2t9yOR9k/9ZHqBjR7h7V1bKEkLg4QFHj0JsrNqr6ehRrROJNyWFjMi0+nv353rYdbZfUesXYGOjfvxau1bbYEIIk5Arl5rVNGgQNGyodtEW5kcKGZFp2VvbM6rmqOS9Mp07q/5k2RVbCIEaHzN6NGzZosbMDB4sj5rMjRQyIlMbVHUQFx5c4Nfrv6qGRo3gyRM4fFjTXEII09KoERw/DhcvQpky8PPPWicSqSWFjMjUHGwcGFZ9GJP3TVYN2bJB+/awapW2wYQQJsfdHXbuhKlT1fYGw4apTSiFaZNCRmR6Q6sP5cSdExy6eUg1vPuu2q4gIkLbYEIIk6PTqW0NfvtN7dPUqJGaIyBMlxQyItNztnPmfe/38T/irxqqVVN9xzKyTwjxAiVKqHU0CxWCUqXAzw/i47VOJVIihYzIEgZXHcyWi1v+2YNpyBAICJA1ZYQQL2Rvrzpvf/pJPY1u3RoiI7VOJf5LChmRJRRzLkbzks2Zf2K+aujUCR4+hF9+0TaYEMLk1a0Lhw7B06dQvTrs3691IvFvUsiILGNotaEsPLmQ6LhosLWF999XvTJCCPEKjo6wbZvadLJZM7VQuAwENg1SyIgso37R+uTPmZ/Vv69WDQMGwNatcP26prmEEOYhWzYYMwbOnIHdu6FxY7hyRetUQgoZkWXodDqGVhvK3GNz1QJ5hQqBry8sWKB1NCGEGSlRQi1FVaYMVKgA06fLcDstSSEjspTunt25pb/Fvhv7VMMHH8CiRTKCTwjxWuzt1c9Ae/fCV1+p3bSlmNGGFDIiS8meLTvveb3H3GNzVcNbb0HJkjB/vrbBhBBmqWpVVcysXas6eC9d0jpR1iOFjMhyBlUdRNClIK6HXVerX336KXz5pdq6QAghXlPJkmoBvaJFoWJF+PBD0Ou1TpV1SCEjspzCjoVpWaol3xz/RjU0bw4FCqhHTEII8QZcXGDePDh6FE6cgCJF1MDghw+1Tpb5SSEjsqSh1Yey6NQiomKjVK/MJ5/AjBlqoQghhHhDnp5qRtMPP8C5c1C+vFpQT6QfKWREllS3cF1K5irJrMOzVEObNpArF/zvf5rmEkKYP51OLaL3009qRlOXLrBkidapMi8pZESWpNPpCGgewLQD07j86DJYWMD48TBtGsTGah1PCJEJ6HRqAb0ff1Q7aX/2GfzxBxgMWifLXKSQEVlWtQLVeKfyO/QP6o8h0QAdO0LOnDB7ttbRhBCZSP36au3NEyfULCc7O6hUCVaskI0ojUEKGZGl+b3tx1/hfzFl3xSwtFQDfidPhosXtY4mhMhE6tRRj5rCwlSvzAcfqB6aXLnUa99+CzExWqc0T1LIiCzNwcaBjZ02Mv3gdL479R1PqlSE/v1Vf7AM/BVCGJm1Nbi7Q79+as2ZAwfUPzdz54KHB9y6pXVC8yOFjMjyKrpWZHnb5Xx56EtcZrhQLPcqztz/nYdtm0JCgtbxhBCZlJWVmuX03nvw++/QoAH4+KidtgMDISLiP2+QXSpTJIWMEEC7su24MPgCZwacYX3vIPbPG8PDk/u46WrHvtK27PN0Yk+Lcly8cer5N//5J9y7p34dF6dWxvrtt+cHDR89Co8epf8fJqNFRsKvv776uPh4GUhtAuYcnsPua7u1jiH+w8JCbXng6alWCP7wQyhbFr77DvbtiiV63CS1L0KrVmazfHBUlHqMlt6s0v8SQpgHnU5HKZdSAFQtUJWrR1tyZ3cQ+R9GERH1GPvAIJ7UrMLSWWPo03WaetPevdCiBeTJQ+L69YQOeZfs5/4kMTGR22ULcnDeR1hls6HJ8oPkCVii/pXauVPNjkpIgDlzUg5z44bavMXWFt5+G3r0UFMgXsNf4X/x8MlD3HO542Dj8Oo3xMWp7X0BNm9WBVq2bOqralWoWfP59xw/Dt26qS2A//wTSpUiMTER3d9Z4w3xPPj9KHm6vIPVxcuQJw/s3w+lSiU7TWxCLA+iHvA0/inHbh/jWtg1XOxc+OPBHxy9fZQaBWrQu1JvvPJ7kZiYyJFbR4iKi6K4c3GKOxd/rfvy3B87IY4fLv7AiTsnKJajEH0tvMjmVUX9uPwfMfExfLD1A+oUrkMvz56Er1vO1Z9WcrNqKSrXbEchtzJqccU3zJHNMttrveev8L/Ikz0Pdla2r/58XL/O74c2M/bqWGysbDjc7zAeeTxe+pYbYTeYdXgWfm/7kcM6x0uzBz8OpmyeskltV0OvsvDkQqa+PTXp8/AqD588ZFvwNrqW74qlheULj1t3bh1jdo1hYf3Z+DzJD4ULq2/yNjaQPXuK74lNiGVg0ECuhV0jj30extcdj2c+z5QvkJj4zw8duXMnNd+JuMOjJ4+okK9Cqv48L5OYmIju/n01MKZwYUB95Nas+SfCihWwzj+EKX+0Jjg+hl1Nf6bM75uo7VGdVR//xE0Lbzaus6F8efVXNDZWDSJu3igWq+zWSefZttXA7lWHGTa9NgUL/idIfDxER6uJDv8R/jScmYdmsuvqLr5u9jVVC1RN9npUbBQJBgMXzuTEO+clrM6dhk6dAPVPWOvWqpDZuFHVX+lFt2fPHrPb5ioqKgpfX1/Cw8NxcEjFP9BCGENsLHff64rzqk0kurtjW6AwiceOsXd4WyJOH6Fl0GV+LmfNts974GTtQO8xq4iJjyH346c8tI7HdfMucn0xC8OvewjLnYOctx/SYVpl3mnxKa3LtP7nOjExavRf4cKq8Pn2WxKrVEHXogUUKwZNmqg4CbFExUbx58M/2bPsc+4/vMHxCrnwyu9NnluPKT5/LbE2VkypHku5Gi0ZW2csNQv9pxiJi1PzQgMDSXz4kPO9m5FoYUG5DfvQNWumXo+NhV27YNIkGDHin2+YV69C5crw8cfwxx9E2VowvXtRAo4FMLvuFHrczsXkvZN5Z+UfbC4Djwb35fOzubDYsFEtQPj77xATQ5xDDgYafmC57SVsEy0IOJaHt89HcyuPDfqibthXqkJg0af8/PtGfjjnie7qVW4aQlnQyo0tLg8ZUWMEn771KbZWtgAkHjvGH4M6cLOGB5Wn/g/XnPlVe2Iifx76AcPCbynlWIxsHhV42rEtnXa8x8U7Z5n2W27q/nAap6gEojq0xnHNJvVj8u3b8MUXxJ08zq07f3KsaDYeJkbSJ8aDmEvnOVTekSqXo3AIf4qtwQKLadNh5Eg4coTbrvZMPj+f97zeo/D1UCLf7cXN3NnY06EKrer0o3LJujy1teK7OT25vfdHGszYQJNyLQl/Go6DjUPyAmDVKrXSWlwcTJ3Kioe76bulLx8e1vHeGStaf5CHO5ZR2FrZsrX7VvVNescOCA8Hd3cSW7Qg9sFdViwfzY18Nvzv9P94u/jbWOgsuB52nei4aLJny86Ei/mpu2ofCXnzst72CgcLJ+Kauxijao4iODSYcBt4Ym9N2OM7FLJwpppzORb9MpPTd0/TZ8QyatfvCcDAoIEsOLmA9R3W07FcRzXmbNIkWL+exGLFSOjcEfr0YcmZpfx0Wa0Y98vVXzAkGviq6Ve85/3ec38Fn8Y/ZebBmcw8NJOh3oPxGTCDag9ssIl6qr5jW1mp75jjx4OXV7L3fvDzBxy4eYDxRXtR5MtF3L5zkTzZHMmRLQclGrYnR7U6oNfzdM8uEn7YjH34E8iRA1at4v7bNZh+YDrzT8wnkUS+avoV71tVV0v3NmkC1asTv/sXLL281d/Ts2fVDyxXrhBbtzYXG1SgQv6K6q9cQhyT9k5i5cH5nF5qi+OTBOIP7GPZ/gCe/riJcl8up36xBgDcPnMAF582WDdpxoauoxiz/xMSbR8xZscF6l8KZ0TjUozNNRL9HTse3ogi39MbFL60i7JxZ/B7azwbquopfSKY8X/soPKDGDrUacGQqRspVsiGFSvg+82RBPIORS7tZEmtVtxp0pAP321H9suX2Pr4GH3OfE7pXGWxvlebPdH+lLkzmVH1BrN/jw3bL+0mrGEPnPRPGLrBm+H3DpLNKpE1M3uRrWpPhrWpR/v2aju7vn1h2KA4Ot37mryTelKgWF6CgoKwt7d/g3+InyeFjBCvacLmYTzd/hNt3d7mx/g/WOn0Fx9UHULV63FU6zCU7LZ//2QTFgZTp0K9enSKWIJVNhtq5qpI8OzxBLeqw1dL7/OgpBuNSx1hZbuV3D97mOL7fqf+lQSe3rnJ2M/r8MQigYe3LtJsxREqPMlB5RuxHGvmydRmOfn15j4MsbGMOmXDxF8MYGuLvqgrT6LCyX/9IZGd25ILOxLXr+dsbXc+KXqV2a3mUbxWC34OPUZU2ANqfPw1NjdDmN6lEKceniVguyW5wmOZOr4etuUrcz38Ot/6fkvuP65B27bEFsjP8mb5iatTi7YjF3GhkC1TuxTA6vc/2OB/h15z6tG4QhtKvjMaD70NeutEinYfzK2PBtJybSvcHYsTeLQYFidPEulZhsQc9pw++RPlj/+F0xMDZM+OrmJFGDVKjXq8eFE9pjt6lAQrS1Z6JnLeqzDj87bHYVYAkeVKEnHlPOFOdrjXaI7VjZvEnTjG/2rY0Pb3eE47PeWPgtYY7OzI+SSObkeesKuSA3ftEng7xJaC1x9ztUB2SlvkxdLBkQS/KUx6HEjfEcsoWPktrIoWx7B2Db97FWS261U8ClZmRHxVDl3fT2D8OS74ePHjgH3YWNkQeCEQ/2968ePKBHLYOZIYHs5DyxiWdCmN4++X6H0yge2+ZSgZ74DHzyewTFCLiehtdCRaWZDg4sJJm4dMG1GVPfePUa1ANUbXGo21pTU1d18mz/gpMHAgXL/O05+2MNs7lq5lOlJ4VRChxVwxFHDjzqLZLDn9P07cOcEBxxFY9OkL+fNDcDCr2pfC8nEonRLKwu5fWHt+PVceX8H2QSgdFh7gfqOa/OmRlw4dJvCBj4HobNA3ujQNH+Tk4v3zPIl7gq2lNU5PdeSMTiDO1powy1girBKJc8hOAZs85P39KjrX/FC1KhXdd9Ko4bts/nMzf3Y7RPZGzcDGhsiPRvL1xjF023absByWxFtnwz3ajmwxcehyuRBWOC+Lbc4z/N1F5LDOQVjUI/Zd2I7N+YvEX/6TXLGWFPHtgZuFI082rKFIt/v88u5+PF3KQkgICV9/jcWqVehOnoTffiNx6FCiosNZWzKGt384R7EuA8DOjse1KnNZf4PTd3+j0IXb+MQV4bF1PJtsrnKqehE22VxlZEQ5hi84w5S3dJzt5YNfuaFk27OPxXtm8cn2aL73tqNTrDv2V2+zo+BT6tzUkcPWQRWPTZpwxOouBXceJdYiEXLnxrZwcRa5hXDL2YqxZxy4efsPbhbISaPTehyfGMDGhhFNDJxt7k1cQizTp50iJK8tH3fKxeOnoQysMpDqBarjYJmd+qMDiNm3hysuOnLnKoh9ThcSCxckuFQ+tl49yYcrj3LQvTi1r93i3MDOFM5XFKZMo3znEoTu/4L8tbfS5db/GLkvJx+WaML4+5so90CNCQyzASsD/NqrL1POLSI2zpLa3fayLnIIoU/CsLW2xkIXwuJttWj12z6OlcrOCC8HGoXEM+j3CN57O44PT3nS4P0u6Fq14vsdwZT9YjLWYZFY7/6eYjVLSyEjhYzQUnRcNH229CEkIoTizsWZ1WQWLtldXvqeOxF3KDuvLDmtc7Kx00aqF6yuflru0YNlgZ+zauEQNm6w5FyJHJzO+YQ5dSzxeasfLtldyJ8jPzUK1uDY7WNcPvITYybvJntsItZWtlg+egx586LbsgVKl1YrE7u6qmVFnz3iuHYNZs4kLHANT55GkDfCwDk3K8rcSyC4iAMLJjSjTKlaNC7emNK5SvIo/C5j9n2Khc6Cm/qbJCQmsK37NmLDHrHk/Sq0//U+ro9iuOFmz8Jv36do/rIUdSpK/f5TyXb3PsTFEZID3uoQwY6BhyjqVBRQ3dTVvqtG9wrdKZenHF03diXOEEdBh4Ice+cI+fUGNdbIy0v1hCS7gXcgMpK7bg442zpjY2WjxgkcOsQTt7zM3DgSt3tRFPSsy+inP+DfP5DGjpV5Mnc2T25dJSEyAiwtcRg4HNu6Dfjh4g+cu3+OEvfjaR1bDDusVJe4lRUJhgR6ft2Ahtsu4ZLNgaW5bxLT8C0+r/85NQrWSPoMjPtlHB/X+Zh8OfIlxbwaepWhC1pjdeYcv5a1ZVWkD83X/8aT2tW43astpZp2UwcaDIQ+eczF4CPY3b5P2XrtsE60IKzJW1hdvUZ8h3YctX7AwQcnsUrUMXLjHX6c0pu87Xqy6cImItYt50t9DfI+jIaZM6FECbVbYZEiJNhk49id43jfMvDHV5/wY6lEgvYuIk/JiqxsNB/nKnXUxkClS6vHhtu2Qa1aqrenaFGoVImoxQu4F3WPok5FsdBZEB0Xzf2o+xRxKpLsf0tMfAw/XPyBJiWa4GjryPjNQ7m8Yw2jL7qQ748bFPr9BnU2tuCruZfJnbcoZwM+ZfrRWeSxz8OcWpNJWL6UYkUrYVW4qFpc5dEjEs+fZ/+aaRS49wQrKxsex4RhY5eTiFJFsPGogGfxmlgEblbTfQ4f5rPQQAL/DGRRy0VcfHSRT/d8yoR1d2l7Izs5Q58wr48He3V/sfpnO+zy5If791Vv4N/fO2LiY2iysgmn754me7bsTKo/iXe93uVK6BW2/LmF3H/+Rdfxa7Au56nGuXl4EGtjRUin5nzvZcO0A9Nwz+WOXTY7fr96hJMlv6RIyx58d+V7xv0yjs2t11Lytxv4759B9qu36HM7N/mfZsMibz7urVjA8adXKLlwE0U6vIPtYz0J3buxbekn5D1xHq9lO+DcH/x0bz/57POpfzf+JcGQwLzj89gWvI2z984SERtBUaeiNC7emPHXCuH8404ICFD/XxMTMTRrSvTxw1jqI7CNhwTrbLQbmIv9+ePY2+dXStsW48eg4zzO4Uz4slt0ChxEthw25BncGSvnnCQULcp2dyhw7ALlvlqNlbOLWrKifHlu3IAPRyYwOrAWZazOs6JGPN4hOqpdicYiEb4vb8EPY1rz2dszKFmgpBQyUsgIc3Tm7hny58xPXvu8qsFgUN+0z5wh0doa3cKFJPbqxd4beymbu2yyb5DJhIer8Sg6nSpa3NxSHM/xX4ZEA70Ce1EkDD411MG2Vj01mvAl4xciYiKo/l11nGydeBr/FFsrW3b33o1tyANwckr+XD0kBLZvV98c27TBkN0OC13yguT03dPUXlIbgHUd1tGkRBMSExNVYZIGUbFRzDg4g+N3jlOzYE0+fevTNJ0v7GkYK86sIN4QT42CNZ5/JPcScQlx3Im4QyHHQs/9+V8pPh5++QU2bIAHD9TjmOhorjapSq/8R7gfdZ+CDgVZ3GoxxZyLJX/v5ctw5AgkJvLHvXN8cXc9p4rZUC5POTp6dKRz+c4qT0iIKgKuXVPXK19ePY759Vf1SOznn9Xn6g0kJiYyce9EJu+ZyJXD1Sh67iYJdrY81j2l1+gSPLCIpkzuMnzX6rukR4EpuRF2g0WnFmFINNDMvRl1i9R9/qCnT8HWlriEOFqtbcXpu6dxsnViXJ1xFLLNB+/2Y1fNfOgb1OKztz7D5UEktG2rCr+33052qtDoUHZe3Unr0q1T/iyGhKhxbR06qB8S/uXT3Z+y+/pudvbcycRfJ7I1eCvN3JsRcDyAH7v+SMNiDZPdn1eOF3r3XVi8WP092rxZbWprLOHhcOgQicWLo3N0hOzZCbM2EBETQSHHQs8dHvk4Fvudm9H9FKQeaZ4+rXpJCxRQvaZDhjz3b09UiB677DpisluxLXgbN8Jv0MOzBxExEXx36juGVx5OXhd5tCSFjMg8nj5Vj6CyZ0/6CdHU3Ay/yQ8Xf8DSwpKOHh1f2fv0Klv+3EL2bNlpXKKxkRIKU3Pizgkq5yqH5d596htgzZqqFyiTexL3hIm/TiQ8Jpz6RevTpXyX1z9JYqIqMHW6VP2AkqESE9XYuMKF/5kY8Jr0ej2Ojo5SyEghI4QQQpif9ChkZB0ZIYQQQpgtKWSEEEIIYbakkBFCCCGE2dKkkDEYDAwdOpTWrVuzePHi514PDw+nZcuW/PTTTxqkE0IIIYS50GRItIWFBZ988gmnTp3i9u3bz72+bNkyPD1fsHS0EEIIIcTfUl3I7Nu3j82bN3Pp0iWioqLYtWsXlpbJ98JYvXo1mzZtIjIyEm9vb0aNGkWuXLlSPF/evHlTbL9+/TrR0dGULFnyNf4YQgghhMiKUv1oKSYmBi8vL7p27Zri61u3bmXFihUMHTqUgIAAoqKimDhx4msHWrx4MX369Hnt94k3t337dq0jZBpyL41D7qPxyL00HrmXpinVhUzjxo3p0aMH5cqVS/H1wMBA2rdvT7169XB3d2fMmDGcPXuW4OBgALZs2cLw4cOZMGHCC69x7NgxChYsSL58L1jRVKQL+ctpPHIvjUPuo/HIvTQeuZemyShjZGJjY7ly5Qr9+/dPanNzc8PV1ZXz58/j7u5O69atad269UvOApcuXeLChQuMGTOG27dvY21tTf78+fH6zy6mQgghhBBgpEJGr9djMBhwdnZO1u7k5ERYWFiK75kyZQqXL18mJiaG8+fPM3PmTHr06EGPHj0AWLp0KXny5EmxiElMTEy6rki72NhYuZdGIvfSOOQ+Go/cS+ORe5l2z+7fs+/jxmCUQuZNAo0fP/6lr79snEx0dDQAhQo9v8mVeDPz5s3TOkKmIffSOOQ+Go/cS+ORe2kc0dHR5MiRwyjnMkoh4+joiIWFBaGhocnaw8LCcHJyMsYlknFxcWH9+vXY2dm9eidRIYQQQpiExMREoqOjcTHiJqJGKWSsra0pUaIEp0+fxtvbG4CQkBDu3r2Lh4eHMS6RjIWFBXny5DH6eYUQQgiRvozVE/NMqgsZvV7P/fv3kxawCw4OxtLSkgIFCmBnZ0ebNm0ICAigVKlS5M+fn2+++QZPT0/c3d2NGlgIIYQQ4hndnj17UjXAZdu2bUyfPv259jlz5lCpUiUAVq1alWxBvNGjR79wQTwhhBBCiLRKdSEjhBBCCGFqNNlrKa1eZysEoaayL1u2LFlb7dq1+eKLLwC4efMms2fP5vz58zg7O9OrVy+aN2+uRVST86qtOVJz7+TzqrzqXjZo0OC59yxatCjZ42m5l7By5Ur27dvHzZs3yZ49O9WqVaN///7JJlbI5zJ1UnMv5XP5aqtXr2bbtm3cv38fGxsbypcvz4ABA5JmFqf351GT3a/TwlhbIWQ1ZcqUYePGjUlfH3/8MQDx8fGMHTsWR0dHFixYQM+ePZk9ezYnT57UOLFpeNnWHKm5d/J5/certjkB+Oyzz5J9TosVK5b0mtxL5dy5c3Ts2JFvv/2WL774guvXrzNp0qSk1+VzmXqvupfPyOfy5dzc3Bg2bBj/+9//mDVrFhYWFowdOxbImM+j2fXI/HsrBIAxY8bQvXt3goODZWDxS1hZWaVY3R49epT79++zcOFCsmfPTrFixThz5gyBgYFJM9CyssaNGwNw+vTp515Lzb2Tz+s/XnYvn8mZM+cLfwqTe6lMmzYt2e+HDBnCkCFDiIyMJEeOHPK5fA2vupfPyOfy5erXr5/s93379qVfv348fvyYCxcupPvn0ax6ZJ5thVC5cuWktn9vhSBe7MqVK7Rr146ePXvi7+9PREQEAH/++SdlypQhe/bsScd6eXlx4cIFraKajVfdO/m8vr5p06bRtm1bhg4dyuHDh5Pa5V6+WHh4ONbW1tjZ2QHyuUyL/97LZ+RzmXoxMTFs27aNQoUK4eTklCGfR7PqkXmTrRAEeHh4MHbsWAoUKMDdu3dZtGgRn3zyCf7+/oSGhj63aKHcz9R51b2Tz+vr6devH15eXlhaWnLgwAHGjx/PzJkz8fb2lnv5ArGxsSxfvhwfH5+k8UbyuXwzKd1LkM9lah0+fJhJkyYRExNDwYIFmT59etJCuen9eTSrQsaYezNkJdWqVUv6dfHixSlSpAg9evTg0qVLGqbK/OTz+nqe7bMGULp0ae7du8eGDRvw9vaWe5mChIQE/Pz8ABg4cGCq3yf38nkvu5fyuUydSpUq8d133/H48WPWr1/P5MmTmTt37ivfZ4x7aFaPljJ6K4TMqkCBAuTIkYOQkBCcnZ2fq3rlfqbOq+6dfF7TplSpUoSEhAByL//LYDAwffp0/vrrL2bMmJHsUYh8Ll/Py+5lSuRzmTI7OzsKFChAhQoVmDBhAteuXePo0aMZ8nk0q0Lm31shPJOeWyFkVvfu3SMyMhJXV1fKlCnDxYsXkzbiBPjtt98oW7ashgnNw6vunXxe0+bKlSu4uroCci//LTExkZkzZ3L+/Hm+/PJLHBwckr0un8vUe9W9TIl8LlMnMTERS0vLDPk8mtWjJUC2QngDCxYsoHbt2uTJk4eQkBAWLFhAuXLlKFWqFAkJCeTOnZvp06fTu3dvLly4wO7du58bzZ9VvWxrjmrVqr3y3snn9R8vu5enT58mLCyMsmXLYmlpyf79+9mxY0dSdz/IvXxm9uzZHD58mKlTpwLw+PFjQP1ka2lpKZ/L1/Cqe3n48GH5XKbCt99+S506dXBxcSE0NJQ1a9bg6OhI+fLlsbGxSffPo1mu7CtbIbyeiRMncvbsWfR6PS4uLlStWpV+/folddv99ddfSYsV5cqVi549e9KiRQttQ5uIV23NkZp7J59X5WX3MjY2lm+//ZY7d+5gYWFB4cKF6d69O3Xq1El2rNzLlBdoA1izZk1ST4F8LlPnVffy2LFj8rlMhcmTJ3P27FnCw8NxdHTE09OTvn37UrBgQSD9P49mWcgIIYQQQoCZjZERQgghhPg3KWSEEEIIYbakkBFCCCGE2ZJCRgghhBBmSwoZIYQQQpgtKWSEEEIIYbakkBFCCCGE2ZJCRgghhBBmSwoZIYQQQpit/wP6lGDFRfAYsAAAAABJRU5ErkJggg==\n", 33 | "text/plain": [ 34 | "
" 35 | ] 36 | }, 37 | "metadata": {}, 38 | "output_type": "display_data" 39 | } 40 | ], 41 | "source": [ 42 | "# Solving least squarse min_x ||Ax - b||^2\n", 43 | "\n", 44 | "np.random.seed(123)\n", 45 | "\n", 46 | "n = 100\n", 47 | "m = 300\n", 48 | "A = np.random.randn(m,n)\n", 49 | "b = np.random.randn(m,1)\n", 50 | "x_star = np.linalg.lstsq(A, b)[0]\n", 51 | "f_star = 0.5 * (np.linalg.norm(A.dot(x_star) - b) ** 2)\n", 52 | "N = 300\n", 53 | "t = 0.002\n", 54 | "\n", 55 | "gs_vanilla = []\n", 56 | "gs_aa_1 = []\n", 57 | "gs_aa_2 = []\n", 58 | "\n", 59 | "x = np.zeros((n,1))\n", 60 | "for i in range(N):\n", 61 | " x -= t * A.T.dot(A.dot(x) - b)\n", 62 | " gs_vanilla.append(np.linalg.norm( A.T.dot(A.dot(x) - b)))\n", 63 | " \n", 64 | "\n", 65 | "aa_mem = 5\n", 66 | "\n", 67 | "aa_wrk = aa.AndersonAccelerator(n, aa_mem, True, regularization=0, max_weight_norm=1e12)\n", 68 | "x = np.zeros((n,1))\n", 69 | "for i in range(N):\n", 70 | " if i > 0: aa_wrk.apply(x, x_prev)\n", 71 | " x_prev = np.copy(x)\n", 72 | " x -= t * A.T.dot(A.dot(x) - b)\n", 73 | " aa_wrk.safeguard(x, x_prev)\n", 74 | " gs_aa_1.append(np.linalg.norm( A.T.dot(A.dot(x) - b)))\n", 75 | "\n", 76 | "aa_wrk = aa.AndersonAccelerator(n, aa_mem, False, regularization=0, max_weight_norm=1e12)\n", 77 | "x = np.zeros((n,1))\n", 78 | "for i in range(N):\n", 79 | " if i > 0: aa_wrk.apply(x, x_prev)\n", 80 | " x_prev = np.copy(x)\n", 81 | " x -= t * A.T.dot(A.dot(x) - b)\n", 82 | " aa_wrk.safeguard(x, x_prev)\n", 83 | " gs_aa_2.append(np.linalg.norm( A.T.dot(A.dot(x) - b)))\n", 84 | "\n", 85 | " \n", 86 | "plt.semilogy(gs_vanilla, label='grad desc')\n", 87 | "plt.semilogy(gs_aa_1, label='AA-I')\n", 88 | "plt.semilogy(gs_aa_2, label='AA-II')\n", 89 | "\n", 90 | "plt.legend()\n", 91 | "plt.show()\n" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "# l1 regularized least squares\n", 101 | "\n", 102 | "np.random.seed(123)\n", 103 | "\n", 104 | "n = 300\n", 105 | "m = 100\n", 106 | "mu = 0.1\n", 107 | "rho = 0.1\n", 108 | "N = 10_000\n", 109 | "\n", 110 | "A = np.random.randn(m,n)\n", 111 | "b = np.random.randn(m,1)\n", 112 | "\n", 113 | "L = la.cho_factor(A.T.dot(A) + rho * np.identity(n))\n", 114 | "\n", 115 | "def soft_thresh(y, t):\n", 116 | " return np.sign(y) * np.maximum(abs(y) - t, 0)\n", 117 | "\n", 118 | " \n", 119 | "z0 = np.linalg.solve(A.T.dot(A), A.T.dot(b))\n", 120 | "\n", 121 | "z = z0\n", 122 | "lam = np.zeros((n,1))\n", 123 | "res_vanilla = []\n", 124 | "ds_vanilla = []\n", 125 | "for i in range(N):\n", 126 | " z_old = z\n", 127 | " x = la.cho_solve(L, rho * (z + lam) + A.T.dot(b))\n", 128 | " z = soft_thresh(x - lam, mu / rho)\n", 129 | " lam = lam - x + z\n", 130 | " res_vanilla.append(np.linalg.norm(x-z))\n", 131 | " ds_vanilla.append(np.linalg.norm(z - z_old))\n", 132 | "\n", 133 | " \n", 134 | "aa_mem = 5\n", 135 | "\n", 136 | "print(\"====================== Type - I ======================\")\n", 137 | "z = z0\n", 138 | "lam = np.zeros((n,1))\n", 139 | "u = np.vstack((z,lam))\n", 140 | "aa_wrk = aa.AndersonAccelerator(2 * n, aa_mem, True, regularization=1e-6, safeguard_factor=1, verbosity=1, max_weight_norm=1e12)\n", 141 | "res_aa_1 = []\n", 142 | "ds_aa_1 = []\n", 143 | "for i in range(N):\n", 144 | " if i > 0: aa_wrk.apply(u, u_old)\n", 145 | " u_old = np.copy(u)\n", 146 | " x = la.cho_solve(L, rho * (z + lam) + A.T.dot(b))\n", 147 | " z = soft_thresh(x - lam, mu / rho)\n", 148 | " lam = lam - x + z\n", 149 | " u = np.vstack((z, lam))\n", 150 | " aa_wrk.safeguard(u, u_old)\n", 151 | " z = u[:n]\n", 152 | " z_old = u_old[:n]\n", 153 | " lam = u[n:]\n", 154 | " \n", 155 | " res_aa_1.append(np.linalg.norm(la.cho_solve(L, rho * (z + lam) + A.T.dot(b)) - z))\n", 156 | " ds_aa_1.append(np.linalg.norm(z - z_old))\n", 157 | "\n", 158 | "print(\"====================== Type - II ======================\")\n", 159 | "z = z0\n", 160 | "lam = np.zeros((n,1))\n", 161 | "u = np.vstack((z,lam))\n", 162 | "aa_wrk = aa.AndersonAccelerator(2 * n, aa_mem, False, regularization=1e-12, safeguard_factor=1, verbosity=1, max_weight_norm=1e12)\n", 163 | "res_aa_2 = []\n", 164 | "ds_aa_2 = []\n", 165 | "for i in range(N):\n", 166 | " if i > 0: aa_wrk.apply(u, u_old)\n", 167 | " u_old = np.copy(u)\n", 168 | " x = la.cho_solve(L, rho * (z + lam) + A.T.dot(b))\n", 169 | " z = soft_thresh(x - lam, mu / rho)\n", 170 | " lam = lam - x + z\n", 171 | " u = np.vstack((z, lam))\n", 172 | " aa_wrk.safeguard(u, u_old)\n", 173 | " z = u[:n]\n", 174 | " z_old = u_old[:n]\n", 175 | " lam = u[n:]\n", 176 | " \n", 177 | " res_aa_2.append(np.linalg.norm(la.cho_solve(L, rho * (z + lam) + A.T.dot(b)) - z))\n", 178 | " ds_aa_2.append(np.linalg.norm(z - z_old))\n", 179 | "\n", 180 | "\n", 181 | "plt.semilogy(res_vanilla, label='admm - res')\n", 182 | "plt.semilogy(ds_vanilla, label='admm - dual')\n", 183 | "plt.semilogy(res_aa_1, label='AA-I - res')\n", 184 | "plt.semilogy(ds_aa_1, label='AA-I - dual')\n", 185 | "plt.semilogy(res_aa_2, label='AA-II - res')\n", 186 | "plt.semilogy(ds_aa_2, label='AA-II -dual')\n", 187 | "#plt.semilogy(res_aa_1_p, label='AA-I-P - res')\n", 188 | "#plt.semilogy(ds_aa_1_p, label='AA-I-P - dual')\n", 189 | "\n", 190 | "plt.legend()\n", 191 | "plt.show()" 192 | ] 193 | } 194 | ], 195 | "metadata": { 196 | "kernelspec": { 197 | "display_name": "Python 3 (ipykernel)", 198 | "language": "python", 199 | "name": "python3" 200 | }, 201 | "language_info": { 202 | "codemirror_mode": { 203 | "name": "ipython", 204 | "version": 3 205 | }, 206 | "file_extension": ".py", 207 | "mimetype": "text/x-python", 208 | "name": "python", 209 | "nbconvert_exporter": "python", 210 | "pygments_lexer": "ipython3", 211 | "version": "3.7.3" 212 | } 213 | }, 214 | "nbformat": 4, 215 | "nbformat_minor": 2 216 | } 217 | --------------------------------------------------------------------------------