├── algopy ├── .gitignore ├── tests │ ├── __init__.py │ ├── test_operators.py │ ├── test_compound.py │ ├── test_utils.py │ └── test_npversion.py ├── utpm │ ├── tests │ │ ├── __init__.py │ │ └── test_utpm_convenience.py │ └── __init__.py ├── fft │ ├── __init__.py │ └── fft.py ├── tracer │ ├── tests │ │ ├── .gitignore │ │ ├── test_output │ │ │ ├── __init__.py │ │ │ └── .gitignore │ │ ├── environment.py │ │ ├── test_future_division.py │ │ └── __init__.py │ └── __init__.py ├── special │ ├── __init__.py │ └── compound.py ├── nthderiv │ └── __init__.py ├── linalg │ ├── __init__.py │ └── linalg.py ├── compound.py └── base_type.py ├── experimental ├── ctps │ ├── tests │ │ ├── __init__.py │ │ └── test_ctps.py │ ├── SConscript │ ├── __init__.py │ ├── SConstruct.EXAMPLE │ └── ctps.py ├── utps │ ├── tests │ │ └── __init__.py │ └── __init__.py ├── examples │ ├── .gitignore │ ├── prettyplotting.py │ └── newtons_method.py ├── performance_tests │ ├── adolc_implementation_for_speed_comparison │ │ ├── .gitignore │ │ ├── algopy_speed.py │ │ ├── SConstruct │ │ └── main.cpp │ ├── .gitignore │ ├── utps_vs_utpm.png │ ├── runtime_comparison.png │ ├── runtime_comparison.bak.png │ ├── profiling_reverse_mode.py │ ├── comparison_forward_reverse.py │ └── UTPS_vs_UTPM.py ├── README.rst ├── sparse_test.py ├── pullback_alternative.py ├── sequential_tape_prototype.py ├── numpy_array_dtype_interface.py └── tests │ └── trash │ ├── unit_test_with_sympy_x_as_vector.py │ ├── unit_test_with_sympy.py │ └── unit_test_with_sympy_x_as_matrix_input.py ├── documentation ├── sphinx │ ├── .gitignore │ ├── index.py │ ├── algopy_logo.png │ ├── example_tracer_cgraph.png │ ├── examples │ │ ├── explicit_euler.png │ │ ├── implicit_euler.png │ │ ├── mayavi_3D_plot.png │ │ ├── taylor_approximation.png │ │ ├── posterior_log_probability_cgraph.png │ │ ├── ampl_minimization_problem.rst │ │ ├── hessian_of_potential_function.rst │ │ ├── iterate.dat │ │ ├── comparison_forward_reverse_mode.rst │ │ ├── error_propagation.py │ │ ├── minimization │ │ │ ├── rosenbrock_banana.py │ │ │ ├── himmelblau_minimization.py │ │ │ └── wood_function_minimization.py │ │ ├── series_expansion.py │ │ ├── comparison_to_complex_step_derivative_approximation.py │ │ ├── posterior_log_probability.rst │ │ ├── minimal_surface.rst │ │ ├── leastsquaresfitting.rst │ │ ├── logistic_regression.rst │ │ ├── first_order_forward.py │ │ ├── leastsquaresfitting.py │ │ ├── explicit_euler.py │ │ ├── posterior_log_probability.py │ │ ├── polarization.py │ │ ├── taylor_series_of_jacobian.py │ │ ├── gauss_newton.py │ │ ├── covariance_matrix_computation.py │ │ ├── comparison_forward_reverse_mode.py │ │ ├── error_propagation.rst │ │ ├── neg_binom_regression.rst │ │ ├── moore_penrose_pseudoinverse.py │ │ ├── series_expansion.rst │ │ ├── first_order_forward.rst │ │ ├── complex_differentiation.rst │ │ ├── hessian_of_potential_function.py │ │ ├── covariance_matrix_computation.rst │ │ ├── neg_binom_regression.py │ │ ├── minimal_surface.py │ │ ├── implicit_euler.py │ │ ├── ode_solvers.rst │ │ ├── expm_identities.py │ │ └── polarization.rst │ ├── symbolic_differentiation.rst │ ├── talks │ │ ├── informal_talk_iwr_heidelberg_theory_and_tools_for_algorithmic_differentiation.pdf │ │ ├── Seventh_EuroAd_Workshop-Sebastian_Walter-Higher_Order_Forward_and_Reverse_Mode_on_Matrices_with_Application_to_Optimal_Experimental_Design.pdf │ │ └── walter_euroad2010_paderborn_univariate_taylor_polynomial_arithmetic_applied_to_matrix_factorizations_in_the_forward_and_reverse_mode_of_algorithmic_differentiation.pdf │ ├── symbolic_differentiation.py │ ├── examples_tracer.py │ ├── speed_comparison │ │ ├── use_uncertainties.py │ │ ├── use_numdifftools.py │ │ ├── benchmark1.py │ │ ├── use_scientific.py │ │ ├── use_adolc.py │ │ ├── use_funcdesigner.py │ │ ├── prettyplotting.py │ │ ├── use_theano.py │ │ └── use_algopy.py │ ├── datastructure_and_algorithms.py │ ├── getting_started.py │ ├── examples_tracer.rst │ ├── datastructure_and_algorithms.rst │ ├── folder_structure.rst │ ├── make.bat │ ├── Makefile │ ├── getting_started.rst │ └── runtime_comparison.py ├── walter_algorithmic_differentiation_in_python_with_algopy.pdf ├── AD_tutorial_TU_Berlin │ ├── walter_AD_tutorial_at_TU_Berlin_17_06_2010.pdf │ ├── example5_code_tracing_with_pyadolc.py │ ├── example6_utps_simple_function.py │ ├── example4_eigh.py │ ├── example1_qr.py │ ├── example9_explicit_euler.py │ ├── example8_reverse_simple_function.py │ ├── prettyplotting.py │ ├── example7_simple_computation_of_the_hessian.py │ ├── example2_qr_numerical_stability.py │ └── example3_qr_for_moore_penrose_pseudo_inverse.py ├── ICCS2010 │ ├── accuracy_of_eigh_for_large_matrices.py │ ├── prettyplotting.py │ ├── degenerated_eigenvalues.py │ ├── stability_of_qr_decomposition_for_low_rank_matrices.py │ └── odoe_example.py └── examples │ ├── numpy_compatibility.py │ ├── differentiation_of_the_symmetric_eigenvalue_decomposition.py │ ├── hessian_computation_in_combined_forward_reverse_mode.py │ ├── comparison_forward_reverse_mode.py │ ├── error_propagation.rst │ ├── error_propagation.py │ ├── covariance_matrix_computation.py │ └── moore_penrose_pseudoinverse.py ├── MANIFEST.in ├── setup.cfg ├── run_tests.py ├── sandbox.py ├── .gitignore ├── .travis.yml ├── pyproject.toml └── CHANGELOG.rst /algopy/.gitignore: -------------------------------------------------------------------------------- 1 | version.py 2 | -------------------------------------------------------------------------------- /algopy/tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /algopy/utpm/tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /algopy/fft/__init__.py: -------------------------------------------------------------------------------- 1 | from .fft import * -------------------------------------------------------------------------------- /experimental/ctps/tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /experimental/utps/tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /algopy/tracer/tests/.gitignore: -------------------------------------------------------------------------------- 1 | test_output 2 | -------------------------------------------------------------------------------- /algopy/tracer/tests/test_output/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /experimental/examples/.gitignore: -------------------------------------------------------------------------------- 1 | !algopy_logo.png 2 | -------------------------------------------------------------------------------- /documentation/sphinx/.gitignore: -------------------------------------------------------------------------------- 1 | *_build 2 | *_static 3 | *.swp -------------------------------------------------------------------------------- /algopy/special/__init__.py: -------------------------------------------------------------------------------- 1 | from .special import * 2 | from .compound import * 3 | -------------------------------------------------------------------------------- /algopy/tracer/__init__.py: -------------------------------------------------------------------------------- 1 | from . import tracer 2 | from .tracer import * 3 | 4 | -------------------------------------------------------------------------------- /algopy/tracer/tests/test_output/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | !__init__.py 4 | -------------------------------------------------------------------------------- /experimental/performance_tests/adolc_implementation_for_speed_comparison/.gitignore: -------------------------------------------------------------------------------- 1 | adolc -------------------------------------------------------------------------------- /documentation/sphinx/index.py: -------------------------------------------------------------------------------- 1 | import os 2 | print(os.path.dirname(os.path.realpath(__file__))) 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.py 2 | recursive-include algopy *.py 3 | recursive-include documentation *.py 4 | -------------------------------------------------------------------------------- /documentation/sphinx/algopy_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/sphinx/algopy_logo.png -------------------------------------------------------------------------------- /algopy/nthderiv/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Explicit nth derivatives without chain rule. 3 | """ 4 | 5 | from .nthderiv import * 6 | -------------------------------------------------------------------------------- /experimental/ctps/SConscript: -------------------------------------------------------------------------------- 1 | Import('env') 2 | ctps = env.SharedLibrary(target='ctps', source=['src/ctps.c']) 3 | 4 | 5 | -------------------------------------------------------------------------------- /experimental/README.rst: -------------------------------------------------------------------------------- 1 | This folder contains experimental stuff (e.g. new algorithms) that may or may 2 | not be used in the future. 3 | -------------------------------------------------------------------------------- /experimental/performance_tests/.gitignore: -------------------------------------------------------------------------------- 1 | *_stats 2 | 3 | !runtime_comparison.png 4 | !runtime_comparison.bak.png 5 | !utps_vs_utpm.png -------------------------------------------------------------------------------- /documentation/sphinx/example_tracer_cgraph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/sphinx/example_tracer_cgraph.png -------------------------------------------------------------------------------- /experimental/performance_tests/utps_vs_utpm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/experimental/performance_tests/utps_vs_utpm.png -------------------------------------------------------------------------------- /documentation/sphinx/examples/explicit_euler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/sphinx/examples/explicit_euler.png -------------------------------------------------------------------------------- /documentation/sphinx/examples/implicit_euler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/sphinx/examples/implicit_euler.png -------------------------------------------------------------------------------- /documentation/sphinx/examples/mayavi_3D_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/sphinx/examples/mayavi_3D_plot.png -------------------------------------------------------------------------------- /experimental/utps/__init__.py: -------------------------------------------------------------------------------- 1 | from .utps import * 2 | 3 | print('WARNING: the submodule algopy.utps is deprecated and tagged for removal!') 4 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/taylor_approximation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/sphinx/examples/taylor_approximation.png -------------------------------------------------------------------------------- /experimental/performance_tests/runtime_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/experimental/performance_tests/runtime_comparison.png -------------------------------------------------------------------------------- /experimental/performance_tests/runtime_comparison.bak.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/experimental/performance_tests/runtime_comparison.bak.png -------------------------------------------------------------------------------- /documentation/sphinx/examples/posterior_log_probability_cgraph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/sphinx/examples/posterior_log_probability_cgraph.png -------------------------------------------------------------------------------- /documentation/sphinx/symbolic_differentiation.rst: -------------------------------------------------------------------------------- 1 | Symbolic Differentiation 2 | ------------------------ 3 | 4 | 5 | .. literalinclude:: symbolic_differentiation.py 6 | :lines: 1- 7 | 8 | -------------------------------------------------------------------------------- /documentation/walter_algorithmic_differentiation_in_python_with_algopy.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/walter_algorithmic_differentiation_in_python_with_algopy.pdf -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [build_sphinx] 2 | source-dir = documentation/sphinx 3 | build-dir = documentation/sphinx/_build 4 | all_files = 1 5 | 6 | [upload_sphinx] 7 | upload-dir = documentation/sphinx/_build/html 8 | -------------------------------------------------------------------------------- /documentation/AD_tutorial_TU_Berlin/walter_AD_tutorial_at_TU_Berlin_17_06_2010.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/AD_tutorial_TU_Berlin/walter_AD_tutorial_at_TU_Berlin_17_06_2010.pdf -------------------------------------------------------------------------------- /run_tests.py: -------------------------------------------------------------------------------- 1 | if __name__ == '__main__': 2 | 3 | try: 4 | import pytest 5 | except: 6 | print('Please install pytest.') 7 | 8 | retcode = pytest.main(["-x", "algopy"]) 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /algopy/linalg/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | This module implements functionality 4 | to apply algopy to numpy.linalg and scipy.linalg functions. 5 | 6 | """ 7 | 8 | from .linalg import * 9 | from .compound import * 10 | 11 | 12 | -------------------------------------------------------------------------------- /algopy/tracer/tests/environment.py: -------------------------------------------------------------------------------- 1 | """ 2 | define environmental settings for the test files 3 | 4 | """ 5 | 6 | import os 7 | class Settings: 8 | output_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)) , 'test_output') 9 | -------------------------------------------------------------------------------- /documentation/sphinx/talks/informal_talk_iwr_heidelberg_theory_and_tools_for_algorithmic_differentiation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/sphinx/talks/informal_talk_iwr_heidelberg_theory_and_tools_for_algorithmic_differentiation.pdf -------------------------------------------------------------------------------- /documentation/sphinx/symbolic_differentiation.py: -------------------------------------------------------------------------------- 1 | import sympy 2 | x,y,z = sympy.symbols('xyz') 3 | 4 | def f(x,y,z): 5 | return x*y * z*x - y + x*(z-x*y) 6 | 7 | u = f(x,y,z) 8 | print('f(x,y,z) = ',u) 9 | g = [u.diff(x), u.diff(y), u.diff(z)] 10 | print('grad f(x,y,z) =', g) 11 | -------------------------------------------------------------------------------- /documentation/sphinx/examples_tracer.py: -------------------------------------------------------------------------------- 1 | 2 | from algopy import CGraph, Function 3 | cg = CGraph() 4 | cg.trace_on() 5 | x = Function(1) 6 | y = Function(3) 7 | z = x * y + x 8 | cg.trace_off() 9 | cg.independentFunctionList = [x,y] 10 | cg.dependentFunctionList = [z] 11 | print(cg) 12 | cg.plot('example_tracer_cgraph.png') -------------------------------------------------------------------------------- /algopy/tests/test_operators.py: -------------------------------------------------------------------------------- 1 | from numpy.testing import * 2 | import numpy 3 | numpy.random.seed(0) 4 | 5 | from algopy import UTPM, Function 6 | from algopy.special import * 7 | 8 | class Test_NumpyOperators(TestCase): 9 | 10 | # here should be tests for functions like 11 | # z = x + y 12 | # z = x * y 13 | pass 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /documentation/AD_tutorial_TU_Berlin/example5_code_tracing_with_pyadolc.py: -------------------------------------------------------------------------------- 1 | import numpy; from numpy import sin,cos; import adolc 2 | def f(x): 3 | return sin(x[0] + cos(x[1])*x[0]) 4 | 5 | adolc.trace_on(1) 6 | x = adolc.adouble([3,7]); adolc.independent(x) 7 | y = f(x) 8 | adolc.dependent(y); adolc.trace_off() 9 | adolc.tape_to_latex(1,[3,7],[0.]) 10 | 11 | -------------------------------------------------------------------------------- /documentation/sphinx/talks/Seventh_EuroAd_Workshop-Sebastian_Walter-Higher_Order_Forward_and_Reverse_Mode_on_Matrices_with_Application_to_Optimal_Experimental_Design.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/sphinx/talks/Seventh_EuroAd_Workshop-Sebastian_Walter-Higher_Order_Forward_and_Reverse_Mode_on_Matrices_with_Application_to_Optimal_Experimental_Design.pdf -------------------------------------------------------------------------------- /sandbox.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import numpy.random 3 | 4 | from algopy import UTPM, Function, CGraph, sum, zeros, diag, dot, qr 5 | 6 | 7 | 8 | D,P,N,M = 2,3,4,5 9 | 10 | X = 2 * numpy.random.rand(D,P,N,M) 11 | Y = 3 * numpy.random.rand(N,M) 12 | AX = UTPM(X) 13 | # AY1 = Y + AX 14 | # AY2 = Y - AX 15 | # AY3 = Y * AX 16 | AY4 = Y / AX 17 | # AY5 = AX + Y 18 | # AY6 = AX - Y 19 | # AY7 = AX * Y 20 | # AY8 = AX / Y -------------------------------------------------------------------------------- /documentation/sphinx/talks/walter_euroad2010_paderborn_univariate_taylor_polynomial_arithmetic_applied_to_matrix_factorizations_in_the_forward_and_reverse_mode_of_algorithmic_differentiation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b45ch1/algopy/HEAD/documentation/sphinx/talks/walter_euroad2010_paderborn_univariate_taylor_polynomial_arithmetic_applied_to_matrix_factorizations_in_the_forward_and_reverse_mode_of_algorithmic_differentiation.pdf -------------------------------------------------------------------------------- /documentation/sphinx/examples/ampl_minimization_problem.rst: -------------------------------------------------------------------------------- 1 | Derivatives for Optimization 2 | ---------------------------- 3 | 4 | Optimization algorithms often require: 5 | 6 | * Hessian of the Lagrangian 7 | * Diagonal elements of the Lagrangian 8 | * etc. 9 | 10 | 11 | The following code shows how these derivatives can be evaluated in AlgoPy: 12 | 13 | .. literalinclude:: ampl_minimization_problem.py 14 | 15 | -------------------------------------------------------------------------------- /documentation/AD_tutorial_TU_Berlin/example6_utps_simple_function.py: -------------------------------------------------------------------------------- 1 | import numpy; from numpy import sin,cos; from taylorpoly import UTPS 2 | 3 | def f(x): 4 | return sin(x[0] + cos(x[1])*x[0]) + x[1]*x[0] 5 | 6 | x = [UTPS([3,1,0],P=2), UTPS([7,0,1],P=2)] 7 | y = f(x) 8 | 9 | print('normal function evaluation y_0 = f(x_0) = ', y.data[0]) 10 | print('gradient evaluation g(x_0) = ', y.data[1:]) 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /algopy/special/compound.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains the implementation of functions 3 | that are not represented as a single node in the 4 | computational graph, but are 5 | treated as a **compound** function. 6 | 7 | Note 8 | ---- 9 | 10 | These functions should be replaced by a dedicated implementation in 11 | 12 | * algopy.Function 13 | * algopy.UTPM 14 | 15 | so they are represented by a single node in the CGraph. 16 | 17 | """ 18 | 19 | -------------------------------------------------------------------------------- /documentation/sphinx/speed_comparison/use_uncertainties.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import uncertainties 3 | import uncertainties.unumpy as unp 4 | 5 | 6 | class EVAL: 7 | def __init__(self, f, x, test = 'f'): 8 | self.f = f 9 | 10 | def gradient(self, x): 11 | sx = unp.uarray((x, np.inf)) 12 | sf = self.f(sx) 13 | return np.array([sf.derivatives[var] for var in sx]) 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /documentation/AD_tutorial_TU_Berlin/example4_eigh.py: -------------------------------------------------------------------------------- 1 | import numpy; from algopy import UTPM 2 | 3 | # symmetric eigenvalue decomposition, forward UTPM 4 | D,P,M,N = 3,1,4,4 5 | Q,R = UTPM.qr(UTPM(numpy.random.rand(D,P,M,N))) 6 | l = UTPM(numpy.random.rand(*(D,P,N))) 7 | l.data[0,0,:4] = [1,1,2,3] 8 | l.data[1,0,:4] = [0,0,3,4] 9 | l.data[2,0,:4] = [1,2,5,6] 10 | L = UTPM.diag(l) 11 | B = UTPM.dot(Q,UTPM.dot(L,Q.T)) 12 | 13 | print('B = \n', B) 14 | l2,Q2 = UTPM.eigh(B) 15 | print('l2 - l =\n',l2 - l) 16 | -------------------------------------------------------------------------------- /documentation/sphinx/speed_comparison/use_numdifftools.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import numdifftools as nd 3 | 4 | class EVAL: 5 | def __init__(self, f, x, test = 'f'): 6 | self.f = f 7 | self.x = x 8 | 9 | self.g = nd.Gradient(f) 10 | 11 | self.H = nd.Hessian(f) 12 | 13 | def gradient(self, x): 14 | return self.g(x) 15 | 16 | def hessian(self, x): 17 | return self.H(x) 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /documentation/sphinx/datastructure_and_algorithms.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from algopy import UTPM, dot 3 | 4 | D,P,M,N = 2,1,3,4 5 | x_data = 7*numpy.arange(D*P*M*N,dtype=float).reshape((D,P,M,N)) 6 | y_data = numpy.arange(D*P*M*N,dtype=float).reshape((D,P,N,M)) 7 | 8 | # calling algorithms directly 9 | z_data = numpy.zeros((D,P,M,M)) 10 | UTPM._dot(x_data, y_data, z_data) 11 | 12 | # use UTPM instance 13 | x = UTPM(x_data) 14 | y = UTPM(y_data) 15 | z = dot(x, y) 16 | 17 | print('z.data - z_data', z.data - z_data) 18 | -------------------------------------------------------------------------------- /experimental/ctps/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | CTPS = Cross Derivative Taylor Polynomial 3 | Implements the factor ring 4 | ..math:: 5 | 6 | R[t_1,...,t_K]/(T_1^2,...,T_K^2) 7 | 8 | and algorithms for arithmetic on this ring, in particular (+,-,*,/,exp,log,sin) 9 | 10 | 11 | Warning: 12 | This module is deprecated and will eventually be removed as part of a 13 | code consolidation. 14 | 15 | """ 16 | from .ctps import * 17 | 18 | print('WARNING: the submodule algopy.ctps is deprecated and tagged for removal!') 19 | -------------------------------------------------------------------------------- /documentation/ICCS2010/accuracy_of_eigh_for_large_matrices.py: -------------------------------------------------------------------------------- 1 | from numpy.testing import * 2 | import numpy 3 | 4 | from algopy.tracer.tracer import * 5 | from algopy.utpm import * 6 | 7 | D,P,N = 4,1,100 8 | A = UTPM(numpy.random.rand(D,P,N,N)) 9 | Q,R = UTPM.qr(A) 10 | l = UTPM(numpy.random.rand(D,P,N)) 11 | L = UTPM.diag(l) 12 | 13 | A = UTPM.dot(Q, UTPM.dot(L,Q.T)) 14 | 15 | l2,Q2 = UTPM.eigh(A) 16 | 17 | print('error between true and reconstructed eigenvalues') 18 | print(l.data[:,:,numpy.argsort(l.data[0,0])] - l2.data) 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /documentation/sphinx/speed_comparison/benchmark1.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import algopy 3 | 4 | class F: 5 | def __init__(self,N): 6 | A = np.arange(N*N,dtype=float).reshape((N,N)) 7 | self.A = np.dot(A.T,A) 8 | 9 | def __call__(self, x): 10 | return 0.5*np.dot(x*x,np.dot(self.A,x)) 11 | 12 | 13 | class G: 14 | def __init__(self,N): 15 | A = np.arange(N*N,dtype=float).reshape((N,N)) 16 | self.A = np.dot(A.T,A) 17 | 18 | def __call__(self, x): 19 | return 0.5*algopy.dot(x*x,algopy.dot(self.A,x)) 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # 2 | # NOTE! Don't add files that are generated in specific 3 | # subdirectories here. Add them in the ".gitignore" file 4 | # in that subdirectory instead. 5 | # 6 | # Normal rules 7 | # 8 | .* 9 | *.*~ 10 | *.py~ 11 | *.pyc 12 | *.txt 13 | trash/* 14 | trash 15 | *.png 16 | *.svg 17 | *.dot 18 | *.o 19 | *.os 20 | *.so 21 | *.eps 22 | *~ 23 | *.log 24 | *.tap 25 | algopy.egg-info 26 | test_experimental.py 27 | MANIFEST 28 | 29 | version.py 30 | 31 | SConstruct 32 | 33 | dist 34 | build 35 | report 36 | virtualenv/ 37 | venv/ 38 | 39 | # exceptions 40 | !.gitignore 41 | !.gitkeep 42 | -------------------------------------------------------------------------------- /experimental/performance_tests/adolc_implementation_for_speed_comparison/algopy_speed.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import numpy 4 | import numpy.linalg 5 | import time 6 | 7 | sys.path = ['../..'] + sys.path 8 | from vector_forward_mode import * 9 | 10 | def speelpenning(x): 11 | return numpy.prod(x) 12 | 13 | 14 | N = 100 15 | P = N*(N+1)/2 16 | D = 3 17 | 18 | x = numpy.zeros(N) 19 | S = numpy.zeros((N,P)) 20 | 21 | ax = double_to_adouble(x,S,D) 22 | 23 | start_time = time.time() 24 | speelpenning(ax) 25 | end_time =time.time() 26 | 27 | print('needed time was %0.6f seconds!'%(end_time-start_time)) -------------------------------------------------------------------------------- /documentation/AD_tutorial_TU_Berlin/example1_qr.py: -------------------------------------------------------------------------------- 1 | import numpy; from algopy import UTPM 2 | 3 | # QR decomposition, UTPM forward 4 | D,P,M,N = 3,1,500,20 5 | A = UTPM(numpy.random.rand(D,P,M,N)) 6 | Q,R = UTPM.qr(A) 7 | B = UTPM.dot(Q,R) 8 | 9 | # check that the results are correct 10 | print('Q.T Q - 1\n',UTPM.dot(Q.T,Q) - numpy.eye(N)) 11 | print('QR - A\n',B - A) 12 | print('triu(R) - R\n', UTPM.triu(R) - R) 13 | 14 | # QR decomposition, UTPM reverse 15 | Bbar = UTPM(numpy.random.rand(D,P,M,N)) 16 | Qbar,Rbar = UTPM.pb_dot(Bbar, Q, R, B) 17 | Abar = UTPM.pb_qr(Qbar, Rbar, A, Q, R) 18 | 19 | print('Abar - Bbar\n',Abar - Bbar) 20 | -------------------------------------------------------------------------------- /documentation/sphinx/speed_comparison/use_scientific.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import Scientific.Functions.Derivatives as SFD 3 | 4 | 5 | class EVAL: 6 | def __init__(self, f, x, test = 'f'): 7 | self.f = f 8 | self.x = x 9 | 10 | def gradient(self, x): 11 | N = len(x) 12 | sx = np.array([SFD.DerivVar(x[i], i, 1) for i in range(N)]) 13 | return np.array(self.f(sx)[1]) 14 | 15 | def hessian(self, x): 16 | N = len(x) 17 | sx = np.array([SFD.DerivVar(x[i], i, 2) for i in range(N)]) 18 | return np.array(self.f(sx)[2]) 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Config file for automatic testing at travis-ci.org 2 | 3 | language: python 4 | 5 | python: 6 | - "2.7_with_system_site_packages" 7 | - "3.2_with_system_site_packages" 8 | 9 | before_script: 10 | # add ppa with more recent scipy version 11 | - sudo add-apt-repository ppa:pylab/stable -y 12 | - sudo apt-get update -qq 13 | - sudo apt-get install -y python-sphinx python-nose python-numpy python-scipy 14 | - sudo apt-get install -y python3-sphinx python3-nose python3-numpy python3-scipy 15 | 16 | script: 17 | - python setup.py install --user 18 | - python run_tests.py 19 | - cd documentation/sphinx 20 | - make html -------------------------------------------------------------------------------- /documentation/sphinx/speed_comparison/use_adolc.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import adolc 3 | 4 | class EVAL: 5 | def __init__(self, f, x, test = 'f'): 6 | self.f = f 7 | self.x = x 8 | 9 | adolc.trace_on(0) 10 | ax = adolc.adouble(x) 11 | adolc.independent(ax) 12 | y = f(ax) 13 | adolc.dependent(y) 14 | adolc.trace_off() 15 | 16 | def function(self, x): 17 | return adolc.function(0,x) 18 | 19 | def gradient(self, x): 20 | return adolc.gradient(0,x) 21 | 22 | def hessian(self, x): 23 | return adolc.hessian(0,x) 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /algopy/tracer/tests/test_future_division.py: -------------------------------------------------------------------------------- 1 | 2 | from numpy.testing import TestCase 3 | 4 | import numpy 5 | import algopy 6 | 7 | 8 | class Test_future_division(TestCase): 9 | 10 | def test_utpm(x): 11 | x = numpy.array([1.,2.,3.]) 12 | ax = algopy.UTPM.init_jacobian(x) 13 | ay = 1./ax 14 | J = algopy.UTPM.extract_jacobian(ay) 15 | #print J 16 | 17 | def test_function(x): 18 | x = numpy.array([1.,2.,3.]) 19 | cg = algopy.CGraph() 20 | fx = algopy.Function(x) 21 | fy = 1./fx 22 | cg.trace_off() 23 | cg.independentFunctionList = [fx] 24 | cg.dependentFunctionList = [fy] 25 | #print cg.jacobian(x) 26 | 27 | 28 | -------------------------------------------------------------------------------- /documentation/examples/numpy_compatibility.py: -------------------------------------------------------------------------------- 1 | """ 2 | This example shows that most computations can be performed by numpy functions 3 | on arrays of UTPM objects. 4 | 5 | Just bear in mind that is much faster use UTPM instances of matrices than numpy.ndarrays 6 | with UTPM elements. 7 | 8 | """ 9 | 10 | import numpy, os 11 | from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv 12 | 13 | N,D,P = 2,2,1 14 | cg = CGraph() 15 | x = numpy.array([ Function(UTPM(numpy.random.rand(*(D,P)))) for n in range(N)]) 16 | A = numpy.outer(x,x) 17 | A = numpy.exp(A) 18 | y = numpy.dot(A,x) 19 | 20 | cg.independentFunctionList = list(x) 21 | cg.dependentFunctionList = list(y) 22 | 23 | cg.plot(os.path.join(os.path.dirname(__file__),'numpy_dot_graph.svg')) 24 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/hessian_of_potential_function.rst: -------------------------------------------------------------------------------- 1 | Hessian of Two Particle Coulomb Potential 2 | ----------------------------------------- 3 | 4 | This is an example from the Scipy-User mailing list. The original discussion 5 | can be found on 6 | http://www.mail-archive.com/numpy-discussion@scipy.org/msg28633.html 7 | 8 | .. literalinclude:: hessian_of_potential_function.py 9 | :lines: 0- 10 | 11 | 12 | As output one obtains:: 13 | 14 | $ python hessian_of_potential_function.py 15 | Optimization terminated successfully. 16 | Current function value: 0.000000 17 | Iterations: 81 18 | Function evaluations: 153 19 | [[ 5.23748399e-12 -2.61873843e-12] 20 | [ -2.61873843e-12 5.23748399e-12]] 21 | 22 | 23 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/iterate.dat: -------------------------------------------------------------------------------- 1 | RUNNING THE L-BFGS-B CODE 2 | 3 | it = iteration number 4 | nf = number of function evaluations 5 | nint = number of segments explored during the Cauchy search 6 | nact = number of active bounds at the generalized Cauchy point 7 | sub = manner in which the subspace minimization terminated: 8 | con = converged, bnd = a bound was reached 9 | itls = number of iterations performed in the line search 10 | stepl = step length used 11 | tstep = norm of the displacement (total step) 12 | projg = norm of the projected gradient 13 | f = function value 14 | 15 | * * * 16 | 17 | Machine precision = 2.220D-16 18 | N = 6 M = 10 19 | 20 | it nf nint nact sub itls stepl tstep projg f 21 | -------------------------------------------------------------------------------- /documentation/AD_tutorial_TU_Berlin/example9_explicit_euler.py: -------------------------------------------------------------------------------- 1 | import numpy; from numpy import sin,cos; from taylorpoly import UTPS 2 | x = numpy.array([UTPS([1,0,0],P=2), UTPS([0,0,0],P=2)]) 3 | p = UTPS([3,1,0],P=2) 4 | def f(x): 5 | return numpy.array([x[1],-p * x[0]]) 6 | 7 | ts = numpy.linspace(0,2*numpy.pi,100) 8 | x_list = [[xi.data.copy() for xi in x]] 9 | for nts in range(ts.size-1): 10 | h = ts[nts+1] - ts[nts] 11 | x = x + h * f(x) 12 | x_list.append([xi.data.copy() for xi in x]) 13 | 14 | xs = numpy.array(x_list) 15 | import matplotlib.pyplot as pyplot 16 | pyplot.plot(ts, xs[:,0,0], '.k-', label = r'$x(t)$') 17 | pyplot.plot(ts, xs[:,0,1], '.r-', label = r'$x_p(t)$') 18 | pyplot.xlabel('time $t$') 19 | pyplot.legend(loc='best') 20 | pyplot.grid() 21 | pyplot.show() 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/comparison_forward_reverse_mode.rst: -------------------------------------------------------------------------------- 1 | Comparison and Combination of Forward and Reverse Mode 2 | ====================================================== 3 | 4 | We show here how the forward and the reverse mode of AD are used and show 5 | that they produce the same result. It is also shown how the forward and the 6 | reverse mode can be combined to compute the Hessian of a function 7 | 8 | We consider the function :math:`f:\mathbb R^N \times \mathbb R^N\rightarrow \mathbb R` defined by 9 | 10 | .. math:: 11 | x,y \mapsto z = x^T y + (x \circ y - x)^T (x-y) 12 | 13 | We want to compute the Hessian of that function. The following code shows how 14 | this can be accomplished by a combined forward/reverse computation. 15 | 16 | .. literalinclude:: comparison_forward_reverse_mode.py 17 | 18 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/error_propagation.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv, zeros 3 | 4 | def f(y): 5 | retval = zeros((3,1),dtype=y) 6 | retval[0,0] = numpy.log(dot(y.T,y)) 7 | retval[1,0] = numpy.exp(dot(y.T,y)) 8 | retval[2,0] = numpy.exp(dot(y.T,y)) - numpy.log(dot(y.T,y)) 9 | return retval 10 | 11 | D,Nm = 2,40 12 | P = Nm 13 | y = UTPM(numpy.zeros((2,P,Nm))) 14 | 15 | y.data[0,:] = numpy.random.rand(Nm) 16 | y.data[1,:] = numpy.eye(Nm) 17 | 18 | 19 | # print f(y) 20 | J = f(y).data[1,:,:,0] 21 | print('Jacobian J(y) = \n', J) 22 | 23 | C_epsilon = 0.3*numpy.eye(Nm) 24 | 25 | print(J.shape) 26 | 27 | C = dot(J.T, dot(C_epsilon,J)) 28 | 29 | print('Covariance matrix of z: C = \n',C) 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /experimental/sparse_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from numpy import * 4 | from adolc import * 5 | 6 | 7 | class SparseMatrix: 8 | """ Sparse Matrix as Vertex List""" 9 | def __init__(self,N_rows,N_cols): 10 | self.N = N_cols 11 | self.M = N_rows 12 | self.R = [] 13 | def __repr__(self): 14 | return str(self.R) 15 | def __str__(self): 16 | return self.__repr__() 17 | 18 | 19 | def mul(Z,X,Y): 20 | """ 21 | Z = dot(X,Y), where Z,X,Y sparse matrices 22 | the bitpattern of Z is given, i.e. not all operations in dot(X,Y) are necessary. 23 | This multiplication function takes advantage of this knowledge. 24 | """ 25 | 26 | 27 | if __name__ == "__main__": 28 | N = 2 29 | A = SparseMatrix(N,N) 30 | A.R.append((0,0,12.)) 31 | A.R.append((1,0,13.)) 32 | 33 | print(A) -------------------------------------------------------------------------------- /documentation/sphinx/examples/minimization/rosenbrock_banana.py: -------------------------------------------------------------------------------- 1 | """ 2 | Minimize the Rosenbrock banana function. 3 | 4 | http://en.wikipedia.org/wiki/Rosenbrock_function 5 | """ 6 | 7 | import numpy 8 | 9 | import minhelper 10 | 11 | def rosenbrock(X): 12 | """ 13 | This R^2 -> R^1 function should be compatible with algopy. 14 | http://en.wikipedia.org/wiki/Rosenbrock_function 15 | A generalized implementation is available 16 | as the scipy.optimize.rosen function 17 | """ 18 | x = X[0] 19 | y = X[1] 20 | a = 1. - x 21 | b = y - x*x 22 | return a*a + b*b*100. 23 | 24 | def main(): 25 | target = [1, 1] 26 | easy_init = [2, 2] 27 | hard_init = [-1.2, 1] 28 | minhelper.show_minimization_results( 29 | rosenbrock, target, easy_init, hard_init) 30 | 31 | if __name__ == '__main__': 32 | main() 33 | 34 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "algopy" 7 | dynamic = ["version", "description", "license", "authors", "keywords"] 8 | readme = "README.rst" 9 | dependencies = ["numpy>=2.0.0", "scipy>=1.14.0", "pytest>=8.2.2"] 10 | requires-python = ">= 3.6" 11 | maintainers = [ 12 | {name = "Sebastian F. Walter", email = "sebastian.walter@gmail.com"} 13 | ] 14 | authors = [ 15 | {name = "Sebastian Walter", email = "sebastian.walter@gmail.com"}, 16 | {name = "Alexander Griffing", email = "argriffi@ncsu.edu"}, 17 | ] 18 | 19 | [project.urls] 20 | Homepage = "https://www.github.com/b45ch1/algopy" 21 | Documentation = "https://pythonhosted.org/algopy/" 22 | Repository = "https://www.github.com/b45ch1/algopy" 23 | Changelog = "https://github.com/b45ch1/algopy/blob/master/CHANGELOG.rst" 24 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/minimization/himmelblau_minimization.py: -------------------------------------------------------------------------------- 1 | """ 2 | Minimize the Himmelblau function. 3 | 4 | http://en.wikipedia.org/wiki/Himmelblau%27s_function 5 | """ 6 | 7 | import numpy 8 | 9 | import minhelper 10 | 11 | 12 | def himmelblau(X): 13 | """ 14 | This R^2 -> R^1 function should be compatible with algopy. 15 | http://en.wikipedia.org/wiki/Himmelblau%27s_function 16 | This function has four local minima where the value of the function is 0. 17 | """ 18 | x = X[0] 19 | y = X[1] 20 | a = x*x + y - 11 21 | b = x + y*y - 7 22 | return a*a + b*b 23 | 24 | 25 | def main(): 26 | target = [3, 2] 27 | easy_init = [3.1, 2.1] 28 | hard_init = [-0.27, -0.9] 29 | minhelper.show_minimization_results( 30 | himmelblau, target, easy_init, hard_init) 31 | 32 | 33 | if __name__ == '__main__': 34 | main() 35 | 36 | -------------------------------------------------------------------------------- /documentation/AD_tutorial_TU_Berlin/example8_reverse_simple_function.py: -------------------------------------------------------------------------------- 1 | import numpy; from numpy import sin,cos; from taylorpoly import UTPS 2 | x1 = UTPS([3,1,0],P=2); x2 = UTPS([7,0,1],P=2) 3 | # forward mode 4 | vm1 = x1 5 | v0 = x2 6 | v1 = cos(v0) 7 | v2 = v1 * vm1 8 | v3 = vm1 + v2 9 | v4 = sin(v3) 10 | y = v4 11 | # reverse mode 12 | v4bar = UTPS([0,0,0],P=2); v3bar = UTPS([0,0,0],P=2) 13 | v2bar = UTPS([0,0,0],P=2); v1bar = UTPS([0,0,0],P=2) 14 | v0bar = UTPS([0,0,0],P=2); vm1bar = UTPS([0,0,0],P=2) 15 | v4bar.data[0] = 1. 16 | v3bar += v4bar*cos(v3) 17 | vm1bar += v3bar; v2bar += v3bar 18 | v1bar += v2bar * vm1; vm1bar += v2bar * v1 19 | v0bar -= v1bar * sin(v0) 20 | g1 = y.data[1:]; g2 = numpy.array([vm1bar.data[0], v0bar.data[0]]) 21 | print('UTPS gradient g(x_0)=', g1) 22 | print('reverse gradient g(x_0)=', g2) 23 | print('Hessian H(x_0)=\n',numpy.vstack([vm1bar.data[1:], v0bar.data[1:]])) 24 | -------------------------------------------------------------------------------- /experimental/performance_tests/profiling_reverse_mode.py: -------------------------------------------------------------------------------- 1 | from numpy import * 2 | import pstats 3 | import profile 4 | import sys 5 | 6 | sys.path = ['..'] + sys.path 7 | from reverse_mode import * 8 | 9 | def profile_taylor_propagation(): 10 | N = 20 11 | A = ones((N,N)) 12 | x = [Tc(n) for n in range(N)] 13 | y = dot(A,x) 14 | 15 | def profile_reverse(): 16 | N = 20 17 | A = ones((N,N)) 18 | def fun(x): 19 | return 0.5* dot(x, dot(A,x)) 20 | 21 | cg = CGraph() 22 | x = [Function(Tc([1.,1.,0.])) for n in range(N)] 23 | g = fun(x) 24 | cg.independentFunctionList = x 25 | cg.dependentFunctionList = [g] 26 | cg.reverse([Tc(1)]) 27 | cg.reverse([Tc(1)]) 28 | cg.reverse([Tc(1)]) 29 | 30 | 31 | if __name__ == "__main__": 32 | profile.run('profile_reverse()', 'profile_reverse_stats') 33 | p = pstats.Stats('profile_reverse_stats') 34 | p.strip_dirs().sort_stats('cumulative').print_stats() 35 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/series_expansion.py: -------------------------------------------------------------------------------- 1 | import numpy; from numpy import sin,cos 2 | from algopy import UTPM 3 | 4 | def f(x): 5 | return sin(cos(x) + sin(x)) 6 | 7 | D = 100; P = 1 8 | x = UTPM(numpy.zeros((D,P))) 9 | x.data[0,0] = 0.3 10 | x.data[1,0] = 1 11 | 12 | y = f(x) 13 | print('coefficients of y =', y.data[:,0]) 14 | 15 | import matplotlib.pyplot as pyplot; import os 16 | zs = numpy.linspace(-1,2,100) 17 | ts = zs -0.3 18 | fzs = f(zs) 19 | 20 | for d in [2,4,10,50,100]: 21 | yzs = numpy.polyval(y.data[:d,0][::-1],ts) 22 | pyplot.plot(zs,yzs, label='%d\'th order approx.'%(d-1)) 23 | 24 | pyplot.plot([0.3], f([0.3]), 'ro') 25 | pyplot.plot(zs,fzs, 'k.') 26 | pyplot.grid() 27 | pyplot.legend(loc='best') 28 | pyplot.xlabel('x') 29 | pyplot.ylabel('y') 30 | pyplot.savefig(os.path.join(os.path.dirname(os.path.realpath(__file__)),'taylor_approximation.png')) 31 | # pyplot.show() 32 | 33 | 34 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/comparison_to_complex_step_derivative_approximation.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import algopy 3 | 4 | def f_fcn(x): 5 | A = algopy.zeros((2,2), dtype=x) 6 | A[0,0] = x[0] 7 | A[1,0] = x[1] * x[0] 8 | A[0,1] = x[1] 9 | Q,R = algopy.qr(A) 10 | return R[0,0] 11 | 12 | 13 | # Method 1: Complex-step derivative approximation (CSDA) 14 | h = 10**-20 15 | x0 = numpy.array([3,2],dtype=float) 16 | x1 = numpy.array([1,0]) 17 | yc = numpy.imag(f_fcn(x0 + 1j * h * x1) - f_fcn(x0 - 1j * h * x1))/(2*h) 18 | 19 | # Method 2: univariate Taylor polynomial arithmetic (UTP) 20 | ax = algopy.UTPM(numpy.zeros((2,1,2))) 21 | ax.data[0,0] = x0 22 | ax.data[1,0] = x1 23 | ay = f_fcn(ax) 24 | 25 | # Method 3: finite differences 26 | h = 10**-8 27 | yf = (f_fcn(x0 + h * x1) - f_fcn(x0))/h 28 | 29 | # Print results 30 | print('CSDA result =',yc) 31 | print('UTP result =',ay.data[1,0]) 32 | print('FD result =',yf) 33 | 34 | -------------------------------------------------------------------------------- /algopy/utpm/__init__.py: -------------------------------------------------------------------------------- 1 | """ UTPM == Univariate Taylor Polynomial of Matrices. 2 | 3 | UTPM arithmetic means to apply functions to 4 | ..math:: 5 | [A]_D = \sum_{d=0}^{D-1} A_d t^d 6 | A_d = \frac{d^d}{dt^d}|_{t=0} \sum_{k=0}^{D-1} A_k t^k 7 | 8 | The underlying data structure is a numpy.array of shape (D,P,N,M) 9 | where 10 | D: D number of coefficients, i.e. D-1 is the degree of the polynomial 11 | P: number of directions 12 | N: number of rows of the matrix A 13 | M: number of cols of the matrix A 14 | 15 | The data structure is stored in the attribute UTPM.data and can be accessed. 16 | 17 | Module Structure: 18 | ~~~~~~~~~~~~~~~~~ 19 | 20 | utpm.algorithms: 21 | algorithms that operate directly on the (D,P,N,M) numpy.array. 22 | 23 | utpm.utpm: 24 | Implementation of the class UTPM that makes is a thin wrapper for the 25 | algorithms implemented in utpm.algorithms. 26 | 27 | """ 28 | 29 | from .utpm import * 30 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/posterior_log_probability.rst: -------------------------------------------------------------------------------- 1 | Posterior Log Probability 2 | ------------------------- 3 | 4 | We want the derivative of a posterior log probability density calculation. 5 | We have a normal distribution with known variance. 6 | 7 | .. literalinclude:: posterior_log_probability.py 8 | :lines: 1- 9 | 10 | as output one obtains:: 11 | 12 | walter@wronski$ python examples/posterior_log_probability.py 13 | function evaluation = 14 | 138.692022348 15 | function evaluation = 138.692022 16 | 1st directional derivative = 44.061288 17 | 2nd directional derivative = 94.000000 18 | finite differences derivative = 19 | 44.0612893726 20 | gradient = 44.0612882911 21 | Hessian vector product = 94.0 22 | 23 | 24 | and the computational graph: 25 | 26 | .. image:: posterior_log_probability_cgraph.png 27 | :align: center 28 | :scale: 50 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /documentation/examples/differentiation_of_the_symmetric_eigenvalue_decomposition.py: -------------------------------------------------------------------------------- 1 | """ 2 | This example shows how derivatives of all eigenvalue of a 100,100 matrix can be 3 | computed and compares it to the exact value. 4 | """ 5 | 6 | from numpy.testing import * 7 | import numpy 8 | 9 | from algopy.tracer.tracer import * 10 | from algopy.utpm import * 11 | from algopy.globalfuncs import * 12 | 13 | D,P,N = 4,1,100 14 | 15 | print('create random matrix and perform QR decomposition ') 16 | A = UTPM(numpy.random.rand(D,P,N,N)) 17 | Q,R = qr(A) 18 | 19 | print('define diagonal matrix') 20 | l = UTPM(numpy.random.rand(D,P,N)) 21 | L = diag(l) 22 | 23 | print('transform to obtain non-diagonal matrix') 24 | A = dot(Q, dot(L,Q.T)) 25 | 26 | print('reconstruct diagonal entries with eigh') 27 | l2,Q2 = eigh(A) 28 | 29 | print('absolute error true - reconstructed diagonal entries') 30 | print(l.data[:,:,numpy.argsort(l.data[0,0])] - l2.data) 31 | 32 | 33 | -------------------------------------------------------------------------------- /documentation/sphinx/speed_comparison/use_funcdesigner.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import FuncDesigner 3 | 4 | class EVAL: 5 | def __init__(self, f, x, test = 'f'): 6 | self.f = f 7 | self.x = x.copy() 8 | 9 | # sA = FuncDesigner.oovar('A',shape=(len(x),len(x))) 10 | sx = FuncDesigner.oovar('x', size = len(x)) 11 | sy = 0.5*FuncDesigner.dot(sx*sx,FuncDesigner.dot(f.A,sx)) 12 | 13 | print('sy=',sy) 14 | 15 | # self.sA = sA 16 | self.sx = sx 17 | self.sy = sy 18 | 19 | def function(self, x): 20 | point = {self.sx:x } 21 | return self.sy(point) 22 | 23 | def gradient(self, x): 24 | point = {self.sx:x} 25 | # print point 26 | # print self.sy 27 | retval = self.sy.D(point)[self.sx] 28 | return retval 29 | 30 | # def hessian(self, x): 31 | # return adolc.hessian(0,x) 32 | 33 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/minimal_surface.rst: -------------------------------------------------------------------------------- 1 | Minimal Surface Problem 2 | ----------------------- 3 | 4 | In this example it is shown how the gradient of the function 5 | 6 | .. literalinclude:: minimal_surface.py 7 | :lines: 4-8 8 | 9 | can be computed conveniently using ALGOPY. It is the objective function of a minimal surface problem. 10 | With boundary constraints and an additional constraint in the domain in form of a cylinder one obtains 11 | a plot like 12 | 13 | .. image:: mayavi_3D_plot.png 14 | :align: center 15 | :scale: 100 16 | 17 | 18 | At first the initial values are specified and the function evaluation is traced 19 | 20 | .. literalinclude:: minimal_surface.py 21 | :lines: 12-27 22 | 23 | One can now compute the gradient of the objective function as follows 24 | 25 | .. literalinclude:: minimal_surface.py 26 | :lines: 30-41 27 | 28 | 29 | 30 | The complete code is 31 | 32 | .. literalinclude:: minimal_surface.py 33 | :lines: 1- 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /experimental/examples/prettyplotting.py: -------------------------------------------------------------------------------- 1 | 2 | import pylab as pyl 3 | 4 | #SETUP PLOTTING PARAMETERS 5 | fig_width_pt = 350.0 # Get this from LaTeX using \showthe\columnwidth 6 | inches_per_pt = 1.0/72.25 # Convert pt to inches 7 | golden_mean = (pyl.sqrt(5)-1.0)/2.0 # Aesthetic ratio 8 | fig_width = fig_width_pt*inches_per_pt # width in inches 9 | fig_height =fig_width*golden_mean # height in inches 10 | fig_size = [fig_width,fig_height] 11 | legend_padding = 0.05 12 | params = { 13 | 'backend': 'ps', 14 | 'ps.usedistiller': 'xpdf', 15 | 'font.family' : 'sans-serif', 16 | 'font.style' : 'normal', 17 | 'font.variant' : 'normal', 18 | 'font.weight' : 'normal', #bold 19 | 'font.stretch' : 'normal', 20 | 'font.size' : 'normal', #large 21 | 'axes.labelsize': 11, 22 | 'text.fontsize': 10, 23 | 'title.fontsize':10, 24 | 'legend.fontsize':10, 25 | 'xtick.labelsize': 11, 26 | 'ytick.labelsize': 11, 27 | 'lines.markersize':5, 28 | 'text.usetex': True, 29 | 'figure.figsize': fig_size} 30 | pyl.rcParams.update(params) 31 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/leastsquaresfitting.rst: -------------------------------------------------------------------------------- 1 | Least-Squares Fitting 2 | --------------------- 3 | 4 | This is an example from the Scipy-User mailing list. The original discussion 5 | can be found on 6 | http://permalink.gmane.org/gmane.comp.python.scientific.user/26494 . 7 | 8 | .. literalinclude:: leastsquaresfitting.py 9 | :lines: 0- 10 | 11 | 12 | We provide the Jacobian of the error function using ALGOPY and compare 13 | 1) the solution when scipy.optimize.leastsq approximates the Jacobian with finite differences 14 | 2) when the Jacobian is provided to scipy.optimize.leastsq 15 | 16 | As output one obtains:: 17 | 18 | $ python leastsquaresfitting.py 19 | Estimates from leastsq 20 | [ 6.79548889e-02 3.68922501e-01 7.55565769e-02 1.41378227e+02 21 | 2.91307741e+00 2.70608242e+02] 1 22 | number of function calls = 26 23 | Estimates from leastsq 24 | [ 6.79548883e-02 3.68922503e-01 7.55565728e-02 1.41378227e+02 25 | 2.91307814e+00 2.70608242e+02] 1 26 | number of function calls = 140 27 | 28 | -------------------------------------------------------------------------------- /documentation/sphinx/getting_started.py: -------------------------------------------------------------------------------- 1 | import numpy, algopy 2 | from algopy import UTPM, exp 3 | 4 | def eval_f(x): 5 | """ some function """ 6 | return x[0]*x[1]*x[2] + exp(x[0])*x[1] 7 | 8 | # forward mode without building the computational graph 9 | # ----------------------------------------------------- 10 | x = UTPM.init_jacobian([3,5,7]) 11 | y = eval_f(x) 12 | algopy_jacobian = UTPM.extract_jacobian(y) 13 | print('jacobian = ',algopy_jacobian) 14 | 15 | # reverse mode using a computational graph 16 | # ---------------------------------------- 17 | 18 | # STEP 1: trace the function evaluation 19 | cg = algopy.CGraph() 20 | x = algopy.Function([1,2,3]) 21 | y = eval_f(x) 22 | cg.trace_off() 23 | cg.independentFunctionList = [x] 24 | cg.dependentFunctionList = [y] 25 | 26 | # STEP 2: use the computational graph to evaluate derivatives 27 | print('gradient =', cg.gradient([3.,5,7])) 28 | print('Jacobian =', cg.jacobian([3.,5,7])) 29 | print('Hessian =', cg.hessian([3.,5.,7.])) 30 | print('Hessian vector product =', cg.hess_vec([3.,5.,7.],[4,5,6])) 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /documentation/ICCS2010/prettyplotting.py: -------------------------------------------------------------------------------- 1 | 2 | import pylab 3 | 4 | #SETUP PLOTTING PARAMETERS 5 | fig_width_pt = 600.0 # Get this from LaTeX using \showthe\columnwidth 6 | inches_per_pt = 1.0/72.25 # Convert pt to inches 7 | golden_mean = (pylab.sqrt(5)-1.0)/2.0 # Aesthetic ratio 8 | fig_width = fig_width_pt*inches_per_pt # width in inches 9 | fig_height =fig_width*golden_mean # height in inches 10 | fig_size = [fig_width,fig_height] 11 | legend_padding = 0.05 12 | params = { 13 | 'backend': 'ps', 14 | 'ps.usedistiller': 'xpdf', 15 | 'font.family' : 'serif',#'sans-serif', 16 | 'font.style' : 'normal', 17 | 'font.variant' : 'normal', 18 | 'font.weight' : 'normal', #bold 19 | 'font.stretch' : 'normal', 20 | # 'font.size' : 'normal', #large 21 | # 'axes.labelsize': 11, 22 | # 'text.fontsize': 12, 23 | # 'title.fontsize':12, 24 | # 'legend.fontsize':12, 25 | # 'xtick.labelsize': 14, 26 | # 'ytick.labelsize': 14, 27 | # 'lines.markersize':10, 28 | 'text.usetex': True, 29 | 'figure.figsize': fig_size 30 | } 31 | pylab.rcParams.update(params) 32 | -------------------------------------------------------------------------------- /documentation/sphinx/speed_comparison/prettyplotting.py: -------------------------------------------------------------------------------- 1 | 2 | import pylab 3 | 4 | #SETUP PLOTTING PARAMETERS 5 | fig_width_pt = 500.0 # Get this from LaTeX using \showthe\columnwidth 6 | inches_per_pt = 1.0/72.25 # Convert pt to inches 7 | golden_mean = (pylab.sqrt(5)-1.0)/2.0 # Aesthetic ratio 8 | fig_width = fig_width_pt*inches_per_pt # width in inches 9 | fig_height =fig_width*golden_mean # height in inches 10 | fig_size = [fig_width,fig_height] 11 | legend_padding = 0.05 12 | params = { 13 | 'backend': 'ps', 14 | 'ps.usedistiller': 'xpdf', 15 | 'font.family' : 'serif',#'sans-serif', 16 | 'font.style' : 'normal', 17 | 'font.variant' : 'normal', 18 | 'font.weight' : 'normal', #bold 19 | 'font.stretch' : 'normal', 20 | #'font.size' : 'large', #large,'normal' 21 | 'axes.labelsize': 16, 22 | 'text.fontsize': 14, 23 | 'title.fontsize':12, 24 | 'legend.fontsize':14, 25 | 'xtick.labelsize': 16, 26 | 'ytick.labelsize': 16, 27 | 'lines.markersize':8, 28 | 'text.usetex': True, 29 | 'figure.figsize': fig_size 30 | } 31 | pylab.rcParams.update(params) 32 | -------------------------------------------------------------------------------- /experimental/ctps/SConstruct.EXAMPLE: -------------------------------------------------------------------------------- 1 | import distutils.sysconfig 2 | import numpy 3 | 4 | 5 | LIBS = [] 6 | INCLUDEPATH = [] 7 | LIBPATH = [] 8 | 9 | # 0: setup the command line parsing 10 | AddOption('--prefix', 11 | dest='prefix', 12 | nargs=1, type='string', 13 | action='store', 14 | metavar='DIR', 15 | help='installation prefix') 16 | 17 | env = Environment( 18 | PREFIX = GetOption('prefix'), 19 | TMPBUILD = '/tmp/builddir', 20 | CPPPATH=[distutils.sysconfig.get_python_inc(),numpy.get_include()] + INCLUDEPATH, 21 | CXXFLAGS="-ftemplate-depth-100 -pthread -fno-strict-aliasing -DNDEBUG -fwrapv -O3 -Wfatal-errors", 22 | LIBPATH= LIBPATH, 23 | LIBS= LIBS, 24 | RPATH = LIBPATH, #include information where shared libraries can be found to avoid errors like: "ImportError: libboost_python-gcc42-mt-1_34_1.so.1.34.1: cannot open shared object file: No such file or directory" 25 | SHLIBPREFIX="lib", #gets rid of lib prefix 26 | ) 27 | env.Append(LINKFLAGS = "-pthread") 28 | 29 | Export('env') 30 | SConscript('algopy/ctps/SConscript') 31 | -------------------------------------------------------------------------------- /documentation/AD_tutorial_TU_Berlin/prettyplotting.py: -------------------------------------------------------------------------------- 1 | 2 | import pylab 3 | 4 | #SETUP PLOTTING PARAMETERS 5 | fig_width_pt = 600.0 # Get this from LaTeX using \showthe\columnwidth 6 | inches_per_pt = 1.0/72.25 # Convert pt to inches 7 | golden_mean = (pylab.sqrt(5)-1.0)/2.0 # Aesthetic ratio 8 | fig_width = fig_width_pt*inches_per_pt # width in inches 9 | fig_height =fig_width*golden_mean # height in inches 10 | fig_size = [fig_width,fig_height] 11 | legend_padding = 0.05 12 | params = { 13 | 'backend': 'ps', 14 | 'ps.usedistiller': 'xpdf', 15 | 'font.family' : 'serif',#'sans-serif', 16 | 'font.style' : 'normal', 17 | 'font.variant' : 'normal', 18 | 'font.weight' : 'normal', #bold 19 | 'font.stretch' : 'normal', 20 | # 'font.size' : 'normal', #large 21 | # 'axes.labelsize': 11, 22 | # 'text.fontsize': 12, 23 | # 'title.fontsize':12, 24 | # 'legend.fontsize':12, 25 | # 'xtick.labelsize': 14, 26 | # 'ytick.labelsize': 14, 27 | # 'lines.markersize':10, 28 | 'text.usetex': True, 29 | 'figure.figsize': fig_size 30 | } 31 | pylab.rcParams.update(params) 32 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/minimization/wood_function_minimization.py: -------------------------------------------------------------------------------- 1 | """ 2 | Minimize the Wood function. 3 | 4 | Problem 3.1 of 5 | "A truncated Newton method with nonmonotone line search 6 | for unconstrained optimization" 7 | Except that I think there is a typo in that paper, 8 | and the minimum is actually at (1,1,1,1) rather than at (0,0,0,0). 9 | """ 10 | 11 | import numpy 12 | 13 | import minhelper 14 | 15 | 16 | def wood(X): 17 | """ 18 | This R^4 -> R^1 function should be compatible with algopy. 19 | """ 20 | x1 = X[0] 21 | x2 = X[1] 22 | x3 = X[2] 23 | x4 = X[3] 24 | return sum(( 25 | 100*(x1*x1 - x2)**2, 26 | (x1-1)**2, 27 | (x3-1)**2, 28 | 90*(x3*x3 - x4)**2, 29 | 10.1*((x2-1)**2 + (x4-1)**2), 30 | 19.8*(x2-1)*(x4-1), 31 | )) 32 | 33 | 34 | def main(): 35 | target = [1, 1, 1, 1] 36 | easy_init = [1.1, 1.2, 1.3, 1.4] 37 | hard_init = [-3, -1, -3, -1] 38 | minhelper.show_minimization_results( 39 | wood, target, easy_init, hard_init) 40 | 41 | 42 | if __name__ == '__main__': 43 | main() 44 | 45 | -------------------------------------------------------------------------------- /algopy/tracer/tests/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This submodule implements a simplistic code tracer that is suitable to trace 3 | array operations as they occur in codes using NumPy. 4 | 5 | The code tracer saves the sequence of operations in a computational graph, 6 | where each node has a reference to its parents. The idea is to create an instance of algopy.CGraph which registers itself in a static variable in the algopy.Function class. Then wrap objects of interest in an algopy.Function instance. Performing operations on the Function instances are recorded in 7 | the computational graph. 8 | 9 | 10 | Example: 11 | -------- 12 | 13 | >>> import algopy 14 | >>> cg = algopy.CGraph() 15 | >>> x = algopy.Function(2) 16 | >>> y = algopy.Function(3) 17 | >>> z = x * y 18 | >>> print cg 19 | 20 | Id: IDs: 0 <- [0] 21 | x: 22 | 2 23 | 24 | 25 | Id: IDs: 1 <- [1] 26 | x: 27 | 3 28 | 29 | 30 | __mul__: IDs: 2 <- [0, 1] 31 | x: 32 | 6 33 | 34 | Independent Function List: 35 | [] 36 | 37 | Dependent Function List: 38 | [] 39 | """ 40 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/logistic_regression.rst: -------------------------------------------------------------------------------- 1 | Logistic Regression 2 | ---------------------------- 3 | 4 | In this example we want to use AlgoPy to help compute the 5 | maximum likelihood estimates for a nonlinear model. 6 | It is based on this 7 | `question `_ 8 | on the scicomp stackexchange. 9 | 10 | 11 | Here is the python code: 12 | 13 | .. literalinclude:: logistic_regression.py 14 | 15 | 16 | Here is its output:: 17 | 18 | hardcoded good values: 19 | [-0.10296645 -0.0332327 -0.01209484 0.44626211 0.92554137 0.53973828 20 | 1.7993371 0.7148045 ] 21 | 22 | neg log likelihood for good values: 23 | 102.173732637 24 | 25 | 26 | hardcoded okay values: 27 | [-0.1 -0.03 -0.01 0.44 0.92 0.53 1.8 0.71] 28 | 29 | neg log likelihood for okay values: 30 | 104.084160515 31 | 32 | 33 | maximum likelihood estimates: 34 | [-0.10296655 -0.0332327 -0.01209484 0.44626209 0.92554133 0.53973824 35 | 1.79933696 0.71480445] 36 | 37 | neg log likelihood for maximum likelihood estimates: 38 | 102.173732637 39 | 40 | -------------------------------------------------------------------------------- /experimental/pullback_alternative.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | def mul(D,x_data, y_data, z_data): 4 | for d in range(D): 5 | for k in range(d+1): 6 | z_data[d] += x_data[k] * y_data[d-k] 7 | 8 | return z_data 9 | 10 | def pbr_mul(D, x_data, y_data, z_data, zbar_data, xbar_data, ybar_data): 11 | for d in range(D): 12 | for k in range(D-d): 13 | xbar_data[d] += zbar_data[d+k] * y_data[k] 14 | ybar_data[d] += zbar_data[d+k] * x_data[k] 15 | 16 | return (xbar_data, ybar_data) 17 | 18 | 19 | if __name__ == "__main__": 20 | """ compute z = x*x*x 21 | """ 22 | 23 | x = numpy.array([5.,1.]) 24 | 25 | # forward sweep 26 | y = mul(2, x, x, numpy.zeros(2)) 27 | z = mul(2, y, x, numpy.zeros(2)) 28 | 29 | # reverse sweep 30 | zbar = numpy.array([0.,1.]) 31 | ybar = numpy.zeros(2) 32 | xbar = numpy.zeros(2) 33 | 34 | pbr_mul(2, y, x, z, zbar, ybar, xbar) 35 | pbr_mul(2, x, x, y, ybar, xbar, xbar) 36 | 37 | print(x) 38 | print(xbar) 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /algopy/fft/fft.py: -------------------------------------------------------------------------------- 1 | import string 2 | import numpy 3 | import numpy.linalg 4 | import scipy.linalg 5 | 6 | from algopy import UTPM, Function 7 | 8 | 9 | def fft(a, n=None, axis=-1): 10 | """ 11 | 12 | equivalent to numpy.fft.fft(a, n=None, axis=-1) 13 | 14 | """ 15 | 16 | if isinstance(a, UTPM): 17 | return UTPM.fft(a, n=n, axis=axis) 18 | 19 | elif isinstance(a, Function): 20 | return Function.fft(a, n=n, axis=axis) 21 | 22 | elif isinstance(a, numpy.ndarray): 23 | return numpy.fft.fft(a, n=n, axis=axis) 24 | 25 | else: 26 | raise NotImplementedError('don\'t know what to do with this instance') 27 | 28 | 29 | def ifft(a, n=None, axis=-1): 30 | """ 31 | 32 | equivalent to numpy.fft.ifft(a, n=None, axis=-1) 33 | 34 | """ 35 | 36 | if isinstance(a, UTPM): 37 | return UTPM.ifft(a, n=n, axis=axis) 38 | 39 | elif isinstance(a, Function): 40 | return Function.ifft(a, n=n, axis=axis) 41 | 42 | elif isinstance(a, numpy.ndarray): 43 | return numpy.fft.ifft(a, n=n, axis=axis) 44 | 45 | else: 46 | raise NotImplementedError('don\'t know what to do with this instance') 47 | -------------------------------------------------------------------------------- /algopy/compound.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains functions like 3 | 4 | algopy.prod 5 | 6 | that are not represented as a single node in the 7 | computational graph, but are treated as a **compound** 8 | function. I.e., tracing algopy.prod will result 9 | in a CGraph with many successive multiplication operations. 10 | 11 | 12 | Note 13 | ---- 14 | 15 | These functions should be replaced by a dedicated implementation in 16 | 17 | * algopy.Function 18 | * algopy.UTPM 19 | 20 | so they are represented by a single node in the CGraph. 21 | 22 | """ 23 | 24 | import numpy 25 | from algopy import zeros, Function, UTPM 26 | 27 | # def prod(x, axis=None, dtype=None, out=None): 28 | # """ 29 | # generic prod function 30 | # """ 31 | 32 | # if axis is not None or dtype is not None or out is not None: 33 | # raise NotImplementedError('') 34 | 35 | # elif isinstance(x, numpy.ndarray): 36 | # return numpy.prod(x) 37 | 38 | # elif isinstance(x, Function) or isinstance(x, UTPM): 39 | # y = zeros(1,dtype=x) 40 | # y[0] = x[0] 41 | # for xi in x[1:]: 42 | # y[0] = y[0] * xi 43 | # return y[0] 44 | 45 | # else: 46 | # raise ValueError('don\'t know what to do with this input!') 47 | 48 | # prod.__doc__ += numpy.prod.__doc__ 49 | -------------------------------------------------------------------------------- /algopy/tests/test_compound.py: -------------------------------------------------------------------------------- 1 | from numpy.testing import * 2 | from unittest import TestCase 3 | 4 | import numpy 5 | numpy.random.seed(0) 6 | 7 | from algopy import UTPM, Function, CGraph, sum, zeros, diag, dot, qr 8 | from algopy.linalg.compound import expm 9 | 10 | class Test_NumpyScipyLinalgFunctions(TestCase): 11 | 12 | 13 | def test_expm(self): 14 | 15 | def f(x): 16 | x = x.reshape((2,2)) 17 | return sum(expm(x)) 18 | 19 | x = numpy.random.random(2*2) 20 | 21 | 22 | # forward mode 23 | 24 | ax = UTPM.init_jacobian(x) 25 | ay = f(ax) 26 | g1 = UTPM.extract_jacobian(ay) 27 | 28 | # reverse mode 29 | 30 | cg = CGraph() 31 | ax = Function(x) 32 | ay = f(ax) 33 | cg.independentFunctionList = [ax] 34 | cg.dependentFunctionList = [ay] 35 | 36 | g2 = cg.gradient(x) 37 | 38 | assert_array_almost_equal(g1, g2) 39 | 40 | def test_utpm_logdet_trace_expm(self): 41 | D, P, N = 3, 5, 4 42 | 43 | x = 0.1 * UTPM(numpy.random.randn(D, P, N, N)) 44 | x = UTPM.dot(x.T, x) 45 | observed_logdet = UTPM.logdet(expm(x)) 46 | desired_logdet = UTPM.trace(x) 47 | assert_allclose(observed_logdet.data, desired_logdet.data) 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/first_order_forward.py: -------------------------------------------------------------------------------- 1 | import numpy; from numpy import array 2 | from algopy import UTPM, zeros 3 | 4 | def F(x): 5 | y = zeros(3, dtype=x) 6 | y[0] = x[0]*x[1] 7 | y[1] = x[1]*x[2] 8 | y[2] = x[2]*x[0] 9 | return y 10 | 11 | x0 = array([1,3,5],dtype=float) 12 | x1 = array([1,0,0],dtype=float) 13 | 14 | D = 2; P = 1 15 | x = UTPM(numpy.zeros((D,P,3))) 16 | x.data[0,0,:] = x0 17 | x.data[1,0,:] = x1 18 | 19 | # normal function evaluation 20 | y0 = F(x0) 21 | 22 | # UTP function evaluation 23 | y = F(x) 24 | 25 | print('y0 = ', y0) 26 | print('y = ', y) 27 | print('y.shape =', y.shape) 28 | print('y.data.shape =', y.data.shape) 29 | print('dF/dx(x0) * x1 =', y.data[1,0]) 30 | 31 | 32 | 33 | import numpy; from numpy import log, exp, sin, cos, abs 34 | import algopy; from algopy import UTPM, dot, inv, zeros 35 | 36 | def f(x): 37 | A = zeros((2,2),dtype=x) 38 | A[0,0] = numpy.log(x[0]*x[1]) 39 | A[0,1] = numpy.log(x[1]) + exp(x[0]) 40 | A[1,0] = sin(x[0])**2 + abs(cos(x[0]))**3.1 41 | A[1,1] = x[0]**cos(x[1]) 42 | return log( dot(x.T, dot( inv(A), x))) 43 | 44 | x = numpy.array([3.,7.]) 45 | x = UTPM.init_jacobian(x) 46 | 47 | y = f(x) 48 | 49 | print('normal function evaluation f(x) = ',y.data[0,0]) 50 | print('Jacobian df/dx = ', UTPM.extract_jacobian(y)) 51 | 52 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/leastsquaresfitting.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from scipy import optimize 3 | 4 | import algopy 5 | 6 | ## This is y-data: 7 | y_data = numpy.array([0.2867, 0.1171, -0.0087, 0.1326, 0.2415, 0.2878, 0.3133, 0.3701, 0.3996, 0.3728, 0.3551, 0.3587, 0.1408, 0.0416, 0.0708, 0.1142, 0, 0, 0]) 8 | 9 | ## This is x-data: 10 | t = numpy.array([67., 88, 104, 127, 138, 160, 169, 188, 196, 215, 240, 247, 271, 278, 303, 305, 321, 337, 353]) 11 | 12 | def fitfunc(p, t): 13 | """This is the equation""" 14 | return p[0] + (p[1] -p[0]) * ((1/(1+algopy.exp(-p[2]*(t-p[3])))) + (1/(1+algopy.exp(p[4]*(t-p[5])))) -1) 15 | 16 | def errfunc(p, t, y): 17 | return fitfunc(p,t) -y 18 | 19 | def jac_errfunc(p, t, y): 20 | ap = algopy.UTPM.init_jacobian(p) 21 | return algopy.UTPM.extract_jacobian(errfunc(ap, t, y)) 22 | 23 | guess = numpy.array([0, max(y_data), 0.1, 140, -0.1, 270]) 24 | p2, C, info, msg, success = optimize.leastsq(errfunc, guess, args=(t, y_data), Dfun = jac_errfunc, full_output=1) 25 | print('Estimates from leastsq \n', p2,success) 26 | print('number of function calls =', info['nfev']) 27 | 28 | p3, C, info, msg, success = optimize.leastsq(errfunc, guess, args=(t, y_data), full_output=1) 29 | print('Estimates from leastsq \n', p3,success) 30 | print('number of function calls =', info['nfev']) 31 | 32 | -------------------------------------------------------------------------------- /experimental/sequential_tape_prototype.py: -------------------------------------------------------------------------------- 1 | from numpy import * 2 | N = 10 3 | M = 3 4 | x = zeros(N) 5 | 6 | ops = { 7 | 'imul': 1, 8 | } 9 | 10 | 11 | class BB_tape: 12 | def __init__(self, bb_id): 13 | self.id = bb_id 14 | self.op_tape = [] 15 | self.loc_tape = [] 16 | def __str__(self): 17 | return str(self.op_tape) + str(self.loc_tape) 18 | 19 | def __repr__(self): 20 | return self.__str__() 21 | 22 | i = 0 23 | curr_bb = 0 24 | bb_tapes = [BB_tape(i) for i in range(10)] 25 | 26 | 27 | #class BB: 28 | #def __init__(self, id): 29 | #global curr_bb 30 | #self.id = id 31 | #curr_bb = id 32 | #def __str__(self): 33 | #return 'b'+str(self.id) 34 | #def __repr__(self): 35 | #return self.__str__() 36 | 37 | class adouble: 38 | def __init__(self,x): 39 | global i 40 | self.x = x 41 | self.id = i 42 | i += 1 43 | def __str__(self): 44 | return str(self.id) 45 | 46 | def __imul__(self,rhs): 47 | bb_tapes[curr_bb].op_tape.append(ops['imul']) 48 | bb_tapes[curr_bb].loc_tape.append((self.id, rhs.id)) 49 | self.x *= rhs.x 50 | return self 51 | 52 | ax = adouble(2.) 53 | ay = adouble(2.) 54 | 55 | 56 | for n in range(N): 57 | curr_bb = 0 58 | ax *= ay 59 | 60 | print(bb_tapes) 61 | -------------------------------------------------------------------------------- /documentation/AD_tutorial_TU_Berlin/example7_simple_computation_of_the_hessian.py: -------------------------------------------------------------------------------- 1 | """ 2 | compute the Hessian H of the function 3 | def f(x): 4 | return sin(x[0] + cos(x[1])*x[0]) 5 | at x = (3,7) 6 | """ 7 | 8 | import numpy; from numpy import sin,cos, array, zeros 9 | from taylorpoly import UTPS 10 | def f_fcn(x): 11 | return sin(x[0] + cos(x[1])*x[0]) 12 | 13 | S = array([[1,0,1],[0,1,1]], dtype=float) 14 | P = S.shape[1] 15 | print('seed matrix with P = %d directions S = \n'%P, S) 16 | x1 = UTPS(zeros(1+2*P), P = P) 17 | x2 = UTPS(zeros(1+2*P), P = P) 18 | x1.data[0] = 3; x1.data[1::2] = S[0,:] 19 | x2.data[0] = 7; x2.data[1::2] = S[1,:] 20 | y = f_fcn([x1,x2]) 21 | print('x1=',x1); print('x2=',x2); print('y=',y) 22 | H = zeros((2,2),dtype=float) 23 | H[0,0] = 2*y.coeff[0,2] 24 | H[1,0] = H[0,1] = (y.coeff[2,2] - y.coeff[0,2] - y.coeff[1,2]) 25 | H[1,1] = 2*y.coeff[1,2] 26 | 27 | def H_fcn(x): 28 | H11 = -(1+cos(x[1]))**2*sin(x[0]+cos(x[1])*x[0]) 29 | H21 = -sin(x[1]) * cos(x[0] + cos(x[1])*x[0]) \ 30 | +sin(x[1]) *x[0]*(1+ cos(x[1]))*sin(x[0]+cos(x[1])*x[0]) 31 | H22 = -cos(x[1])*x[0]*cos(x[0]+cos(x[1])*x[0])\ 32 | -(sin(x[1])*x[0])**2*sin(x[0]+cos(x[1])*x[0]) 33 | return array([[H11, H21],[H21,H22]]) 34 | 35 | print('symbolic Hessian - AD Hessian = \n', H - H_fcn([3,7])) 36 | print('exact interpolation Hessian H(x_0) = \n', H) 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /documentation/sphinx/examples_tracer.rst: -------------------------------------------------------------------------------- 1 | The Code Tracer 2 | ================ 3 | 4 | ALGOPY features a simple code tracer. A trace of the executed code is necessary 5 | for the reverse mode of AD. The basic idea is to wrap an object `x` in an 6 | `algopy.Function` instance. Performing operations with `Function` instances 7 | are recorded in a computational graph, implemented in `algopy.CGraph`. 8 | 9 | For example:: 10 | 11 | >>> import numpy 12 | >>> from algopy import CGraph, Function 13 | >>> cg = CGraph() 14 | >>> cg.trace_on() 15 | >>> x = Function(1) 16 | >>> y = Function(3) 17 | >>> z = x * y + x 18 | >>> cg.trace_off() 19 | >>> cg.independentFunctionList = [x,y] 20 | >>> cg.dependentFunctionList = [z] 21 | >>> print cg 22 | >>> cg.plot('example_tracer_cgraph.png') 23 | 24 | 25 | 26 | Id: IDs: 0 <- [0] 27 | x: 28 | 1 29 | 30 | 31 | Id: IDs: 1 <- [1] 32 | x: 33 | 3 34 | 35 | 36 | __mul__: IDs: 2 <- [0, 1] 37 | x: 38 | 3 39 | 40 | 41 | __add__: IDs: 3 <- [2, 0] 42 | x: 43 | 4 44 | 45 | Independent Function List: 46 | [0, 1] 47 | 48 | Dependent Function List: 49 | [3] 50 | 51 | 52 | and the plotted computational graph is: 53 | 54 | .. image:: example_tracer_cgraph.png 55 | :align: center 56 | :scale: 100 57 | 58 | 59 | -------------------------------------------------------------------------------- /algopy/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from numpy.testing import * 2 | import numpy 3 | import numpy.random 4 | numpy.random.seed(0) 5 | 6 | from algopy.utpm import UTPM 7 | from algopy.utils import * 8 | 9 | class TestUtils ( TestCase ): 10 | 11 | def test_utpm2base_and_dirs(self): 12 | D,P,N,M,K = 2,3,4,5,6 13 | u = UTPM( numpy.arange(D*P*N*M*K).reshape((D,P,N,M,K))) 14 | x,V = utpm2base_and_dirs(u) 15 | assert_array_almost_equal(x, numpy.arange(N*M*K).reshape((N,M,K))) 16 | assert_array_almost_equal(V, numpy.arange(P*N*M*K,D*P*N*M*K).reshape((D-1,P,N,M,K)).transpose((2,3,4,1,0))) 17 | 18 | def test_utpm2dirs(self): 19 | D,P,N,M,K = 2,3,4,5,6 20 | u = UTPM( numpy.arange(D*P*N*M*K).reshape((D,P,N,M,K))) 21 | Vbar = utpm2dirs(u) 22 | assert_array_almost_equal(Vbar, numpy.arange(D*P*N*M*K).reshape((D,P,N,M,K)).transpose((2,3,4,1,0)) ) 23 | 24 | 25 | def test_piv2mat(self): 26 | correct = numpy.array([[0,1,0,0,0], 27 | [0,0,0,0,1], 28 | [1,0,0,0,0], 29 | [0,0,1,0,0], 30 | [0,0,0,1,0]],dtype=float) 31 | piv = [2,2,3,4,4] 32 | assert_array_almost_equal(correct, piv2mat(piv)) 33 | 34 | def test_piv2det(self): 35 | piv = [2,2,3,4,4] 36 | assert_array_almost_equal(1, piv2det(piv)) 37 | 38 | -------------------------------------------------------------------------------- /documentation/sphinx/datastructure_and_algorithms.rst: -------------------------------------------------------------------------------- 1 | Datastructure and Algorithms 2 | ---------------------------- 3 | 4 | The mathematical object is a Univariate Taylor Polynomial over Matrices (UTPM) 5 | 6 | .. math:: 7 | [x]_D = [x_0, \dots, x_{D-1}] = \sum_{d=0}^{D-1} x_d T^d \;, 8 | 9 | where each :math:`x_d` is some array, e.g. a (5,7) array. 10 | This mathematical object is described by numpy.ndarray with shape (D,P, 5,7). 11 | P>1 triggers a vectorized function evaluation. 12 | 13 | All algorithms are implemented 14 | in the following fashion:: 15 | 16 | def add(x_data, y_data, z_data): 17 | z_data[...] = x_data[...] + y_data[...] 18 | 19 | where the inputs `x_data,y_data` are numpy.ndarray's and `add` changes the elements 20 | of the numpy.ndarray z. I.e., the algorithms are implemented in a similar way as LAPACK or Fortran functions 21 | in general. One can find the UTPM algorithms in `algopy/utpm/algorithms.py` where they are class functions 22 | of a mixin class. 23 | 24 | In practice, working with such algorithms is cumbersome. ALGOPY therefore also 25 | offers the class `algopy.UTPM` which is a thin wrapper around the algorithms 26 | and provides overloaded functions and operators. The data is saved in the attribute UTPM.data. 27 | 28 | The following code shows how to use the algorithms 29 | directly and using the syntactic sugar provided by the UTPM class. 30 | 31 | .. literalinclude:: datastructure_and_algorithms.py 32 | 33 | 34 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/explicit_euler.py: -------------------------------------------------------------------------------- 1 | import numpy; from numpy import sin,cos; from algopy import UTPM, zeros 2 | D,P = 4,1 3 | x = UTPM(numpy.zeros((D,P,2))) 4 | x.data[0,:,0] = 1 5 | p = UTPM(numpy.zeros((D,P))) 6 | p.data[0,:] = 3; p.data[1,:] = 1 7 | 8 | def f(x): 9 | retval = x.zeros_like() 10 | retval[0] = x[1] 11 | retval[1] = -p* x[0] 12 | return retval 13 | 14 | # compute AD solution 15 | ts = numpy.linspace(0,2*numpy.pi,2000) 16 | x_list = [x.data.copy() ] 17 | for nts in range(ts.size-1): 18 | h = ts[nts+1] - ts[nts] 19 | x = x + h * f(x) 20 | x_list.append(x.data.copy()) 21 | 22 | # analytical solution 23 | def x_analytical(t,p): 24 | return numpy.cos(numpy.sqrt(p)*t) 25 | 26 | def x_p_analytical(t,p): 27 | return -0.5*numpy.sin(numpy.sqrt(p)*t)*p**(-0.5)*t 28 | 29 | xs = numpy.array(x_list) 30 | print(xs.shape) 31 | import matplotlib.pyplot as pyplot; import os 32 | pyplot.plot(ts, xs[:,0,0,0], ',k-', label = r'$x(t)$') 33 | pyplot.plot(ts, x_analytical(ts,p.data[0,0]), 'k-.', label = r'analytic $x(t)$') 34 | pyplot.plot(ts, xs[:,1,0,0], ',r-', label = r'$x_p(t)$') 35 | pyplot.plot(ts, x_p_analytical(ts,p.data[0,0]), 'r-.', label = r'analytic $x_p(t)$') 36 | pyplot.plot(ts, xs[:,2,0,0], ',b-', label = r'$x_{pp}(t)$') 37 | pyplot.plot(ts, xs[:,3,0,0], ',m-', label = r'$x_{ppp}(t)$') 38 | 39 | 40 | 41 | pyplot.xlabel('time $t$') 42 | pyplot.legend(loc='best') 43 | pyplot.grid() 44 | pyplot.savefig(os.path.join(os.path.dirname(os.path.realpath(__file__)),'explicit_euler.png')) 45 | # pyplot.show() 46 | 47 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/posterior_log_probability.py: -------------------------------------------------------------------------------- 1 | import numpy; from algopy import UTPM, zeros, Function, CGraph; import os 2 | 3 | def LogNormalLikelihood(x, mu, sigma): 4 | return sigma *(x - mu)**2 - numpy.log(.5*sigma/numpy.pi) 5 | 6 | def logp(x, mu, sigma): 7 | return numpy.sum(LogNormalLikelihood(x, mu, sigma)) + LogNormalLikelihood(mu , mu_prior_mean, mu_prior_sigma) 8 | 9 | mu_prior_mean = 1 10 | mu_prior_sigma = 5 11 | 12 | actual_mu = 3.1 13 | sigma = 1.2 14 | N = 35 15 | x = numpy.random.normal(actual_mu, sigma, size = N) 16 | mu = UTPM([[3.5],[1],[0]]) #unknown variable 17 | 18 | print('function evaluation =\n',logp(x,3.5,sigma)) 19 | 20 | # forward mode with ALGOPY 21 | utp = logp(x, mu, sigma).data[:,0] 22 | print('function evaluation = %f\n1st directional derivative = %f\n2nd directional derivative = %f'%(utp[0], 1.*utp[1], 2.*utp[2])) 23 | 24 | # finite differences solution: 25 | print('finite differences derivative =\n',(logp(x,3.5+10**-8,sigma) - logp(x, 3.5, sigma))/10**-8) 26 | 27 | # trace function evaluation 28 | cg = CGraph() 29 | mu = Function(UTPM([[3.5],[1],[0]])) #unknown variable 30 | out = logp(x, mu, sigma) 31 | cg.trace_off() 32 | cg.independentFunctionList = [mu] 33 | cg.dependentFunctionList = [out] 34 | cg.plot(os.path.join(os.path.dirname(os.path.realpath(__file__)),'posterior_log_probability_cgraph.png')) 35 | 36 | # reverse mode with ALGOPY 37 | outbar = UTPM([[1.],[0],[0]]) 38 | cg.pullback([outbar]) 39 | 40 | gradient = mu.xbar.data[0,0] 41 | Hess_vec = mu.xbar.data[1,0] 42 | 43 | print('gradient = ', gradient) 44 | print('Hessian vector product = ', Hess_vec) 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /algopy/tests/test_npversion.py: -------------------------------------------------------------------------------- 1 | from numpy.testing import assert_, assert_raises 2 | from algopy._npversion import NumpyVersion 3 | 4 | def test_main_versions(): 5 | assert_(NumpyVersion('1.8.0') == '1.8.0') 6 | for ver in ['1.9.0', '2.0.0', '1.8.1']: 7 | assert_(NumpyVersion('1.8.0') < ver) 8 | 9 | for ver in ['1.7.0', '1.7.1', '0.9.9']: 10 | assert_(NumpyVersion('1.8.0') > ver) 11 | 12 | def test_version_1_point_10(): 13 | # regression test for gh-2998. 14 | assert_(NumpyVersion('1.9.0') < '1.10.0') 15 | assert_(NumpyVersion('1.11.0') < '1.11.1') 16 | assert_(NumpyVersion('1.11.0') == '1.11.0') 17 | assert_(NumpyVersion('1.99.11') < '1.99.12') 18 | 19 | def test_alpha_beta_rc(): 20 | assert_(NumpyVersion('1.8.0rc1') == '1.8.0rc1') 21 | for ver in ['1.8.0', '1.8.0rc2']: 22 | assert_(NumpyVersion('1.8.0rc1') < ver) 23 | 24 | for ver in ['1.8.0a2', '1.8.0b3', '1.7.2rc4']: 25 | assert_(NumpyVersion('1.8.0rc1') > ver) 26 | 27 | assert_(NumpyVersion('1.8.0b1') > '1.8.0a2') 28 | 29 | 30 | def test_dev_version(): 31 | assert_(NumpyVersion('1.9.0.dev-Unknown') < '1.9.0') 32 | for ver in ['1.9.0', '1.9.0a1', '1.9.0b2', '1.9.0b2.dev-ffffffff']: 33 | assert_(NumpyVersion('1.9.0.dev-f16acvda') < ver) 34 | 35 | assert_(NumpyVersion('1.9.0.dev-f16acvda') == '1.9.0.dev-11111111') 36 | 37 | 38 | def test_dev_a_b_rc_mixed(): 39 | assert_(NumpyVersion('1.9.0a2.dev-f16acvda') == '1.9.0a2.dev-11111111') 40 | assert_(NumpyVersion('1.9.0a2.dev-6acvda54') < '1.9.0a2') 41 | 42 | 43 | def test_raises(): 44 | for ver in ['1.9', '1,9.0', '1.7.x']: 45 | assert_raises(ValueError, NumpyVersion, ver) 46 | 47 | 48 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/polarization.py: -------------------------------------------------------------------------------- 1 | import algopy, numpy 2 | import algopy.exact_interpolation as ei 3 | 4 | def eval_F(x): 5 | retval = algopy.zeros(3, dtype=x) 6 | retval[0] = x[0]*x[1] 7 | retval[1] = x[1]*x[2] 8 | retval[2] = x[0] - x[1] 9 | return retval 10 | 11 | D,M,N = 3,3,3 12 | P = 2*M*N 13 | 14 | # STEP 1: generate all necessary directions 15 | Id = numpy.eye(N) 16 | dirs = numpy.zeros((P, N)) 17 | 18 | count = 0 19 | for n1 in range(N): 20 | for n2 in range(N): 21 | dirs[count] += Id[n1] 22 | dirs[count] += Id[n2] 23 | 24 | dirs[count + 1] += Id[n1] 25 | dirs[count + 1] -= Id[n2] 26 | 27 | count += 2 28 | 29 | print('dirs =') 30 | print(dirs) 31 | 32 | # STEP 2: use these directions to initialize the UTPM instance 33 | xdata = numpy.zeros((D,2*N*N,N)) 34 | xdata[0] = [1,2,3] 35 | xdata[1] = dirs 36 | 37 | # STEP 3: compute function F in UTP arithmetic 38 | x = algopy.UTPM(xdata) 39 | y = eval_F(x) 40 | 41 | # STEP 4: use polarization identity to build univariate Taylor polynomial 42 | # of the Jacobian J 43 | Jdata = numpy.zeros((D-1, N, N, N)) 44 | count, count2 = 0,0 45 | 46 | # build J_0 47 | for n in range(N): 48 | Jdata[0,:,:,n] = y.data[1, 2*n*(N+1), :]/2. 49 | 50 | # build J_1 51 | count = 0 52 | for n1 in range(N): 53 | for n2 in range(N): 54 | Jdata[1,n2,:,n1] = 0.5*( y.data[2, count] - y.data[2, count+1]) 55 | count += 2 56 | 57 | # initialize UTPM instance 58 | J = algopy.UTPM(Jdata) 59 | 60 | # STEP 5: evaluate Phi in UTP arithmetic 61 | Phi = algopy.trace(algopy.dot(J.T, J)) 62 | print('Phi=',Phi) 63 | print('gradient of Phi =', Phi.data[1,:]) 64 | 65 | 66 | -------------------------------------------------------------------------------- /experimental/performance_tests/adolc_implementation_for_speed_comparison/SConstruct: -------------------------------------------------------------------------------- 1 | import distutils.sysconfig 2 | import numpy 3 | 4 | LIBS = ['adolc', 5 | #'boost_python-gcc42-mt-1_34_1' 6 | #'boost_python-gcc42-1_34_1' 7 | ] 8 | LIBPATH = [ 9 | #'/u/walter/workspace/python_extension_for_adolc/adolc-1.11.0-trunk/lib', 10 | '/data/walter/opt_software/adolc-1.10.2/lib', 11 | #'/u/walter/workspace/python_extension_for_adolc/adolc-1.10.2/lib', 12 | #'/data/walter/opt_software/boost_1_34_1/bin.v2/libs/python/build/gcc-4.2.1/release/threading-multi' 13 | #'/data/walter/opt_software/boost_1_34_1/bin.v2/libs/python/build/gcc-4.2.1/release' 14 | ] 15 | INCLUDEPATH = [ 16 | '/data/walter/opt_software/adolc-1.10.2/include', 17 | #'/u/walter/workspace/python_extension_for_adolc/adolc-1.11.0-trunk/include', 18 | #'/u/walter/opt_software/my_global_cpp_libaries', 19 | #'/u/walter/workspace/python_extension_for_adolc/adolc-1.10.2/include/adolc', 20 | #'/data/walter/opt_software/boost_1_34_1', 21 | #'/usr/include/python2.5' 22 | ] 23 | 24 | env = Environment( 25 | CPPPATH=[distutils.sysconfig.get_python_inc(),numpy.get_include()] + INCLUDEPATH, 26 | CXXFLAGS="-ftemplate-depth-100 -DBOOST_PYTHON_DYNAMIC_LIB -O2", 27 | LIBPATH=["/usr/lib/python2.5/config"] + LIBPATH, 28 | LIBS= LIBS, 29 | RPATH = LIBPATH, #include information where shared libraries can be found to avoid errors like: "ImportError: libboost_python-gcc42-mt-1_34_1.so.1.34.1: cannot open shared object file: No such file or directory" 30 | SHLIBPREFIX="", #gets rid of lib prefix 31 | ) 32 | Default('.') 33 | adolc = env.Program(target='adolc', source=['main.cpp']) 34 | #env.Install("./release/adolc", adolc) 35 | 36 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/taylor_series_of_jacobian.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import algopy 3 | from algopy import CGraph, UTPM, Function 4 | 5 | def eval_g(x, y): 6 | """ some vector-valued function """ 7 | retval = algopy.zeros(3, dtype=x) 8 | retval[0] = algopy.sin(x**2 + y) 9 | retval[1] = algopy.cos(x+y) - x 10 | retval[2] = algopy.sin(x)**2 + algopy.cos(x)**2 11 | return retval 12 | 13 | # trace the function evaluation 14 | # and store the computational graph in cg 15 | cg = CGraph() 16 | ax = 3. 17 | ay = 5. 18 | fx = Function(ax) 19 | fy = Function(ay) 20 | fz = eval_g(fx, fy) 21 | cg.independentFunctionList = [fx, fy] 22 | cg.dependentFunctionList = [fz] 23 | 24 | # compute Taylor series 25 | # 26 | # Jx( 1. + 2.*t + 3.*t**2 + 4.*t**3 + 5.*t**5, 27 | # 6. + 7.*t + 8.*t**2 + 9.*t**3 + 10.*t**5 ) 28 | # Jy( 1. + 2.*t + 3.*t**2 + 4.*t**3 + 5.*t**5, 29 | # 6. + 7.*t + 8.*t**2 + 9.*t**3 + 10.*t**5 ) 30 | # 31 | # where 32 | # 33 | # Jx = dg/dx 34 | # Jy = dg/dy 35 | 36 | 37 | # setup input Taylor polynomials 38 | D,P = 5, 3 # order D=5, number of directions P 39 | ax = UTPM(numpy.zeros((D, P))) 40 | ay = UTPM(numpy.zeros((D, P))) 41 | ax.data[:, :] = numpy.array([1., 2. ,3., 4. ,5.]).reshape((5,1)) # input Taylor polynomial 42 | ay.data[:, :] = numpy.array([6., 7. ,8., 9. ,10.]).reshape((5,1)) # input Taylor polynomial 43 | 44 | # forward sweep 45 | cg.pushforward([ax, ay]) 46 | 47 | azbar = UTPM(numpy.zeros((D, P, 3))) 48 | azbar.data[0, ...] = numpy.eye(3) 49 | 50 | # reverse sweep 51 | cg.pullback([azbar]) 52 | 53 | # get results 54 | Jx = cg.independentFunctionList[0].xbar 55 | Jy = cg.independentFunctionList[1].xbar 56 | 57 | print('Taylor series of Jx =\n', Jx) 58 | print('Taylor series of Jy =\n', Jy) 59 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/gauss_newton.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | # from __future__ import division 3 | import numpy as np 4 | from numpy.linalg import norm 5 | import algopy 6 | 7 | 8 | def gauss_algopy(eval_f, x0, tol=10e-5): 9 | 10 | x0 = np.array(x0, dtype=float) 11 | cg = algopy.CGraph() 12 | x = algopy.Function(x0) 13 | y = eval_f(x) 14 | cg.trace_off() 15 | cg.independentFunctionList = [x] 16 | cg.dependentFunctionList = [y] 17 | sol = gauss_solver(eval_f, cg.jacobian, x0, tol) 18 | return sol 19 | 20 | def gauss_solver(f, jac, x0, tol): 21 | 22 | i=0 23 | x = x0 24 | res=1 25 | res_alt=10e10 26 | while((res>tol) and (50>i) and (abs(res-res_alt)>tol**2)): 27 | 28 | i+=1 29 | r=np.matrix(f(x)) 30 | 31 | D = np.matrix(jac(x)) 32 | DD=np.linalg.solve(D.T*D,D.T*r.T) 33 | 34 | x = np.matrix(x).T - DD 35 | x= np.array(x).flatten().tolist() 36 | res_alt=res 37 | res = norm(r) 38 | print(i,': ',res) 39 | 40 | return x 41 | 42 | 43 | if __name__ == '__main__': 44 | 45 | def eval_f1(x): 46 | y = algopy.zeros(3, dtype=x) 47 | y[0] = x[1] + x[2] - 5. 48 | y[1] = -x[0] + x[2]**3 + 1. 49 | y[2] = -x[1] + algopy.tan(x[0]) -x[2] + 7. 50 | return y 51 | 52 | def eval_f2(x): 53 | y = algopy.zeros(3, dtype=x) 54 | y[0] = x[1] + x[2] - 5. 55 | y[1] = -1./x[0] + x[2]**3 + 1. 56 | y[2] = -x[1] + algopy.tan(x[0]) -x[2] + 7. 57 | return y 58 | 59 | x0 = np.array([100.0, 1.0, 1.0]) 60 | sol = gauss_algopy(eval_f1, x0) 61 | print(sol) 62 | 63 | x0 = np.array([100.0, 1.0, 1.0]) 64 | sol = gauss_algopy(eval_f2, x0) 65 | print(sol) 66 | -------------------------------------------------------------------------------- /documentation/ICCS2010/degenerated_eigenvalues.py: -------------------------------------------------------------------------------- 1 | """ 2 | This example shows that the push forward of the eigenvalue decomposition indeed 3 | fails for symmetric matrices with degenerated eigenvalues. 4 | """ 5 | 6 | import numpy 7 | from algopy.utpm import * 8 | 9 | 10 | # Build Symmetric Matrix with degenerate eigenvalues 11 | D,P,N = 3,1,4 12 | A = UTPM(numpy.zeros((D,P,N,N))) 13 | V = UTPM(numpy.random.rand(D,P,N,N)) 14 | 15 | # A.data[0,0] = numpy.diag([2,2,3,3,2,5.]) 16 | # A.data[1,0] = numpy.diag([5,5,3,1,1,3.]) 17 | # A.data[2,0] = numpy.diag([3,1,3,1,1,3.]) 18 | 19 | A.data[0,0] = numpy.diag([2,2,2,5.]) 20 | A.data[1,0] = numpy.diag([5,5,6,6.]) 21 | A.data[2,0] = numpy.diag([1,1,1,1.]) 22 | 23 | 24 | V,Rtilde = UTPM.qr(V) 25 | A = UTPM.dot(UTPM.dot(V.T, A), V) 26 | 27 | # sanity check: solution of the zero'th coefficient using numpy 28 | A0 = A.data[0,0] 29 | l0,Q0 = numpy.linalg.eigh(A0) 30 | L0 = numpy.diag(l0) 31 | B0 = numpy.dot(numpy.dot(Q0,L0), Q0.T) 32 | 33 | # pushforward: general UTPM solution 34 | l,Q = UTPM.eigh(A) 35 | L = UTPM.diag(l) 36 | 37 | # pullback 38 | lbar = UTPM(numpy.random.rand(*(D,P,N))) 39 | Qbar = UTPM(numpy.random.rand(*(D,P,N,N))) 40 | Abar = UTPM.pb_eigh( lbar, Qbar, A, l, Q) 41 | 42 | Abar = Abar.data[0,0] 43 | Adot = A.data[1,0] 44 | 45 | Lbar = UTPM._diag(lbar.data)[0,0] 46 | Ldot = UTPM._diag(l.data)[1,0] 47 | 48 | Qbar = Qbar.data[0,0] 49 | Qdot = Q.data[1,0] 50 | 51 | # print l 52 | 53 | # print 'check pushforward:' 54 | print('Q.T A Q - L =\n', UTPM.dot(Q.T, UTPM.dot(A,Q)) - L) 55 | # print 'Q.T Q - I =\n', UTPM.dot(Q.T, Q) - numpy.eye(N) 56 | # print 'check pullback:' 57 | # print 'error measure of the pullback = ', numpy.trace(numpy.dot(Abar.T, Adot)) - numpy.trace( numpy.dot(Lbar.T, Ldot) + numpy.dot(Qbar.T, Qdot)) 58 | 59 | -------------------------------------------------------------------------------- /documentation/ICCS2010/stability_of_qr_decomposition_for_low_rank_matrices.py: -------------------------------------------------------------------------------- 1 | """ 2 | Experimentally test how stable the QR decomposition is. 3 | 4 | Low rank matrices A are built by low rank updates, then it is tested how big the 5 | residuals res_1 = QR - A and res_2 = Q.T Q - I and res_3 = P_L ( R) 6 | 7 | """ 8 | 9 | from numpy.testing import * 10 | import numpy 11 | import matplotlib.pyplot as pyplot 12 | import prettyplotting 13 | 14 | from algopy import UTPM, qr, dot, triu 15 | 16 | D,P,N = 3,1,3 17 | 18 | x1 = UTPM(numpy.random.rand(D,P,N,1)) 19 | x2 = UTPM(numpy.random.rand(D,P,N,1)) 20 | x3 = UTPM(numpy.random.rand(D,P,N,1)) 21 | 22 | def alpha(beta): 23 | return numpy.array([1., 2.**(-beta/2.), 2.**(-beta)]) 24 | 25 | # create matrix A by lowrank updates, alpha triggeres what the rank is 26 | 27 | 28 | res_1_list = [] 29 | res_2_list = [] 30 | res_3_list = [] 31 | 32 | betas = list(range(0,200,10)) 33 | for beta in betas: 34 | a = alpha(beta) 35 | A = a[0]*dot(x1,x1.T) + a[1]*dot(x2,x2.T) + a[2]*dot(x3,x3.T) 36 | 37 | Q,R = qr(A) 38 | res_1 = numpy.abs((dot(Q,R) - A).data).max() 39 | res_2 = numpy.abs((dot(Q.T,Q) - numpy.eye(N)).data).max() 40 | res_3 = numpy.abs((triu(R) - R).data).max() 41 | 42 | res_1_list.append(res_1) 43 | res_2_list.append(res_2) 44 | res_3_list.append(res_3) 45 | 46 | 47 | pyplot.figure() 48 | pyplot.semilogy(betas, res_1_list, 'kd', label=r'max $ (| QR - A |)$') 49 | pyplot.semilogy(betas, res_2_list, 'ko', label=r'max $ (| Q^T Q - I |)$') 50 | pyplot.semilogy(betas, res_3_list, 'k.', label=r'max $ (| P_R \circ R - R |)$') 51 | pyplot.xlabel(r'$\beta$') 52 | 53 | 54 | pyplot.legend( loc='best') 55 | # # pyplot.savefig('/tmp/qr_residuals_rank%d.eps'%rank) 56 | 57 | pyplot.show() 58 | -------------------------------------------------------------------------------- /documentation/AD_tutorial_TU_Berlin/example2_qr_numerical_stability.py: -------------------------------------------------------------------------------- 1 | """ 2 | Experimentally test how stable the QR decomposition is. 3 | 4 | Low rank matrices A are built by low rank updates, then it is tested how big the 5 | residuals res_1 = QR - A and res_2 = Q.T Q - I and res_3 = P_L ( R) 6 | 7 | """ 8 | 9 | from numpy.testing import * 10 | import numpy 11 | import matplotlib.pyplot as pyplot 12 | import prettyplotting 13 | 14 | from algopy import UTPM, qr, dot, triu 15 | 16 | D,P,N = 4,1,3 17 | 18 | x1 = UTPM(numpy.random.rand(D,P,N,1)) 19 | x2 = UTPM(numpy.random.rand(D,P,N,1)) 20 | x3 = UTPM(numpy.random.rand(D,P,N,1)) 21 | 22 | def alpha(beta): 23 | return numpy.array([1., 2.**(-beta/2.), 2.**(-beta)]) 24 | 25 | # create matrix A by lowrank updates, alpha triggeres what the rank is 26 | 27 | 28 | res_1_list = [] 29 | res_2_list = [] 30 | res_3_list = [] 31 | 32 | betas = list(range(0,200,10)) 33 | for beta in betas: 34 | print('perform QR decomposition') 35 | a = alpha(beta) 36 | A = a[0]*dot(x1,x1.T) + a[1]*dot(x2,x2.T) + a[2]*dot(x3,x3.T) 37 | 38 | Q,R = qr(A) 39 | res_1 = numpy.abs((dot(Q,R) - A).data).max() 40 | res_2 = numpy.abs((dot(Q.T,Q) - numpy.eye(N)).data).max() 41 | res_3 = numpy.abs((triu(R) - R).data).max() 42 | 43 | res_1_list.append(res_1) 44 | res_2_list.append(res_2) 45 | res_3_list.append(res_3) 46 | 47 | 48 | pyplot.figure() 49 | pyplot.semilogy(betas, res_1_list, 'kd', label=r'max $ (| QR - A |)$') 50 | pyplot.semilogy(betas, res_2_list, 'ko', label=r'max $ (| Q^T Q - I |)$') 51 | pyplot.semilogy(betas, res_3_list, 'k.', label=r'max $ (| P_R \circ R - R |)$') 52 | pyplot.xlabel(r'$\beta$') 53 | 54 | 55 | pyplot.legend( loc='best') 56 | pyplot.savefig('qr_stability.eps') 57 | 58 | pyplot.show() 59 | -------------------------------------------------------------------------------- /documentation/examples/hessian_computation_in_combined_forward_reverse_mode.py: -------------------------------------------------------------------------------- 1 | """ 2 | This example explains how the Hessian of a function 3 | f: R^N -->R 4 | 5 | can be combined in the combinded forward/reverse mode of AD 6 | """ 7 | 8 | 9 | 10 | import numpy 11 | from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv, solve, trace 12 | 13 | # first order derivatives, one directional derivative 14 | # D - 1 is the degree of the Taylor polynomial 15 | # P directional derivatives at once 16 | # M number of rows of A 17 | # N number of cols of A 18 | D,M,N = 2,3,3 19 | P = M*N 20 | 21 | # generate badly conditioned matrix A 22 | 23 | A = UTPM(numpy.zeros((D,P,M,N))) 24 | A.data[0,:] = numpy.random.rand(*(M,N)) 25 | 26 | for m in range(M): 27 | for n in range(N): 28 | p = m*N + n 29 | A.data[1,p,m,n] = 1. 30 | 31 | 32 | cg = CGraph() 33 | A = Function(A) 34 | y = trace(inv(A)) 35 | cg.trace_off() 36 | 37 | cg.independentFunctionList = [A] 38 | cg.dependentFunctionList = [y] 39 | 40 | ybar = y.x.zeros_like() 41 | ybar.data[0,:] = 1. 42 | cg.pullback([ybar]) 43 | 44 | # check gradient 45 | g_forward = numpy.zeros(N*N) 46 | g_reverse = numpy.zeros(N*N) 47 | 48 | for m in range(M): 49 | for n in range(N): 50 | p = m*N + n 51 | g_forward[p] = y.x.data[1,p] 52 | g_reverse[p] = A.xbar.data[0,0,m,n] 53 | 54 | numpy.testing.assert_array_almost_equal(g_forward, g_reverse) 55 | 56 | H = numpy.zeros((M,N,M,N)) 57 | for m in range(M): 58 | for n in range(N): 59 | for m2 in range(M): 60 | for n2 in range(N): 61 | p = m2*N + n2 62 | H[m,n, m2,n2] = A.xbar.data[1,p,m,n] 63 | 64 | 65 | H_reshaped = H.reshape((M*N, M*N)) 66 | 67 | print(H_reshaped - H_reshaped.T) 68 | 69 | # print y.x.data[1] 70 | -------------------------------------------------------------------------------- /documentation/sphinx/speed_comparison/use_theano.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import numpy 3 | import time 4 | 5 | 6 | import numpy 7 | import adolc 8 | 9 | class EVAL: 10 | def __init__(self, f, x, test = 'f'): 11 | self.f = f 12 | self.x = x 13 | self.A = f.A 14 | 15 | x = theano.tensor.dvector('x') 16 | A = theano.tensor.dmatrix('A') 17 | y = 0.5*theano.dot(x*x,theano.dot(A,x)) 18 | 19 | if test == 'f': 20 | self.eval_f = theano.function([x,A], y) 21 | 22 | elif test == 'g': 23 | gy = theano.tensor.grad(y,x) 24 | self.eval_grad = theano.function([x,A], gy) 25 | elif test == 'h': 26 | gy = theano.tensor.grad(y,x) 27 | hy, updates = theano.scan( lambda i, gy, x,A: theano.tensor.grad(gy[i], x), sequences = theano.tensor.arange(gy.shape[0]), non_sequences = [gy,x,A]) 28 | self.eval_hess = theano.function([x,A], hy) 29 | 30 | def function(self, x): 31 | return float(self.eval_f(x,self.A)) 32 | 33 | def gradient(self, x): 34 | return self.eval_grad(x,self.A) 35 | 36 | def hessian(self, x): 37 | return self.eval_hess(x,self.A) 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | # #A = numpy.random.rand(2,2) 47 | # N = 100 48 | 49 | # x = theano.tensor.dvector('x') 50 | # A = theano.tensor.dmatrix('A') 51 | 52 | # y = 0.5 * theano.dot(x, theano.dot(A,x)) 53 | 54 | # gy = theano.tensor.grad(y,x) 55 | # dlogistic = theano.function([x,A], gy) 56 | 57 | # A = numpy.random.rand(N,N) 58 | # A = numpy.dot(A.T,A) 59 | # x = numpy.ones(N) 60 | # start_time = time.time() 61 | # g = dlogistic(x,A) 62 | # end_time = time.time() 63 | 64 | 65 | # print end_time - start_time 66 | 67 | # print g - numpy.dot(A,x) 68 | -------------------------------------------------------------------------------- /documentation/sphinx/speed_comparison/use_algopy.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import algopy 3 | 4 | class EVAL: 5 | def __init__(self, f, x, test = 'f'): 6 | self.f = f 7 | self.x = x.copy() 8 | 9 | cg = algopy.CGraph() 10 | x = np.array([algopy.Function(x[i]) for i in range(len(x))]) 11 | y = f(x) 12 | # print 'y=',y 13 | cg.trace_off() 14 | cg.independentFunctionList = x 15 | cg.dependentFunctionList = [y] 16 | self.cg = cg 17 | 18 | def function(self, x): 19 | return self.cg.function(x) 20 | 21 | def gradient(self, x): 22 | return self.cg.gradient(x) 23 | 24 | # def hessian(self, x): 25 | # return adolc.hessian(0,x) 26 | 27 | 28 | 29 | class EVAL2: 30 | def __init__(self, f, x, test = 'f'): 31 | self.f = f 32 | self.x = x.copy() 33 | 34 | if test != 'fg' and test != 'fh': 35 | cg = algopy.CGraph() 36 | x = algopy.Function(x) 37 | y = f(x) 38 | cg.trace_off() 39 | cg.independentFunctionList = [x] 40 | cg.dependentFunctionList = [y] 41 | self.cg = cg 42 | 43 | 44 | def function(self, x): 45 | return self.cg.function([x]) 46 | 47 | def gradient(self, x): 48 | return self.cg.gradient([x]) 49 | 50 | def forwardgradient(self, x): 51 | tmp = algopy.UTPM.init_jacobian(x) 52 | return algopy.UTPM.extract_jacobian(self.f(tmp)) 53 | 54 | def hessian(self, x): 55 | return self.cg.hessian([x]) 56 | 57 | def forwardhessian(self, x): 58 | tmp = algopy.UTPM.init_hessian(x) 59 | return algopy.UTPM.extract_hessian(len(x), self.f(tmp)) 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /documentation/AD_tutorial_TU_Berlin/example3_qr_for_moore_penrose_pseudo_inverse.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a simple test that differentiates the Moore-Penrose pseudo-inverse 3 | computation. Explicitly: 4 | 5 | ..math:: 6 | A^\dagger = (A^T A)^{-1} A^T 7 | 8 | Two methods are compared. First the naive approach by first computing A^T A, 9 | then invert it and then multiplication with A^T. 10 | Then the QR approach. 11 | 12 | """ 13 | 14 | 15 | import numpy 16 | from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv, solve 17 | 18 | D,P,M,N = 2,1,5,2 19 | 20 | # generate badly conditioned matrix A 21 | A = UTPM(numpy.zeros((D,P,M,N))) 22 | x = UTPM(numpy.zeros((D,P,M,1))) 23 | y = UTPM(numpy.zeros((D,P,M,1))) 24 | 25 | x.data[0,0,:,0] = [1,1,1,1,1] 26 | x.data[1,0,:,0] = [1,1,1,1,1] 27 | 28 | y.data[0,0,:,0] = [1,2,1,2,1] 29 | y.data[1,0,:,0] = [1,2,1,2,1] 30 | 31 | alpha = 10**-5 32 | A = dot(x,x.T) + alpha*dot(y,y.T) 33 | A = A[:,:2] 34 | 35 | # Method 1: Naive approach 36 | Apinv = dot(inv(dot(A.T,A)),A.T) 37 | 38 | print('naive approach: A Apinv A - A = 0 \n', dot(dot(A, Apinv),A) - A) 39 | print('naive approach: Apinv A Apinv - Apinv = 0 \n', dot(dot(Apinv, A),Apinv) - Apinv) 40 | print('naive approach: (Apinv A)^T - Apinv A = 0 \n', dot(Apinv, A).T - dot(Apinv, A)) 41 | print('naive approach: (A Apinv)^T - A Apinv = 0 \n', dot(A, Apinv).T - dot(A, Apinv)) 42 | 43 | 44 | # Method 2: Using the differentiated QR decomposition 45 | Q,R = qr(A) 46 | tmp1 = solve(R.T, A.T) 47 | tmp2 = solve(R, tmp1) 48 | Apinv = tmp2 49 | 50 | print('QR approach: A Apinv A - A = 0 \n', dot(dot(A, Apinv),A) - A) 51 | print('QR approach: Apinv A Apinv - Apinv = 0 \n', dot(dot(Apinv, A),Apinv) - Apinv) 52 | print('QR approach: (Apinv A)^T - Apinv A = 0 \n', dot(Apinv, A).T - dot(Apinv, A)) 53 | print('QR approach: (A Apinv)^T - A Apinv = 0 \n', dot(A, Apinv).T - dot(A, Apinv)) 54 | -------------------------------------------------------------------------------- /experimental/numpy_array_dtype_interface.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | class oofun: 4 | is_dtype = True 5 | 6 | def __init__(self, data): 7 | self.data = numpy.asarray(data,dtype=float) 8 | 9 | def __mul__(self, other): 10 | return oofun(oofun.multiply(self.data, other.data)) 11 | 12 | 13 | @classmethod 14 | def spsd_multiply(cls, x_data, y_data, z_data = None): 15 | """ 16 | Single Program Single Data implementation can only do one data type multiplication at a time 17 | 18 | raw algorithm that operates on possibly nested containers (array,list,tuple,dict) of elementary Python data types: float, int, ... 19 | 20 | as an example, we use here numy.array of shape = (3,) with dtype float for x_data, y_data, z_data 21 | 22 | if wanted, memory allocation of z_data can be done before calling this function 23 | (would not be of use here, since there is no convolve function where the output array can be preallocated) 24 | 25 | """ 26 | 27 | z_data = x_data * y_data 28 | return z_data 29 | 30 | @classmethod 31 | def spmd_multiply(cls, array_of_x_data, array_of_y_data, array_of_z_data = None): 32 | """ 33 | Single Program Multiple Data implementation is responsible for efficient vectorized multiplication 34 | 35 | array_of_x_data is an array of shape = (N1,N2,...) + (3,) 36 | 37 | """ 38 | 39 | array_of_z_data = array_of_x_data * array_of_y_data 40 | return array_of_z_data 41 | 42 | 43 | def __str__(self): 44 | return str(self.data) + 'a' 45 | 46 | 47 | class array: 48 | def __init__(self, input_list, dtype=float): 49 | if dtype != float: 50 | 51 | 52 | 53 | 54 | x = oofun([1,2,3]) 55 | y = oofun([4,5,6]) 56 | 57 | z = x * y 58 | 59 | print x 60 | print y 61 | print z 62 | 63 | x_array = array([oofun([1,2,3]),oofun([4,5,6]])],dtype=oofun) 64 | y_array = array([oofun([7,8,9]),oofun([3,5,1]])],dtype=oofun) 65 | 66 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/covariance_matrix_computation.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from algopy import CGraph, Function, UTPM, dot, qr, qr_full, eigh, inv, solve, zeros 3 | 4 | def eval_covariance_matrix_naive(J1, J2): 5 | M,N = J1.shape 6 | K,N = J2.shape 7 | tmp = zeros((N+K, N+K), dtype=J1) 8 | tmp[:N,:N] = dot(J1.T,J1) 9 | tmp[:N,N:] = J2.T 10 | tmp[N:,:N] = J2 11 | return inv(tmp)[:N,:N] 12 | 13 | def eval_covariance_matrix_qr(J1, J2): 14 | M,N = J1.shape 15 | K,N = J2.shape 16 | Q,R = qr_full(J2.T) 17 | Q2 = Q[:,K:].T 18 | J1_tilde = dot(J1,Q2.T) 19 | Q,R = qr(J1_tilde) 20 | V = solve(R.T, Q2) 21 | return dot(V.T,V) 22 | 23 | 24 | # dimensions of the involved matrices 25 | D,P,M,N,K,Nx = 2,1,5,3,1,1 26 | 27 | # trace the function evaluation of METHOD 1: nullspace method 28 | cg1 = CGraph() 29 | J1 = Function(UTPM(numpy.random.rand(*(D,P,M,N)))) 30 | J2 = Function(UTPM(numpy.random.rand(*(D,P,K,N)))) 31 | C = eval_covariance_matrix_qr(J1, J2) 32 | y = C[0,0] 33 | cg1.trace_off() 34 | cg1.independentFunctionList = [J1, J2] 35 | cg1.dependentFunctionList = [y] 36 | print('covariance matrix: C =\n',C) 37 | 38 | # trace the function evaluation of METHOD 2: naive method (potentially numerically unstable) 39 | cg2 = CGraph() 40 | J1 = Function(J1.x) 41 | J2 = Function(J2.x) 42 | C2 = eval_covariance_matrix_naive(J1, J2) 43 | y = C2[0,0] 44 | cg2.trace_off() 45 | cg2.independentFunctionList = [J1, J2] 46 | cg2.dependentFunctionList = [y] 47 | print('covariance matrix: C =\n',C2) 48 | 49 | # check that both algorithms returns the same result 50 | print('difference between naive and nullspace method:\n',C - C2) 51 | 52 | # compute the gradient for another value of J1 and J2 53 | J1 = numpy.random.rand(*(M,N)) 54 | J2 = numpy.random.rand(*(K,N)) 55 | 56 | g1 = cg1.gradient([J1,J2]) 57 | g2 = cg2.gradient([J1,J2]) 58 | 59 | print('naive approach: dy/dJ1 = ', g1[0]) 60 | print('naive approach: dy/dJ2 = ', g1[1]) 61 | 62 | print('nullspace approach: dy/dJ1 = ', g2[0]) 63 | print('nullspace approach: dy/dJ2 = ', g2[1]) 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/comparison_forward_reverse_mode.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv, zeros 3 | 4 | def f(x,y): 5 | return dot(x.T, y) + dot((x*y-x).T, (x-y)) 6 | 7 | 8 | # create an UTPM instance 9 | D,N,M = 2,3,2 10 | P = 2*N 11 | 12 | x = UTPM(numpy.zeros((D,P,2*N,1))) 13 | x.data[0,:] = numpy.random.rand(2*N,1) 14 | x.data[1,:,:,0] = numpy.eye(P) 15 | y = x[N:] 16 | x = x[:N] 17 | 18 | # wrap the UTPM instance in a Function instance to trace all operations 19 | # that have x as an argument 20 | # create a CGraph instance that to store the computational trace 21 | cg = CGraph().trace_on() 22 | x = Function(x) 23 | y = Function(y) 24 | z = f(x,y) 25 | cg.trace_off() 26 | 27 | # define dependent and independent variables in the computational procedure 28 | cg.independentFunctionList = [x,y] 29 | cg.dependentFunctionList = [z] 30 | 31 | # Since the UTPM instrance is wrapped in a Function instance we have to access it 32 | # by y.x. That means the Jacobian is 33 | grad1 = z.x.data[1,:,0] 34 | 35 | print('forward gradient g(x) = \n', grad1) 36 | 37 | # Now we want to compute the same Jacobian in the reverse mode of AD 38 | # before we do that we have a look what the computational graph looks like: 39 | # print 'Computational graph is', cg 40 | 41 | # the reverse mode is called by cg.pullback([ybar]) 42 | # it is a little hard to explain what's going on here. Suffice to say that we 43 | # now compute one row of the Jacobian instead of one column as in the forward mode 44 | 45 | zbar = z.x.zeros_like() 46 | 47 | # compute gradient in the reverse mode 48 | zbar.data[0,:,0,0] = 1 49 | cg.pullback([zbar]) 50 | grad2_x = x.xbar.data[0,0] 51 | grad2_y = y.xbar.data[0,0] 52 | grad2 = numpy.concatenate([grad2_x, grad2_y]) 53 | 54 | print('reverse gradient g(x) = \n', grad2) 55 | 56 | #check that the forward computed gradient equals the reverse gradient 57 | print('difference forward/reverse gradient=\n',grad1 - grad2) 58 | 59 | # one can also easiliy extract the Hessian 60 | H = numpy.zeros((2*N,2*N)) 61 | H[:,:N] = x.xbar.data[1,:,:,0] 62 | H[:,N:] = y.xbar.data[1,:,:,0] 63 | 64 | print('Hessian = \n', H) 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /documentation/sphinx/folder_structure.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | AlgoPy file and folder structure 3 | ================================= 4 | --------------------------------- 5 | an overview 6 | --------------------------------- 7 | 8 | 9 | The general idea 10 | ================ 11 | 12 | This document explains how AlgoPy is structured. 13 | 14 | * The submodule tracer contains two classes 15 | 16 | - CGraph 17 | - Function 18 | 19 | CGraph stands for computational graph and is a directed acyclic graph. 20 | A node in the CGraph is an instance of Function. 21 | 22 | It is called "tracer" since the sequence of operations to evaluate some 23 | function:: 24 | 25 | def eval_f(x): 26 | ... 27 | return y 28 | 29 | is recorded and stored in a CGraph instance. 30 | 31 | Using CGraph.gradient, CGraph.jacobian, etc. one can evaluate derivatives 32 | in the forward and reverse mode of AD. 33 | 34 | The function `CGraph.pullback` cycles in reverse direction through the 35 | sequence of operations and calls at each Node the method 36 | `Function.pullback`, which in turn calls one of the classmethods 37 | `UTPM.pb_*`. 38 | 39 | 40 | * The submodule UTPM contains an implementation of the algebraic class 41 | of matrix Taylor polynomials. 42 | It is an "extension" of numpy.ndarray. 43 | 44 | 45 | algopy/tests/test_*.py 46 | ====================== 47 | 48 | Normally, for each file `file.py` there exists a 49 | `tests/test_file.py`. 50 | 51 | In this folder there are tests that check the functionality 52 | as seen by a user. 53 | 54 | * test_globalfuncs.py checks that algopy.dot, algopy.sin etc. 55 | correctly call UTPM.sin, Function.sin etc. 56 | 57 | * test_linalg.py checks that forward and reverse mode yields 58 | the same results for linear algebra functions 59 | 60 | * test_special.py checks that forward and reverse mode yields 61 | the same results for linear algebra functions 62 | 63 | * test_operators.py checks that forward and reverse mode yields 64 | the same results for operators __add__, __mul__, __iadd__, etc. 65 | Also check whether broadcasting works correctly. 66 | 67 | * test_examples.py contains more complex examples 68 | 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /experimental/tests/trash/unit_test_with_sympy_x_as_vector.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ####################################################### 3 | # This is a Unit Test t that makes use of the Python # 4 | # Module Sympy. # 5 | ####################################################### 6 | 7 | import sympy as sym 8 | from numpy import array, zeros, ones, shape 9 | from numpy.random import random 10 | from numpy.linalg import norm 11 | from forward_mode import * 12 | 13 | N = 4 14 | 15 | xs = array([sym.Symbol('x%d'%n) for n in range(N)]) 16 | # computing the function f: R^(NxD) -> R symbolically 17 | fs = 0 18 | for n in range(1,N): 19 | for m in range(n): 20 | fs += 1/(xs[n] - xs[m]) 21 | 22 | # computing the gradient symbolically 23 | dfs = array([sym.diff(fs, xs[n]) for n in range(N)]) 24 | 25 | # computing the Hessian symbolically 26 | ddfs = array([[ sym.diff(dfs[m], xs[n]) for n in range(N)] for m in range(N)]) 27 | 28 | 29 | def sym_f(x): 30 | symdict = dict() 31 | for n in range(N): 32 | symdict[xs[n]] = x[n] 33 | return fs.subs_dict(symdict).evalf() 34 | 35 | 36 | def sym_df(x): 37 | symdict = dict() 38 | for n in range(N): 39 | symdict[xs[n]] = x[n] 40 | return array([dfs[n].subs_dict(symdict).evalf() for n in range(N)]) 41 | 42 | def sym_ddf(x): 43 | symdict = dict() 44 | for n in range(N): 45 | symdict[xs[n]] = x[n] 46 | return array([[ ddfs[m,n].subs_dict(symdict).evalf() for n in range(N)] for m in range(N)],dtype=float) 47 | 48 | 49 | def f(x): 50 | retval = 0. 51 | for n in range(1,N): 52 | for m in range(n): 53 | retval += 1./(x[n] - x[m]) 54 | return retval 55 | 56 | def ad_df(x): 57 | return gradient(f,x) 58 | 59 | def ad_ddf(x): 60 | return hessian(f,x) 61 | 62 | 63 | # point at which the derivatives should be evaluated 64 | x = random(N) 65 | 66 | print('\n\n') 67 | print('Sympy function = Ad function check (should be almost zero)') 68 | print(f(x) - sym_f(x)) 69 | 70 | print('\n\n') 71 | print('Sympy vs Ad Derived Gradient check (should be almost zero)') 72 | print(ad_df(x) - sym_df(x)) 73 | 74 | print('\n\n') 75 | print('Sympy vs Ad Derived Hessian check (should be almost zero)') 76 | print(sym_ddf(x) - ad_ddf(x)) 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/error_propagation.rst: -------------------------------------------------------------------------------- 1 | Linear Error Propagation 2 | ========================= 3 | 4 | 5 | This example shows how ALGOPY can be used for linear error propagation. 6 | 7 | Consider the error model 8 | 9 | .. math:: 10 | y = x + \epsilon 11 | 12 | where :math:`x` a vector and :math:`\epsilon` a random vector that is normally distributed with zero mean and 13 | covariance matrix :math:`\Sigma^2`. The :math:`y` is the observed quantity and :math:`x` is a real vector 14 | representing the "true" value. 15 | 16 | One defines some estimator :math:`\hat x` for :math:`x`, e.g. the arithmetic 17 | mean :math:`\hat x = \sum_{i=1}^{N_m} y_i`. 18 | We assume that confidence region of the estimate :math:`\hat x` is known and has an 19 | associated confidence region described by its covariance matrix 20 | 21 | .. math:: 22 | \Sigma^2 = \mathbb E[(\hat x - E[\hat x])(\hat x - E[\hat x])^T] 23 | 24 | The question is: What can we say about the confidence region of the function 25 | :math:`f(y)` when the confidence region of :math:`y` is described by the 26 | covariance matrix :math:`\Sigma^2`? 27 | 28 | .. math:: 29 | f: \mathbb R^N \rightarrow \mathbb R^M \\ 30 | \hat x \mapsto \hat x = f(\hat x) 31 | 32 | 33 | For affine (linear) functions 34 | 35 | .. math:: 36 | z = f(y) = Ay + b 37 | 38 | the approach is described in the wikipedia article http://en.wikipedia.org/wiki/Propagation_of_uncertainty . 39 | Nonlinear functions are simply linearized about the estimate :math:`\hat y` of :math:`\mathbb E[y]`. 40 | In the vicinity of :math:`\hat y`, the linear model approximates the nonlinear function often quite well. 41 | To linearize the function, the Jacobian :math:`J(\hat y)` of the function :math:`f(\hat y)` has to be computed, i.e.: 42 | 43 | .. math:: 44 | z \approx f(y) = f(\hat y) + J(\hat y) (y - \hat y) 45 | 46 | The covariance matrix of :math:`z` is defined as 47 | 48 | .. math:: 49 | C = \mathbb E[z z^T] = \mathbb E[ J y y^T J^T] = J \Sigma^2 J^T \; . 50 | 51 | That means if we know :math:`J(y)`, we can approximately compute the confidence region if 52 | :math:`f(\hat y)` is sufficiently linear. 53 | 54 | To compute the Jacobian one can use the forward or the reverse mode of AD. 55 | 56 | 57 | .. literalinclude:: error_propagation.py 58 | 59 | -------------------------------------------------------------------------------- /experimental/ctps/ctps.py: -------------------------------------------------------------------------------- 1 | import os 2 | import ctypes 3 | import numpy 4 | 5 | from algopy.base_type import Ring 6 | 7 | _ctps = numpy.ctypeslib.load_library('libctps', os.path.dirname(__file__)) 8 | 9 | double_ptr = ctypes.POINTER(ctypes.c_double) 10 | argtypes1 = [ctypes.c_int, double_ptr, double_ptr, double_ptr] 11 | 12 | _ctps.ctps_add.argtypes = argtypes1 13 | _ctps.ctps_sub.argtypes = argtypes1 14 | _ctps.ctps_mul.argtypes = argtypes1 15 | _ctps.ctps_div.argtypes = argtypes1 16 | 17 | class CTPS(Ring): 18 | def __init__(self, data): 19 | """ 20 | CTPS = Cross Derivative Taylor Polynomial 21 | Implements the factor ring R[t1,...,tK]/ 22 | 23 | Calls C functions internally. I.e. functionality *should* be the same as for the class CTPS. 24 | """ 25 | self.data = numpy.array(data) 26 | 27 | @classmethod 28 | def __scalar_to_data__(cls, xdata, x): 29 | xdata[0] = x 30 | 31 | @classmethod 32 | def __zeros_like__(cls, data): 33 | return numpy.zeros_like(data) 34 | 35 | @classmethod 36 | def add(cls, retval_data, lhs_data, rhs_data): 37 | K = retval_data.size 38 | _ctps.ctps_add(K, 39 | lhs_data.ctypes.data_as(double_ptr), 40 | rhs_data.ctypes.data_as(double_ptr), 41 | retval_data.ctypes.data_as(double_ptr)) 42 | 43 | @classmethod 44 | def sub(cls, retval_data, lhs_data, rhs_data): 45 | K = retval_data.size 46 | _ctps.ctps_sub(K, 47 | lhs_data.ctypes.data_as(double_ptr), 48 | rhs_data.ctypes.data_as(double_ptr), 49 | retval_data.ctypes.data_as(double_ptr)) 50 | 51 | @classmethod 52 | def mul(cls, retval_data, lhs_data, rhs_data): 53 | K = retval_data.size 54 | _ctps.ctps_mul(K, 55 | lhs_data.ctypes.data_as(double_ptr), 56 | rhs_data.ctypes.data_as(double_ptr), 57 | retval_data.ctypes.data_as(double_ptr)) 58 | 59 | @classmethod 60 | def div(cls, retval_data, lhs_data, rhs_data): 61 | K = retval_data.size 62 | _ctps.ctps_div(K, 63 | lhs_data.ctypes.data_as(double_ptr), 64 | rhs_data.ctypes.data_as(double_ptr), 65 | retval_data.ctypes.data_as(double_ptr)) 66 | 67 | def __repr__(self): 68 | return self.__str__() 69 | 70 | def __str__(self): 71 | return str(self.data) 72 | -------------------------------------------------------------------------------- /algopy/utpm/tests/test_utpm_convenience.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test UTPM convenience functions. 3 | 4 | Test extraction of the Jacobian, the Hessian, and the Hessian-vector product. 5 | These tests use the exact solutions implemented in scipy.optimize. 6 | 7 | """ 8 | 9 | import numpy as np 10 | import pytest 11 | from numpy.testing import assert_allclose 12 | 13 | import algopy 14 | 15 | try: 16 | import scipy.optimize 17 | has_scipy = True 18 | except ImportError: 19 | has_scipy = False 20 | 21 | 22 | def rosen(x): 23 | """ 24 | Arbitrary-dimensional Rosenbrock function for testing. 25 | """ 26 | return algopy.sum(100.0 * (x[1:] - x[:-1]**2.0)**2.0 + (1 - x[:-1])**2.0) 27 | 28 | 29 | @pytest.mark.skipif(not has_scipy, reason='scipy is required for this test') 30 | def test_rosen_jac(): 31 | x_raw = np.array((0.1, 0.3, 0.4)) 32 | expected_jac = scipy.optimize.rosen_der(x_raw) 33 | x = algopy.UTPM.init_jacobian(x_raw) 34 | y = rosen(x) 35 | observed_jac = algopy.UTPM.extract_jacobian(y) 36 | assert_allclose(observed_jac, expected_jac) 37 | 38 | 39 | @pytest.mark.skipif(not has_scipy, reason='scipy is required for this test') 40 | def test_rosen_hess(): 41 | x_raw = np.array((0.1, 0.3, 0.4)) 42 | N = np.size(x_raw) 43 | expected_hess = scipy.optimize.rosen_hess(x_raw) 44 | x = algopy.UTPM.init_hessian(x_raw) 45 | y = rosen(x) 46 | observed_hess = algopy.UTPM.extract_hessian(N, y) 47 | assert_allclose(observed_hess, expected_hess) 48 | 49 | 50 | @pytest.mark.skipif(not has_scipy, reason='scipy is required for this test') 51 | def test_scipy_rosen_hess_prod(): 52 | x_raw = np.array((0.1, 0.3, 0.4, 12)) 53 | v = np.array((-1, 0, 0.123, 2)) 54 | expected_hess = scipy.optimize.rosen_hess(x_raw) 55 | expected_hess_prod = scipy.optimize.rosen_hess_prod(x_raw, v) 56 | assert_allclose(np.dot(expected_hess, v), expected_hess_prod) 57 | 58 | 59 | @pytest.mark.skipif(not has_scipy, reason='scipy is required for this test') 60 | def test_rosen_hess_vec(): 61 | x_raw = np.array((0.1, 0.3, 0.4, 12)) 62 | N = np.size(x_raw) 63 | v = np.array((-1, 0, 0.123, 2)) 64 | expected_hess_vec = scipy.optimize.rosen_hess_prod(x_raw, v) 65 | x = algopy.UTPM.init_hess_vec(x_raw, v) 66 | y = rosen(x) 67 | observed_hess_vec = algopy.UTPM.extract_hess_vec(N, y) 68 | assert_allclose(observed_hess_vec, expected_hess_vec) 69 | 70 | 71 | if __name__ == '__main__': 72 | run_module_suite() 73 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/neg_binom_regression.rst: -------------------------------------------------------------------------------- 1 | Negative Binomial Regression 2 | ---------------------------- 3 | 4 | In this example we want to use AlgoPy to help compute the 5 | maximum likelihood estimates and standard errors of parameters 6 | of a nonlinear model. 7 | This follows the 8 | `statsmodels `_ 9 | generic maximum likelihood 10 | `example 11 | `_ 13 | which uses the 14 | `medpar 15 | `_ 16 | dataset. 17 | 18 | .. math:: 19 | 20 | \mathcal{L}(\beta_j; y, \alpha) = \sum_{i=1}^n y_i \log & 21 | \left( \frac{\alpha \exp(X'_i \beta)}{1 + \alpha \exp(X'_i \beta)} \right) 22 | - \frac{1}{\alpha} \log \left( 1 + \alpha \exp(X'_i \beta) \right) \\ 23 | & + \log \Gamma(y_i + 1/\alpha) - 24 | \log \Gamma(y_i + 1) - \log \Gamma(1/\alpha) 25 | 26 | Here is the python code: 27 | 28 | .. literalinclude:: neg_binom_regression.py 29 | 30 | 31 | Here is its output:: 32 | 33 | Optimization terminated successfully. 34 | Current function value: 4797.476603 35 | Iterations: 10 36 | Function evaluations: 11 37 | Gradient evaluations: 10 38 | Hessian evaluations: 10 39 | search results: 40 | [ 2.31027893 0.22124897 0.70615882 -0.06795522 -0.12906544 0.44575671] 41 | 42 | aic: 43 | 9606.95320507 44 | 45 | standard error using observed fisher information, 46 | with hessian computed using algopy: 47 | [ 0.06794736 0.05059255 0.07613111 0.05326133 0.06854179 0.01981577] 48 | 49 | standard error using observed fisher information, 50 | with hessian computed using numdifftools: 51 | [ 0.06794736 0.05059255 0.07613111 0.05326133 0.06854179 0.01981577] 52 | 53 | 54 | The agreement between this output and the statsmodels example results 55 | suggests that the statsmodels 56 | `caveat 57 | `_ 59 | about numerical precision may not be the result of 60 | numerical problems with the derivatives, 61 | but rather that the R MASS 62 | `implementation 63 | `_ 64 | may be giving less precise standard error estimates 65 | or may not be using the observed fisher information to get the 66 | standard error estimates in the most straightforward way. 67 | 68 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/moore_penrose_pseudoinverse.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv, solve 3 | 4 | # first order derivatives, one directional derivative 5 | # D - 1 is the degree of the Taylor polynomial 6 | # P directional derivatives at once 7 | # M number of rows of A 8 | # N number of cols of A 9 | D,P,M,N = 2,1,5,2 10 | 11 | # generate badly conditioned matrix A 12 | A = UTPM(numpy.zeros((D,P,M,N))) 13 | x = UTPM(numpy.zeros((D,P,M,1))) 14 | y = UTPM(numpy.zeros((D,P,M,1))) 15 | 16 | x.data[0,0,:,0] = [1,1,1,1,1] 17 | x.data[1,0,:,0] = [1,1,1,1,1] 18 | 19 | y.data[0,0,:,0] = [1,2,1,2,1] 20 | y.data[1,0,:,0] = [1,2,1,2,1] 21 | 22 | alpha = 10**-5 23 | A = dot(x,x.T) + alpha*dot(y,y.T) 24 | 25 | A = A[:,:2] 26 | 27 | 28 | # Method 1: Naive approach 29 | Apinv = dot(inv(dot(A.T,A)),A.T) 30 | 31 | print('naive approach: A Apinv A - A = 0 \n', dot(dot(A, Apinv),A) - A) 32 | print('naive approach: Apinv A Apinv - Apinv = 0 \n', dot(dot(Apinv, A),Apinv) - Apinv) 33 | print('naive approach: (Apinv A)^T - Apinv A = 0 \n', dot(Apinv, A).T - dot(Apinv, A)) 34 | print('naive approach: (A Apinv)^T - A Apinv = 0 \n', dot(A, Apinv).T - dot(A, Apinv)) 35 | 36 | 37 | # Method 2: Using the differentiated QR decomposition 38 | Q,R = qr(A) 39 | tmp1 = solve(R.T, A.T) 40 | tmp2 = solve(R, tmp1) 41 | Apinv = tmp2 42 | 43 | print('QR approach: A Apinv A - A = 0 \n', dot(dot(A, Apinv),A) - A) 44 | print('QR approach: Apinv A Apinv - Apinv = 0 \n', dot(dot(Apinv, A),Apinv) - Apinv) 45 | print('QR approach: (Apinv A)^T - Apinv A = 0 \n', dot(Apinv, A).T - dot(Apinv, A)) 46 | print('QR approach: (A Apinv)^T - A Apinv = 0 \n', dot(A, Apinv).T - dot(A, Apinv)) 47 | 48 | # Method 3: Stable evaluation of the analytical derivative formula 49 | 50 | A0 = A.data[0,0] 51 | A1 = A.data[1,0] 52 | 53 | Q0, R0 = numpy.linalg.qr(A0) 54 | 55 | # compute nominal solution 56 | tmp1 = solve(R0.T, A0.T) 57 | C0 = solve(R0, tmp1) 58 | 59 | # compute first directional derivative 60 | tmp2 = A1.T - dot( dot(A1.T, A0) + dot(A0.T, A1), C0) 61 | tmp1 = solve(R0.T, tmp2) 62 | C1 = solve(R0, tmp1) 63 | 64 | Apinv.data[0,0] = C0 65 | Apinv.data[1,0] = C1 66 | 67 | print('analytical approach: A Apinv A - A = 0 \n', dot(dot(A, Apinv),A) - A) 68 | print('analytical approach: Apinv A Apinv - Apinv = 0 \n', dot(dot(Apinv, A),Apinv) - Apinv) 69 | print('analytical approach: (Apinv A)^T - Apinv A = 0 \n', dot(Apinv, A).T - dot(Apinv, A)) 70 | print('analytical approach: (A Apinv)^T - A Apinv = 0 \n', dot(A, Apinv).T - dot(A, Apinv)) 71 | -------------------------------------------------------------------------------- /documentation/ICCS2010/odoe_example.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from numpy.testing import * 3 | from prettyplotting import * 4 | from algopy.utpm import * 5 | 6 | # STEP 0: setting parameters and general problem definition 7 | ############################################################ 8 | 9 | # Np = number of parameters p 10 | # Nq = number of control variables q 11 | # Nm = number of measurements 12 | # P = number of vectorized operations at once (SPMD) 13 | 14 | D,Nq,Np,Nm = 2,1,11,30 15 | P = Np 16 | q = UTPM(numpy.zeros((D,P,Nq))) 17 | q0 = 1. 18 | q.data[0,:,0] = q0 19 | p = UTPM(numpy.zeros((D,P,Np))) 20 | p0 = numpy.random.rand(Np) 21 | p.data[0,:,:] = p0 22 | p.data[1,:,:] = numpy.eye(P) 23 | 24 | B = UTPM(numpy.zeros((D,P,Nm, Np))) 25 | B0 = numpy.random.rand(Nm,Np) 26 | B.data[0,:] = B0 27 | 28 | 29 | # STEP 1: compute push forward 30 | ############################################################ 31 | 32 | G = UTPM.dot(B, p) 33 | F = G * q[0] 34 | J = F.FtoJT().T 35 | 36 | Q,R = UTPM.qr(J) 37 | Id = numpy.eye(Np) 38 | Rinv = UTPM.solve(R,Id) 39 | C = UTPM.dot(Rinv,Rinv.T) 40 | l,U = UTPM.eigh(C) 41 | arg = UTPM.argmax(l) 42 | 43 | # check correctness of the push forward 44 | tmp1 = q0**-2* numpy.linalg.inv(numpy.dot(B0.T,B0)) 45 | l0, U0 = numpy.linalg.eigh( tmp1 ) 46 | 47 | assert_array_almost_equal( J.data[0,0], B0) 48 | assert_array_almost_equal( C.data[0,0], tmp1) 49 | assert_array_almost_equal(l.data[0,0], l0) 50 | assert_array_almost_equal(U.data[0,0], U0) 51 | 52 | 53 | 54 | # STEP 2: compute pullback 55 | ############################################################ 56 | 57 | lbar = UTPM(numpy.zeros(l.data.shape)) 58 | lbar.data[0,0, arg] = 1. 59 | Ubar = UTPM(numpy.zeros(U.data.shape)) 60 | 61 | Cbar = UTPM.pb_eigh(lbar, Ubar, C, l, U) 62 | Rinvbar, RinvTbar = UTPM.pb_dot(Cbar, Rinv, Rinv.T, C) 63 | Rinvbar += RinvTbar.T 64 | Rbar, Idbar = UTPM.pb_solve(Rinvbar, R, Id, Rinv) 65 | Qbar = UTPM(numpy.zeros(Q.data.shape)) 66 | Jbar = UTPM.pb_qr(Qbar, Rbar, J, Q, R) 67 | 68 | Fbar = Jbar.T.JTtoF() 69 | qbars = UTPM.dot(G.T,Fbar) 70 | 71 | # accumulate over the different directions 72 | qbar = UTPM(numpy.zeros((D,1))) 73 | qbar.data[:,0] = numpy.sum( qbars.data[:,:], axis=1) 74 | 75 | ############################################# 76 | # compare with analytical solution 77 | ############################################# 78 | c = numpy.max(numpy.linalg.eig( numpy.linalg.inv(numpy.dot(B0.T, B0)))[0]) 79 | dPhidq = - 2* c * q0**-3 80 | 81 | assert_almost_equal( dPhidq, qbar.data[1,0]) 82 | print('symbolical - UTPM pullback = %e'%( numpy.abs(dPhidq - qbar.data[1,0]))) 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/series_expansion.rst: -------------------------------------------------------------------------------- 1 | Univariate Taylor Series Expansions 2 | ----------------------------------------------- 3 | 4 | As an easy example we want to compute the Taylor series expansion of 5 | 6 | .. math:: 7 | y = f(x) = \sin(\cos(x) + \sin(x)) 8 | 9 | about :math:`x_0 = 0.3`. The first thing to notice is that we can as well compute the 10 | Taylor series expansion of 11 | 12 | .. math:: 13 | y = f(x_0 + t) = \sin(\cos(x_0 + t) + \sin(x_0 + t)) 14 | 15 | about :math:`t = 0`. Taylor's theorem yields 16 | 17 | .. math:: 18 | f(x_0 + t) &= \sum_{d=0}^{D-1} y_d t^d + R_{D}(t) \\ 19 | \mbox{where } \quad y_d &= \left. \frac{1}{d!} \frac{d^d }{d t^d}f(x_0 + t) \right|_{t = 0} \;. 20 | 21 | and :math:`R_D(x)` is the remainder term. 22 | 23 | Slightly rewritten one has 24 | 25 | .. math:: 26 | y(t) = f(x(t)) + \mathcal O(t^D) 27 | 28 | i.e., one has a polynomial :math:`x(t) = \sum_{d=0}^{D-1} x_d t^d` as input and 29 | computes a polynomial :math:`y(t) = \sum_{d=0}^{D-1} y_d t^d + \mathcal O(t^d)` as output. 30 | 31 | This is now formulated in a way that can be used with ALGOPY. 32 | 33 | .. literalinclude:: series_expansion.py 34 | :lines: 0-13 35 | 36 | Don't be confused by the P. It can be used to evaluate several Taylor series expansions 37 | at once. The important point to notice is that the D in the code is the same D 38 | as in the formula above. I.e., it is the number of coefficients in the polynomials. 39 | The important point is 40 | 41 | .. warning:: The coefficients of the univariate Taylor polynomial (UTP) are stored in 42 | the attribute UTPM.data. It is a x.ndarray with shape (D,P) + shape of the coefficient. 43 | In this example, the coefficients :math:`x_d` are scalars and thus x.data.shape = (D,P). 44 | However, if the the coefficients were vectors of size N, then x.data.shape would be (D,P,N), 45 | and if the coefficients were matrices with shape (M,N), then x.data.shape would be (D,P,M,N). 46 | 47 | 48 | 49 | 50 | To see that ALGOPY indeed computes the correct Taylor series expansion we plot 51 | the original function and the Taylor polynomials evaluated at different orders. 52 | 53 | .. literalinclude:: series_expansion.py 54 | :lines: 14- 55 | 56 | 57 | .. figure:: taylor_approximation.png 58 | :align: center 59 | :scale: 50 60 | 61 | This plot shows Taylor approximations of different orders. 62 | The point :math:`x_0 = 0.3` is plotted as a red dot and the original 63 | function is plotted as black dots. One can see that the higher the order, 64 | the better the approximation. 65 | 66 | 67 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/first_order_forward.rst: -------------------------------------------------------------------------------- 1 | First Order Derivatives in the Forward Mode of Algorithmic Differentiation 2 | ========================================================================== 3 | 4 | Example 1: 5 | 6 | In this example we want to show how one can extract derivatives 7 | from the computed univariate Taylor polynomials (UTP). For simplicity we only show 8 | first-order derivatives but ALGOPY also supports the computation of higher-order 9 | derivatives by an interpolation-evaluation approach. 10 | 11 | The basic observation is that by use of the chain rule one obtains functions 12 | :math:`F: \mathbb R^N \rightarrow \mathbb R^M` 13 | 14 | .. math:: 15 | \left. \frac{d}{d t} F(x_0 + x_1 t) \right|_{t=0} = \left. \frac{d}{d x} f(x) \right|_{x = x_0} \cdot x_1\;. 16 | 17 | i.e. a Jacobian-vector product. 18 | 19 | Again, we look a simple contrived example and we want to compute the first column 20 | of the Jacobian, i.e., :math:`x_1 = (1,0,0)`. 21 | 22 | .. literalinclude:: first_order_forward.py 23 | :lines: 1-30 24 | 25 | As output one gets:: 26 | 27 | y0 = [ 3. 15. 5.] 28 | y = [[[ 3. 15. 5.]] 29 | 30 | [[ 3. 0. 5.]]] 31 | y.shape = (3,) 32 | y.data.shape = (2, 1, 3) 33 | dF/dx(x0) * x1 = [ 3. 0. 5.] 34 | 35 | 36 | and the question is how to interpret this result. First off, y0 is just the usual 37 | function evaluation using numpy but y represent a univariate Taylor polynomial (UTP). 38 | One can see that each coefficient of the polynomial has the shape (3,). We extract 39 | the directional derivative as the first coefficient of the UTP. 40 | 41 | One can see that this is indeed the numerical value of first column of the Jacobian J(1,3,5):: 42 | 43 | def J(x): 44 | ret = numpy.zeros((3,3),dtype=float) 45 | ret[0,:] = [x[1], x[0], 0 ] 46 | ret[1,:] = [0, , x[2], x[1]] 47 | ret[2,:] = [x[2], 0 , x[0]] 48 | 49 | 50 | Example 2: 51 | 52 | ALGOPY can not only be used to compute series expansions of simple functions 53 | as shown above. A particular strenght of ALGOPY is that it allows to compute series 54 | expansions through numerical linear algebra functions. 55 | Consider the contrived example that appears in similar form in statistically 56 | motivated functions. It is the goal to compute the directional derivative 57 | 58 | .. math:: 59 | \nabla_x f((3,5)) \cdot \begin{pmatrix} 7 \\ 11 \end{pmatrix} 60 | 61 | .. literalinclude:: first_order_forward.py 62 | :lines: 32- 63 | -------------------------------------------------------------------------------- /documentation/examples/comparison_forward_reverse_mode.py: -------------------------------------------------------------------------------- 1 | """ 2 | We show here how the forward and the reverse mode of AD are used and show 3 | that they produce the same result. 4 | 5 | We consider the function f:R^N ---> R defined by 6 | 7 | def f(x,y): 8 | return dot(x,y) - x*(x-y) 9 | 10 | We want to compute the Hessian of that function. 11 | 12 | """ 13 | 14 | import numpy 15 | from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv, zeros 16 | 17 | 18 | def f(x,N): 19 | return dot(x[:N],x[N:])*x[N:] - x[:N]*(x[:N]-x[N:]) 20 | 21 | # create a CGraph instance that to store the computational trace 22 | cg = CGraph() 23 | 24 | # create an UTPM instance 25 | D,N,M = 2,3,2 26 | P = N 27 | 28 | A = UTPM(numpy.zeros((D,P,M,N))) 29 | x = UTPM(numpy.zeros((D,P,N,1))) 30 | 31 | x.data[0,:] = numpy.random.rand(N,1) 32 | A.data[0,:] = numpy.random.rand(M,N) 33 | 34 | x.data[1,:,:,0] = numpy.eye(P) 35 | 36 | 37 | x = Function(x) 38 | A = Function(A) 39 | 40 | 41 | # wrap the UTPM instance in a Function instance to trace all operations 42 | # that have x as an argument 43 | # x = Function(x) 44 | 45 | y = dot(A,x) 46 | 47 | # define dependent and independent variables in the computational procedure 48 | cg.independentFunctionList = [x,A] 49 | cg.dependentFunctionList = [y] 50 | 51 | # for such linear function we already know the Jacobian: df/dx = A 52 | # y.data is a (D,P,N) array, i.e. we have to transpose to get the Jacobian 53 | # Since the UTPM instrance is wrapped in a Function instance we have to access it 54 | # by y.x. That means the Jacobian is 55 | J = y.x.data[1].T 56 | 57 | # # checking against the analytical result 58 | print('J - A =\n', J - A.x.data[0,0]) 59 | 60 | # Now we want to compute the same Jacobian in the reverse mode of AD 61 | # before we do that we have a look what the computational graph looks like: 62 | # print 'Computational graph is', cg 63 | 64 | # the reverse mode is called by cg.pullback([ybar]) 65 | # it is a little hard to explain what's going on here. Suffice to say that we 66 | # now compute one row of the Jacobian instead of one column as in the forward mode 67 | 68 | ybar = y.x.zeros_like() 69 | 70 | # compute first row of J 71 | ybar.data[0,0,0,0] = 1 72 | cg.pullback([ybar]) 73 | J_row1 = x.xbar.data[0,0] 74 | 75 | # compute second row of J 76 | ybar.data[...] = 0 77 | ybar.data[0,0,1,0] = 1 78 | cg.pullback([ybar]) 79 | J_row2 = x.xbar.data[0,0] 80 | 81 | # build Jacobian 82 | J2 = numpy.vstack([J_row1.T, J_row2.T]) 83 | print('J - J2 =\n', J - J2) 84 | 85 | # one can also easiliy extract the Hessian which is here a (M,N,N)-tensor 86 | # e.g. the hessian of y[1] is zero since y[1] is linear in x 87 | print('Hessian of y[1] w.r.t. x = \n',x.xbar.data[1,:,:,0]) 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/complex_differentiation.rst: -------------------------------------------------------------------------------- 1 | Differentiation of programs containing complex arithmetic 2 | --------------------------------------------------------- 3 | 4 | Complex problems are often formulated in simple equations in the complex numbers. 5 | Therefore, quite often codes containing complex arithmetic have to be differentiated. 6 | Algopy has a partial support for complex arithmetic. In this section, we use the simple function 7 | 8 | 9 | .. math:: 10 | f: \mathbb R^N \to {\mathbb C} \\ 11 | x \mapsto y = f(x) = \sum_{i=1}^N \sum_{j=1}^N A_{ij} x_j 12 | 13 | where :math:`A \in {\mathbb C}^{N \times N}`. 14 | 15 | In that form, it is not clear what the gradient :math:`\nabla_x f(x)` actually means. 16 | There are two possibilites: either change the signature to :math:`f: {\mathbb C}^N \to {\mathbb C}` and compute the complex derivative, 17 | or reformulate the function 18 | 19 | .. math:: 20 | f: \mathbb R^N \to \mathbb R \\ 21 | x \mapsto y = f(x) = {\mathrm Re} ( \sum_{i=1}^N \sum_{j=1}^N A_{ij} x_j) 22 | 23 | to map from reals to reals. 24 | 25 | 26 | variant 1: :math:`\mathbb C^N \to \mathbb C`:: 27 | 28 | import algopy 29 | import numpy as np 30 | 31 | def f(x, A, module): 32 | y = module.dot(A, x) 33 | return module.sum(y) 34 | 35 | size = 4 36 | Ar = np.random.random((size, size)) 37 | Ai = np.random.random((size, size)) 38 | Ac = Ar +1j*Ai 39 | A = Ac 40 | x = np.random.random((size,)) + 0j 41 | 42 | cg = algopy.CGraph() 43 | xf = algopy.Function(x) 44 | sf = f(xf, A, algopy) 45 | cg.trace_off() 46 | 47 | print 'sf.x=', sf.x 48 | assert sf.x == f(x , A, np) 49 | 50 | cg.independentFunctionList = [xf] 51 | cg.dependentFunctionList = [sf] 52 | gf = cg.gradient(x) 53 | ganalytic = np.sum(A, axis=0) 54 | 55 | print 'sf.x=', sf.x 56 | print 'gf=\n',gf 57 | print 'np.sum(A, axis=0)=\n',ganalytic 58 | assert np.allclose(gf, ganalytic) 59 | 60 | 61 | 62 | variant 2: :math:`\mathbb R^N \to \mathbb R`:: 63 | 64 | import algopy 65 | import numpy as np 66 | 67 | def f(x, A, module): 68 | y = module.dot(A, x) 69 | return module.real(module.sum(y)) 70 | 71 | size = 4 72 | Ar = np.random.random((size, size)) 73 | Ai = np.random.random((size, size)) 74 | Ac = Ar +1j*Ai 75 | A = Ac 76 | x = np.random.random((size,)) 77 | 78 | cg = algopy.CGraph() 79 | xf = algopy.Function(x) 80 | sf = f(xf, A, algopy) 81 | cg.trace_off() 82 | 83 | print 'sf.x=', sf.x 84 | assert sf.x == f(x , A, np) 85 | 86 | cg.independentFunctionList = [xf] 87 | cg.dependentFunctionList = [sf] 88 | gf = cg.gradient(x) 89 | ganalytic = np.real(np.sum(A, axis=0)) 90 | 91 | print 'gf=\n',gf 92 | print 'np.sum(A, axis=0)=\n',ganalytic 93 | assert np.allclose(gf, ganalytic) 94 | 95 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/hessian_of_potential_function.py: -------------------------------------------------------------------------------- 1 | # import a tool to use / as a symbol for normal division 2 | 3 | 4 | #import system data 5 | import sys, os 6 | 7 | #Loading the required packages 8 | import scipy as sp 9 | import numpy as np 10 | import matplotlib as mpl 11 | import matplotlib.pyplot as plt 12 | 13 | # and subpackages 14 | from scipy import * 15 | from scipy import linalg, optimize, constants 16 | 17 | import numpy 18 | import algopy 19 | import time 20 | 21 | 22 | 23 | #----------------------------------------------------------------------------------------- 24 | # Hamiltonian H=sum_i(p_i^2/(2m)+ 1/2 * m * w^2 x_i^2)+ sum_(i!=j)(a/|x_i-x_j|) 25 | #----------------------------------------------------------------------------------------- 26 | 27 | class classicalHamiltonian: 28 | def __init__(self): 29 | 30 | self.N = 2 #N is a scalar, it's the number of ions in the chain 31 | f = 1000000 #f is a scalar, it's the trap frequency 32 | self.w = 2*pi*f #w is a scalar, it's the angular velocity corresponding to the trap frequency 33 | self.C = (4*pi*constants.epsilon_0)**(-1)*constants.e**2 #C is a scalar, it's the Coulomb constant times the electronic charge in SI 34 | self.m = 39.96*1.66e-27 #m is the mass of a single trapped ion in the chain 35 | 36 | def potential(self, positionvector): #Defines the potential that is going to be minimized 37 | 38 | x= positionvector #x is an 1-d array (vector) of lenght N that contains the positions of the N ions 39 | w= self.w 40 | C= self.C 41 | m= self.m 42 | 43 | #First we consider the potential of the harmonic osszilator 44 | Vx = 0.5 * m * (w**2) * sum(x**2) 45 | 46 | for i in range(len(x)): 47 | for j in range(i+1, len(x)): 48 | Vx += C * (abs(x[i]-x[j]))**(-1) #then we add the coulomb interaction 49 | 50 | return Vx 51 | 52 | def initialposition(self): #Defines the initial position as an estimate for the minimize process 53 | 54 | N= self.N 55 | x_0 = r_[-(N-1)/2:(N-1)/2:N*1j] 56 | return x_0 57 | 58 | def normal_modes(self, eigenvalues): #the computed eigenvalues of the matrix Vx are of the form (normal_modes)^2*m. 59 | m = self.m 60 | normal_modes = sqrt(eigenvalues/m) 61 | return normal_modes 62 | 63 | c=classicalHamiltonian() 64 | xopt = optimize.fmin(c.potential, c.initialposition(), xtol = 1e-10) 65 | 66 | 67 | x = algopy.UTPM.init_hessian(xopt) 68 | y = c.potential(x) 69 | hessian = algopy.UTPM.extract_hessian(2, y) 70 | 71 | print(hessian) 72 | 73 | 74 | -------------------------------------------------------------------------------- /documentation/examples/error_propagation.rst: -------------------------------------------------------------------------------- 1 | """ 2 | This example shows how ALGOPY can be used for linear error propagation. 3 | 4 | Consider the error model:: 5 | 6 | y = x + \epsilon 7 | 8 | where x a vector and \epsilon a random variable with zero mean and 9 | covariance matrix \Sigma^2. The y is the observed quantity and x is a real vector 10 | representing the "true" value. 11 | 12 | One can find some estimator \hat x that is in some or another way optimal. 13 | For instance one take 100 samples and obtain y_1,y_2,....,y_100 and take the 14 | arithmetic mean as an estimator for x. In the following we simply assume that 15 | some estimate \hat x is known and has an associated confidence region described 16 | by its covariance matrix Sigma^2 = E[(\hat x - E[\hat x])(\hat x - E[\hat x])^T] 17 | 18 | However, not \hat x is of interest but some function f(\hat x):: 19 | 20 | f: R^N ---> R^M 21 | \hat x ---> \hat x = f(\hat x) 22 | 23 | The question is: 24 | 25 | What can we say about the confidence region of the function f(y) when 26 | the confidence region of y is described by the covariance matrix \Sigma^2? 27 | 28 | For affine (linear) functions:: 29 | 30 | z = f(y) = Ay + b 31 | 32 | the procedure is described in the 33 | wikipedia article http://en.wikipedia.org/wiki/Propagation_of_uncertainty . 34 | 35 | For nonlinear functions can be linearized about an estimate \hat y of E[y]. 36 | In the vicinity of \hat y, the linear model approximates the nonlinear function often quite well. 37 | 38 | To linearize the function, the Jacobian J(\hat y) of the function f(\hat y) has to be computed, i.e.: 39 | 40 | z \approx f(y) = f(\hat y) + J(\hat y) (y - \hat y) 41 | 42 | The covariance matrix of z is defined as C = E[z z^T] = E[ J y y^T J^T] = J \Sigma^2 J^T. 43 | That means if we know J(y), we can approximately compute the confidence region if 44 | f(\hat y) is sufficiently linear. 45 | 46 | To compute the Jacobian one can use the forward and the reverse mode of AD: 47 | In the forward mode of AD one computes Nm directional derivatives, i.e. P = Nm. 48 | In the reverse mode of AD one computes M adjoint derivatives, i.e. Q = M. 49 | """ 50 | 51 | import numpy 52 | from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv, zeros 53 | 54 | def f(y): 55 | retval = zeros((3,1),dtype=y) 56 | retval[0,0] = numpy.log(dot(y.T,y)) 57 | retval[1,0] = numpy.exp(dot(y.T,y)) 58 | retval[2,0] = numpy.exp(dot(y.T,y)) - numpy.log(dot(y.T,y)) 59 | return retval 60 | 61 | D,Nm = 2,40 62 | P = Nm 63 | y = UTPM(numpy.zeros((2,P,Nm))) 64 | 65 | y.data[0,:] = numpy.random.rand(Nm) 66 | y.data[1,:] = numpy.eye(Nm) 67 | 68 | 69 | # print f(y) 70 | J = f(y).data[1,:,:,0] 71 | print 'Jacobian J(y) = \n', J 72 | 73 | C_epsilon = 0.3*numpy.eye(Nm) 74 | 75 | print J.shape 76 | 77 | C = dot(J.T, dot(C_epsilon,J)) 78 | 79 | print 'Covariance matrix of z: C = \n',C 80 | 81 | 82 | 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /documentation/examples/error_propagation.py: -------------------------------------------------------------------------------- 1 | """ 2 | This example shows how ALGOPY can be used for linear error propagation. 3 | 4 | Consider the error model:: 5 | 6 | y = x + \epsilon 7 | 8 | where x a vector and \epsilon a random variable with zero mean and 9 | covariance matrix \Sigma^2. The y is the observed quantity and x is a real vector 10 | representing the "true" value. 11 | 12 | One can find some estimator \hat x that is in some or another way optimal. 13 | For instance one take 100 samples and obtain y_1,y_2,....,y_100 and take the 14 | arithmetic mean as an estimator for x. In the following we simply assume that 15 | some estimate \hat x is known and has an associated confidence region described 16 | by its covariance matrix Sigma^2 = E[(\hat x - E[\hat x])(\hat x - E[\hat x])^T] 17 | 18 | However, not \hat x is of interest but some function f(\hat x):: 19 | 20 | f: R^N ---> R^M 21 | \hat x ---> \hat x = f(\hat x) 22 | 23 | The question is: 24 | 25 | What can we say about the confidence region of the function f(y) when 26 | the confidence region of y is described by the covariance matrix \Sigma^2? 27 | 28 | For affine (linear) functions:: 29 | 30 | z = f(y) = Ay + b 31 | 32 | the procedure is described in the 33 | wikipedia article http://en.wikipedia.org/wiki/Propagation_of_uncertainty . 34 | 35 | For nonlinear functions can be linearized about an estimate \hat y of E[y]. 36 | In the vicinity of \hat y, the linear model approximates the nonlinear function often quite well. 37 | 38 | To linearize the function, the Jacobian J(\hat y) of the function f(\hat y) has to be computed, i.e.: 39 | 40 | z \approx f(y) = f(\hat y) + J(\hat y) (y - \hat y) 41 | 42 | The covariance matrix of z is defined as C = E[z z^T] = E[ J y y^T J^T] = J \Sigma^2 J^T. 43 | That means if we know J(y), we can approximately compute the confidence region if 44 | f(\hat y) is sufficiently linear. 45 | 46 | To compute the Jacobian one can use the forward and the reverse mode of AD: 47 | In the forward mode of AD one computes Nm directional derivatives, i.e. P = Nm. 48 | In the reverse mode of AD one computes M adjoint derivatives, i.e. Q = M. 49 | """ 50 | 51 | import numpy 52 | from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv, zeros 53 | 54 | def f(y): 55 | retval = zeros((3,1),dtype=y) 56 | retval[0,0] = numpy.log(dot(y.T,y)) 57 | retval[1,0] = numpy.exp(dot(y.T,y)) 58 | retval[2,0] = numpy.exp(dot(y.T,y)) - numpy.log(dot(y.T,y)) 59 | return retval 60 | 61 | D,Nm = 2,40 62 | P = Nm 63 | y = UTPM(numpy.zeros((2,P,Nm))) 64 | 65 | y.data[0,:] = numpy.random.rand(Nm) 66 | y.data[1,:] = numpy.eye(Nm) 67 | 68 | 69 | # print f(y) 70 | J = f(y).data[1,:,:,0] 71 | print('Jacobian J(y) = \n', J) 72 | 73 | C_epsilon = 0.3*numpy.eye(Nm) 74 | 75 | print(J.shape) 76 | 77 | C = dot(J.T, dot(C_epsilon,J)) 78 | 79 | print('Covariance matrix of z: C = \n',C) 80 | 81 | 82 | 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/covariance_matrix_computation.rst: -------------------------------------------------------------------------------- 1 | Covariance Matrix Computation 2 | ============================= 3 | 4 | In this example it is the goal to compute the gradient of one element 5 | of the covariance matrix :math:`C` of a constrained parameter estimation problem, 6 | i.e, 7 | 8 | .. math:: 9 | \nabla_{J_1, J_2} y 10 | 11 | where :math:`y = C_{11}`. 12 | The covariance matrix satisfies the equation 13 | 14 | .. math:: 15 | C = \begin{pmatrix} I & 0 \end{pmatrix} \ 16 | \begin{pmatrix} J_1^T J_1 & J_2^T \\ J_2 & 0 \end{pmatrix}^{-1} \ 17 | \begin{pmatrix} I \\ 0 \end{pmatrix} 18 | 19 | where :math:`J_1` and :math:`J_2`. 20 | 21 | Two possibilities are compared: 22 | 1) filling a big matrix with elements, then invert it and return a view of 23 | of the upper left part of the matrix 24 | 25 | 2) Computation of the Nullspace of J2 with a QR decomposition. 26 | The formula is :math:`C = Q_2^T( Q_2 J_1^T J_1 Q_2^T)^{-1} Q_2`. 27 | 28 | Potentially, using the QR decomposition twice, i.e. once to compute :math:`Q_2` and 29 | then for :math:`J_1` compute :math:`Q_2^T` to avoid the multiplication which would square the condition 30 | number, may be numerically more stable. This has not been tested yet though. 31 | 32 | The setup for both possibilities to compute the covariance matrix and their derivatives is the same 33 | 34 | .. literalinclude:: covariance_matrix_computation.py 35 | :lines: 1-25 36 | 37 | where: 38 | * D - 1 is the degree of the Taylor polynomial 39 | * P directional derivatives at once 40 | * M number of rows of J1 41 | * N number of cols of J1 42 | * K number of rows of J2 (must be smaller than N) 43 | 44 | At first the naive function evaluation is traced. 45 | 46 | .. literalinclude:: covariance_matrix_computation.py 47 | :lines: 27-36 48 | 49 | then the nullspace method is traced 50 | 51 | .. literalinclude:: covariance_matrix_computation.py 52 | :lines: 38-47 53 | 54 | After that, the function evaluation is stored in the CGraph instance and can be used 55 | to compute the gradient. 56 | 57 | .. literalinclude:: covariance_matrix_computation.py 58 | :lines: 52- 59 | 60 | 61 | One obtains the output:: 62 | 63 | naive approach: dy/dJ1 = [[ 1.09167163 -0.37815832 -0.45414733] 64 | [ 0.57524052 -0.19926504 -0.23930634] 65 | [-1.6063055 0.55642903 0.66824064] 66 | [-0.1674705 0.05801228 0.06966956] 67 | [-1.23363017 0.42733318 0.51320363]] 68 | naive approach: dy/dJ2 = [[ 0.1039174 -0.0359973 -0.04323077]] 69 | nullspace approach: dy/dJ1 = [[ 1.09167163 -0.37815832 -0.45414733] 70 | [ 0.57524052 -0.19926504 -0.23930634] 71 | [-1.6063055 0.55642903 0.66824064] 72 | [-0.1674705 0.05801228 0.06966956] 73 | [-1.23363017 0.42733318 0.51320363]] 74 | nullspace approach: dy/dJ2 = [[ 0.1039174 -0.0359973 -0.04323077]] 75 | 76 | As one can see, both methods yield the same result. 77 | -------------------------------------------------------------------------------- /algopy/linalg/linalg.py: -------------------------------------------------------------------------------- 1 | import string 2 | import numpy 3 | import numpy.linalg 4 | import scipy.linalg 5 | 6 | from algopy import UTPM, Function 7 | 8 | numpy_linalg_function_names = ['inv', 'solve', 'eigh', 'eig', 'svd', 'qr', 'cholesky','transpose', 'det'] 9 | scipy_linalg_function_names = ['lu'] 10 | 11 | 12 | function_template = string.Template(''' 13 | def $function_name(*args, **kwargs): 14 | """ 15 | generic implementation of $function_name 16 | 17 | this function calls, depending on the input arguments, 18 | either 19 | 20 | * numpy.$function_name 21 | * numpy.linalg.$function_name 22 | * args[i].__class__ 23 | 24 | """ 25 | case,arg = 0,0 26 | for na,a in enumerate(args): 27 | if hasattr(a.__class__, '$function_name'): 28 | case = 1 29 | arg = na 30 | break 31 | 32 | if case==1: 33 | return getattr(args[arg].__class__, '$function_name')(*args, **kwargs) 34 | 35 | elif case==0: 36 | return $namespace.__getattribute__('$function_name')(*args, **kwargs) 37 | 38 | else: 39 | return $namespace.__getattribute__('$function_name')(*args, **kwargs) 40 | ''') 41 | 42 | for function_name in numpy_linalg_function_names: 43 | exec(function_template.substitute(function_name=function_name, namespace='numpy.linalg')) 44 | 45 | for function_name in scipy_linalg_function_names: 46 | exec(function_template.substitute(function_name=function_name, namespace='scipy.linalg')) 47 | 48 | def qr_full(A): 49 | """ 50 | Q,R = qr_full(A) 51 | 52 | This function is merely a wrapper of 53 | UTPM.qr_full, Function.qr_full, scipy.linalg.qr 54 | 55 | Parameters 56 | ---------- 57 | 58 | A: algopy.UTPM or algopy.Function or numpy.ndarray 59 | A.shape = (M,N), M >= N 60 | 61 | Returns 62 | -------- 63 | 64 | Q: same type as A 65 | Q.shape = (M,M) 66 | 67 | R: same type as A 68 | R.shape = (M,N) 69 | 70 | 71 | """ 72 | 73 | if isinstance(A, UTPM): 74 | return UTPM.qr_full(A) 75 | 76 | elif isinstance(A, Function): 77 | return Function.qr_full(A) 78 | 79 | elif isinstance(A, numpy.ndarray): 80 | return scipy.linalg.qr(A) 81 | 82 | else: 83 | raise NotImplementedError('don\'t know what to do with this instance') 84 | 85 | 86 | def eigh1(A): 87 | """ 88 | generic implementation of eigh1 89 | """ 90 | 91 | if isinstance(A, UTPM): 92 | return UTPM.eigh1(A) 93 | 94 | elif isinstance(A, Function): 95 | return Function.eigh1(A) 96 | 97 | elif isinstance(A, numpy.ndarray): 98 | A = UTPM(A.reshape((1,1) + A.shape)) 99 | retval = UTPM.eigh1(A) 100 | return retval[0].data[0,0], retval[1].data[0,0],retval[2] 101 | 102 | else: 103 | raise NotImplementedError('don\'t know what to do with this instance') 104 | 105 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/neg_binom_regression.py: -------------------------------------------------------------------------------- 1 | """ 2 | Negative Binomial Regression 3 | 4 | This is an algopy implementation of the statsmodels example 5 | http://statsmodels.sourceforge.net/devel/examples/generated/example_gmle.html 6 | . 7 | """ 8 | 9 | import functools 10 | 11 | import numpy 12 | import scipy.optimize 13 | import algopy 14 | import numdifftools 15 | import pandas 16 | import patsy 17 | 18 | g_url = 'http://vincentarelbundock.github.com/Rdatasets/csv/COUNT/medpar.csv' 19 | 20 | def get_aic(y, X, theta): 21 | return 2*len(theta) + 2*get_neg_ll(y, X, theta) 22 | 23 | def get_neg_ll(y, X, theta): 24 | alpha = theta[-1] 25 | beta = theta[:-1] 26 | a = alpha * algopy.exp(algopy.dot(X, beta)) 27 | ll = algopy.sum( 28 | -y*algopy.log1p(1/a) + 29 | -algopy.log1p(a) / alpha + 30 | algopy.special.gammaln(y + 1/alpha) + 31 | -algopy.special.gammaln(y + 1) + 32 | -algopy.special.gammaln(1/alpha)) 33 | neg_ll = -ll 34 | return neg_ll 35 | 36 | def eval_grad(f, theta): 37 | theta = algopy.UTPM.init_jacobian(theta) 38 | return algopy.UTPM.extract_jacobian(f(theta)) 39 | 40 | def eval_hess(f, theta): 41 | theta = algopy.UTPM.init_hessian(theta) 42 | return algopy.UTPM.extract_hessian(len(theta), f(theta)) 43 | 44 | def main(): 45 | 46 | # read the data from the internet into numpy arrays 47 | medpar = pandas.read_csv(g_url) 48 | y_patsy, X_patsy = patsy.dmatrices('los~type2+type3+hmo+white', medpar) 49 | y = numpy.array(y_patsy).flatten() 50 | X = numpy.array(X_patsy) 51 | 52 | # define the objective function and the autodiff gradient and hessian 53 | f = functools.partial(get_neg_ll, y, X) 54 | g = functools.partial(eval_grad, f) 55 | h = functools.partial(eval_hess, f) 56 | 57 | # init the search for max likelihood parameters 58 | theta0 = numpy.array([ 59 | numpy.log(numpy.mean(y)), 60 | 0, 0, 0, 0, 61 | 0.5, 62 | ], dtype=float) 63 | 64 | # do the max likelihood search 65 | results = scipy.optimize.fmin_ncg( 66 | f, 67 | theta0, 68 | fprime=g, 69 | fhess=h, 70 | avextol=1e-6, 71 | ) 72 | 73 | # compute the hessian a couple of different ways 74 | algopy_hessian = h(results) 75 | num_hessian = numdifftools.Hessian(f)(results) 76 | 77 | # report the results of the search including aic and standard error 78 | print('search results:') 79 | print(results) 80 | print() 81 | print('aic:') 82 | print(get_aic(y, X, results)) 83 | print() 84 | print('standard error using observed fisher information,') 85 | print('with hessian computed using algopy:') 86 | print(numpy.sqrt(numpy.diag(scipy.linalg.inv(algopy_hessian)))) 87 | print() 88 | print('standard error using observed fisher information,') 89 | print('with hessian computed using numdifftools:') 90 | print(numpy.sqrt(numpy.diag(scipy.linalg.inv(num_hessian)))) 91 | print() 92 | 93 | 94 | if __name__ == '__main__': 95 | main() 96 | 97 | -------------------------------------------------------------------------------- /experimental/tests/trash/unit_test_with_sympy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ####################################################### 3 | # This is a Unit Test t that makes use of the Python # 4 | # Module Sympy. # 5 | ####################################################### 6 | 7 | import sympy as sym 8 | from numpy import array, zeros, ones, shape 9 | from numpy.random import random 10 | from numpy.linalg import norm 11 | from forward_mode import * 12 | 13 | N = 4 14 | D = 3 15 | M = N + 3 16 | 17 | xs = array([[sym.Symbol('x%d%d'%(n,d)) for d in range(D)] for n in range(N)]) 18 | # computing the function f: R^(NxD) -> R symbolically 19 | fs = 0 20 | for n in range(1,N): 21 | for m in range(n): 22 | tmp = 0 23 | for d in range(D): 24 | tmp += (xs[n,d] - xs[m,d])**2 25 | tmp = sym.sqrt(tmp) 26 | fs += tmp 27 | 28 | # computing the gradient symbolically 29 | dfs = array([[sym.diff(fs, xs[n,d]) for d in range(D)] for n in range(N)]) 30 | 31 | # computing the Hessian symbolically 32 | ddfs = array([[[[ sym.diff(dfs[m,e], xs[n,d]) for d in range(D)] for n in range(N)] for e in range(D) ] for m in range(N)]) 33 | 34 | 35 | # function f 36 | def f(x): 37 | retval = 0. 38 | for n in range(1,N): 39 | for m in range(n): 40 | retval += 1./ norm(x[n,:] - x[m,:]) 41 | return retval 42 | 43 | def df(x): 44 | g = zeros(shape(x),dtype=float) 45 | for n in range(N): 46 | for d in range(D): 47 | for m in range(N): 48 | if n != m: 49 | g[n,d] += (x[n,d] - x[m,d])/norm(x[n,:]-x[m,:]) 50 | return g 51 | 52 | def ddf(x): 53 | N,D = shape(x) 54 | H = zeros((N,D,N,D),dtype=float) 55 | for n in range(N): 56 | for d in range(D): 57 | for m in range(N): 58 | for e in range(D): 59 | for l in range(N): 60 | if l==n: 61 | continue 62 | H[n,d,m,e] -= (( (m==n) * (d==e) - (m==l)*(d==e) ) - 3* (x[n,d] - x[l,d])/norm(x[n,:]-x[l,:])**2 * ( (n==m) - (m==l))*( x[n,e] - x[l,e]))/norm(x[n,:] - x[l,:])**3 63 | return H 64 | 65 | def sym_df(x): 66 | symdict = dict() 67 | for n in range(N): 68 | for d in range(D): 69 | symdict[xs[n,d]] = x[n,d] 70 | return array([[dfs[n,d].subs_dict(symdict).evalf() for d in range(D)] for n in range(N)]) 71 | 72 | def sym_ddf(x): 73 | symdict = dict() 74 | for n in range(N): 75 | for d in range(D): 76 | symdict[xs[n,d]] = x[n,d] 77 | return array([[[[ ddfs[m,e,n,d].subs_dict(symdict).evalf() for d in range(D)] for n in range(N)] for e in range(D)] for m in range(N)],dtype=float) 78 | 79 | def ad_df(x): 80 | return gradient(f,x) 81 | 82 | def ad_ddf(x): 83 | return hessian(f,x) 84 | 85 | 86 | # point at which the derivatives should be evaluated 87 | x = random((N,D)) 88 | print('\n\n') 89 | print('Sympy vs Hand Derived Gradient check (should be almost zero)') 90 | print(df(x) - sym_df(x)) 91 | print('Sympy vs Ad Derive Gradient check (should be almost zero)') 92 | print(ad_df(x) - sym_df(x)) 93 | 94 | #print '\n\n' 95 | #print 'Hessian check (should be almost zero)' 96 | #print ddf(x) - sym_ddf(x) 97 | 98 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /documentation/examples/covariance_matrix_computation.py: -------------------------------------------------------------------------------- 1 | """ 2 | In this example it is the goal to compute derivatives of the covariance matrix 3 | of a constrained parameter estimation problem. 4 | 5 | I.e. compute 6 | 7 | / J1.T J1 J2.T \^-1 / 1 \ 8 | C = (1,0) | | | | 9 | \ J2 0 / \ 0 / 10 | 11 | where J1 = J1(x) and J2 = J2(x). The vector x is Nx dimensional. 12 | 13 | Two possibilities are compared: 14 | 1) filling a big matrix with elements, then invert it and return a view of 15 | of the upper left part of the matrix 16 | 17 | 2) Computation of the Nullspace of J2 with a QR decomposition. 18 | The formula is:: 19 | C = Q2.T( Q2 J1.T J1 Q2.T)^-1 Q2 . 20 | Potentially, using the QR decomposition twice, i.e. once to compute Q2 and 21 | then for J1 Q2.T to avoid the multiplication which would square the condition 22 | number, may be numerically more stable. This has not been tested yet though. 23 | This example only shows the computation of Q2. 24 | 25 | I.e. this small example is a demonstration how the QR decomposition on polynomial 26 | matrices can be used. 27 | """ 28 | 29 | import numpy 30 | from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv, solve 31 | 32 | # first order derivatives, one directional derivative 33 | # D - 1 is the degree of the Taylor polynomial 34 | # P directional derivatives at once 35 | # M number of rows of J1 36 | # N number of cols of J1 37 | # K number of rows of J2 (must be smaller than N) 38 | D,P,M,N,K,Nx = 2,1,100,3,1,1 39 | 40 | 41 | # METHOD 1: nullspace method 42 | cg1 = CGraph() 43 | 44 | J1 = Function(UTPM(numpy.random.rand(*(D,P,M,N)))) 45 | J2 = Function(UTPM(numpy.random.rand(*(D,P,K,N)))) 46 | 47 | 48 | Q,R = Function.qr_full(J2.T) 49 | Q2 = Q[:,K:].T 50 | 51 | J1_tilde = dot(J1,Q2.T) 52 | Q,R = qr(J1_tilde) 53 | V = solve(R.T, Q2) 54 | C = dot(V.T,V) 55 | cg1.trace_off() 56 | 57 | cg1.independentFunctionList = [J1, J2] 58 | cg1.dependentFunctionList = [C] 59 | 60 | print('covariance matrix: C =\n',C) 61 | print('check that Q2.T spans the nullspace of J2:\n', dot(J2,Q2.T)) 62 | 63 | # METHOD 2: image space method (potentially numerically unstable) 64 | cg2 = CGraph() 65 | 66 | J1 = Function(J1.x) 67 | J2 = Function(J2.x) 68 | 69 | M = Function(UTPM(numpy.zeros((D,P,N+K,N+K)))) 70 | M[:N,:N] = dot(J1.T,J1) 71 | M[:N,N:] = J2.T 72 | M[N:,:N] = J2 73 | C2 = inv(M)[:N,:N] 74 | cg2.trace_off() 75 | 76 | cg2.independentFunctionList = [J1, J2] 77 | cg2.dependentFunctionList = [C2] 78 | 79 | print('covariance matrix: C =\n',C2) 80 | print('difference between image and nullspace method:\n',C - C2) 81 | 82 | Cbar = UTPM(numpy.random.rand(D,P,N,N)) 83 | 84 | cg1.pullback([Cbar]) 85 | 86 | cg2.pullback([Cbar]) 87 | print('J1\n',cg2.independentFunctionList[0].xbar - cg1.independentFunctionList[0].xbar) 88 | print('J2\n',cg2.independentFunctionList[1].xbar - cg1.independentFunctionList[1].xbar) 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | **Version 0.7.1** 2 | 3 | * Support for numpy==2.0.0. Dropping support for numpy 1.x.x. 4 | 5 | **Version 0.6.0** 6 | 7 | * Updated Algopy to support recent versions of numpy==1.26.4 and scipy==1.13.1 8 | * Scipy has removed support for confluent hypergeometric functions. Thus, the support of these functions in Algopy has been also removed in this version. 9 | * Numpy has removed support for nosetests. Thus, algopy now uses pytest to run tests via `pytest -v algopy` 10 | 11 | **Version 0.5.0** 12 | 13 | * add Python 3 compatibility 14 | * add Travis CI 15 | 16 | **Version 0.4.0** 17 | 18 | * added support for a variety of new functions, mostly contributed by 19 | Alex Griffing, NCSU: 20 | expm, hyp1f1, hyperu, hyp2f0, polygamma, psi, erf, erfi, dawsn, logit, expit 21 | 22 | **Version 0.3.2** 23 | 24 | * improved error reporting in the reverse mode: when "something goes wrong" 25 | in cg.gradient([1.,2.,3.]) one now gets a much more detailed traceback 26 | * added A.reshape(...) support to the reverse mode 27 | * improved support for broadcasting for UTPM instances 28 | 29 | **Version 0.3.1** 30 | 31 | * replaced algopy.sum by a faster implementation 32 | * fixed a bug in getitem of the UTPM instance: now works also with numpy.int64 33 | as index 34 | * added dedicated algopy.sum and algopy.prod 35 | * added UTPM.pb_sqrt 36 | * fixed bug in tracing operations involving neg(x) 37 | * added algopy.outer 38 | * changed API of CGraph.hessian, CGraph.jac_vec etc. One has now to write 39 | CGraph.jacobian(x) instead of CGraph.jacobian([x]). 40 | 41 | **Version 0.3.0** 42 | 43 | * renamed push_forward to pushforward, this is more consistent w.r.t. the pullback 44 | * UTPM.__repr__ now returns a string of the form `UTPM(...)` 45 | * refactored the tracer: it should now be possible to trace the function evaluation with normal numpy.ndarrays. After that, one can use cg.pushforward with UTPM instances or call cg.gradient, etc. 46 | * UTPM.reshape is now a method, not a class method 47 | * added broadcasting support for __setitem__, iadd and isub 48 | * added Function.ndim 49 | * added preliminary complex numbers support for arithmetic with UTPM instances (reverse mode using the tracer is not supported yet) 50 | * UTPM.reshape now can also take integers as input, not only tuples of integers 51 | * added UTPM.tan, UTPM.arcsin, UTPM.arccos, UTPM.arctan, UTPM.sinh, UTPM.cosh, UTPM.tanh 52 | * made init_hessian and extract_hessian generic (to make it useful for complex valued functions) 53 | * added comparison operators <,>,<=,>=,== to UTPM 54 | * added UTPM.init_jac_vec and UTPM.extract_jac_vec 55 | * added CGraph.function, CGraph.gradient, CGraph.hessian, CGraph.hess_vec 56 | 57 | **Version 0.2.3** 58 | 59 | * added UTPM.init_jacobian and UTPM.extract_jacobian 60 | * added UTPM.init_hessian and UTPM.extract_hessian 61 | * added UTPM.init_tensor and UTPM.extract_tensor 62 | * added UTPM.__len__, i.e. len(x) works now for x a UTPM instance 63 | * fixed a bug in algopy.dot(x,y) in the case when x is a numpy.ndarray and y is a UTPM instance 64 | 65 | **Version 0.2.2** 66 | 67 | * fixed some broadcasting bugs with UTPM instances 68 | * fixed a bug in algopy.zeros 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /experimental/examples/newtons_method.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from pylab import * 3 | import sys 4 | sys.path = ['..'] + sys.path 5 | from matrix_ad import * 6 | import adolc 7 | import numpy.random 8 | import scipy.optimize 9 | import numpy.linalg 10 | 11 | 12 | """ 13 | goal: compute the Hessian of the function 14 | 15 | Phi(F) = trace(F*F) 16 | F = [[x*y,x**2],[x**2*y,y**3*x]] 17 | """ 18 | 19 | 20 | # OBJECTIVE FUNCTION 21 | # ------------------ 22 | def Phi(F): 23 | return trace( dot(F.T,F)) 24 | 25 | def ffcn(x): 26 | return 0.5*array( 27 | [[(x[0]-17.)*(x[0]-17.), (x[0]-17.)*(x[0]-17.)], 28 | [ x[1]-19. , x[1]-19.]]) 29 | 30 | # TAPING THE FUNCTIONS 31 | # -------------------- 32 | # taping function ffcn 33 | u = 3.; v = 7. 34 | ax = array([adolc.adouble(u), adolc.adouble(v)]) 35 | adolc.trace_on(1) 36 | ax[0].is_independent(u) 37 | ax[1].is_independent(v) 38 | ay = ffcn(ax) 39 | for n in range(2): 40 | for m in range(2): 41 | adolc.depends_on(ay[n,m]) 42 | adolc.trace_off() 43 | 44 | # taping matrix functions with algopy 45 | x = array([u,v]) 46 | F = ffcn(x) 47 | Fdot = zeros((2,2)) 48 | cg = CGraph() 49 | FF = Function(Mtc(F)) 50 | Fy = Phi(FF) 51 | cg.independentFunctionList = [FF] 52 | cg.dependentFunctionList = [Fy] 53 | 54 | # COMPUTING THE HESSIAN H = d^2 Phi/ dx^2 55 | # --------------------------------------- 56 | # need for that to propagate two directions 57 | # then reverse 58 | 59 | def gradient_and_hessian_of_Phi(x): 60 | H = zeros((2,2)) # Hessian 61 | g = zeros(2) # gradient 62 | V = zeros((2,1)) 63 | F = zeros((2,2)) 64 | Fdot = zeros((2,2)) 65 | D = 1 66 | keep = D+1 67 | 68 | 69 | for n in range(2): 70 | # 1: hos_forward, propagate two directions 71 | V[n,0] = 1. 72 | (y,W) = adolc.hos_forward(1,x,V,keep) 73 | V[n,0] = 0. 74 | F[0,:] = y[:2] 75 | F[1,:] = y[2:] 76 | Fdot[0,:] = W[:2,0] 77 | Fdot[1,:] = W[2:,0] 78 | 79 | # 2: matrix forward 80 | cg.forward([Mtc(F,Fdot)]) 81 | 82 | # 3: matrix reverse 83 | Phibar = array([[1.]]) 84 | Phibardot = array([[0.]]) 85 | cg.reverse([Mtc(Phibar, Phibardot)]) 86 | 87 | # 4: hov_reverse 88 | U = zeros((1,4,2)) 89 | U[0,:,0] = cg.independentFunctionList[0].xbar.X.flatten() 90 | U[0,:,1] = cg.independentFunctionList[0].xbar.Xdot.flatten() 91 | res = adolc.hov_ti_reverse(1,U)[0].copy() 92 | g[:] = res[0,:,0] 93 | H[n,:] = res[0,:,1] 94 | 95 | return (g,H) 96 | 97 | def newtons_method(x0): 98 | x = x0.copy() 99 | 100 | g = numpy.inf 101 | k = 0 102 | while numpy.linalg.norm(g)>10**-12: 103 | print('iteration: %2d'%k); k+=1 104 | (g,H) = gradient_and_hessian_of_Phi(x) 105 | # check correctness of the Hessian 106 | # true gradient: d Phi = [ 2 (x[0] - 17)**3, (x[1] - 19 ] 107 | # true Hessian : d**2 Phi = [[ 6 ( x[0] - 17)**2, 0 ],[0,1]] 108 | assert abs( 6*(x[0]-17.)**2 - H[0,0]) <= 10**-9 109 | assert abs( H[1,0]) <= 10**-9 110 | assert abs( H[0,1]) <= 10**-9 111 | assert abs( 1. - H[1,1]) <= 10**-9 112 | 113 | 114 | # compute new search direction 115 | delta_x = numpy.linalg.solve(H,-g) 116 | #update x 117 | x += delta_x 118 | return x 119 | 120 | x = numpy.array([13.,17.]) 121 | print('Solution found by Newton\'s method:', newtons_method(x)) 122 | print('True solution: [17,19]') 123 | -------------------------------------------------------------------------------- /documentation/examples/moore_penrose_pseudoinverse.py: -------------------------------------------------------------------------------- 1 | """ 2 | In this example it is the goal to compute _derivatives_ of the Moore-Penrose pseudoinverse 3 | in a stable manner. We compare different possibilities: 4 | 5 | 1) A naive approach where A^T A is explicitly computed (numerically unstable) 6 | 2) A QR approach where at first a QR decomposition of A is formed and the inverse 7 | is computed by a forward and then back substition of R. This method uses 8 | the differentiated QR decomposition. 9 | 3) A direct approach where an analytic formula for the derivatives of the 10 | Moore-Penrose Formula is derived. Then usage of the QR decomposition is 11 | used to make the computation of the analytical formula numerically stable. 12 | 13 | More explicitly, we compute:: 14 | 15 | A^\dagger = (A^T A)^{-1} A^T 16 | 17 | where A is a (M,N) array (M >= N) with possibly bad condition number. 18 | """ 19 | 20 | import numpy 21 | from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv, solve 22 | 23 | # first order derivatives, one directional derivative 24 | # D - 1 is the degree of the Taylor polynomial 25 | # P directional derivatives at once 26 | # M number of rows of A 27 | # N number of cols of A 28 | D,P,M,N = 2,1,5,2 29 | 30 | # generate badly conditioned matrix A 31 | A = UTPM(numpy.zeros((D,P,M,N))) 32 | x = UTPM(numpy.zeros((D,P,M,1))) 33 | y = UTPM(numpy.zeros((D,P,M,1))) 34 | 35 | x.data[0,0,:,0] = [1,1,1,1,1] 36 | x.data[1,0,:,0] = [1,1,1,1,1] 37 | 38 | y.data[0,0,:,0] = [1,2,1,2,1] 39 | y.data[1,0,:,0] = [1,2,1,2,1] 40 | 41 | alpha = 10**-5 42 | A = dot(x,x.T) + alpha*dot(y,y.T) 43 | 44 | A = A[:,:2] 45 | 46 | 47 | # Method 1: Naive approach 48 | Apinv = dot(inv(dot(A.T,A)),A.T) 49 | 50 | print('naive approach: A Apinv A - A = 0 \n', dot(dot(A, Apinv),A) - A) 51 | print('naive approach: Apinv A Apinv - Apinv = 0 \n', dot(dot(Apinv, A),Apinv) - Apinv) 52 | print('naive approach: (Apinv A)^T - Apinv A = 0 \n', dot(Apinv, A).T - dot(Apinv, A)) 53 | print('naive approach: (A Apinv)^T - A Apinv = 0 \n', dot(A, Apinv).T - dot(A, Apinv)) 54 | 55 | 56 | # Method 2: Using the differentiated QR decomposition 57 | Q,R = qr(A) 58 | tmp1 = solve(R.T, A.T) 59 | tmp2 = solve(R, tmp1) 60 | Apinv = tmp2 61 | 62 | print('QR approach: A Apinv A - A = 0 \n', dot(dot(A, Apinv),A) - A) 63 | print('QR approach: Apinv A Apinv - Apinv = 0 \n', dot(dot(Apinv, A),Apinv) - Apinv) 64 | print('QR approach: (Apinv A)^T - Apinv A = 0 \n', dot(Apinv, A).T - dot(Apinv, A)) 65 | print('QR approach: (A Apinv)^T - A Apinv = 0 \n', dot(A, Apinv).T - dot(A, Apinv)) 66 | 67 | # Method 3: Stable evaluation of the analytical derivative formula 68 | 69 | A0 = A.data[0,0] 70 | A1 = A.data[1,0] 71 | 72 | Q0, R0 = numpy.linalg.qr(A0) 73 | 74 | # compute nominal solution 75 | tmp1 = solve(R0.T, A0.T) 76 | C0 = solve(R0, tmp1) 77 | 78 | # compute first directional derivative 79 | tmp2 = A1.T - dot( dot(A1.T, A0) + dot(A0.T, A1), C0) 80 | tmp1 = solve(R0.T, tmp2) 81 | C1 = solve(R0, tmp1) 82 | 83 | Apinv.data[0,0] = C0 84 | Apinv.data[1,0] = C1 85 | 86 | print('analytical approach: A Apinv A - A = 0 \n', dot(dot(A, Apinv),A) - A) 87 | print('analytical approach: Apinv A Apinv - Apinv = 0 \n', dot(dot(Apinv, A),Apinv) - Apinv) 88 | print('analytical approach: (Apinv A)^T - Apinv A = 0 \n', dot(Apinv, A).T - dot(Apinv, A)) 89 | print('analytical approach: (A Apinv)^T - A Apinv = 0 \n', dot(A, Apinv).T - dot(A, Apinv)) 90 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/minimal_surface.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import algopy 3 | 4 | def O_tilde(u): 5 | """ this is the objective function""" 6 | M = numpy.shape(u)[0] 7 | h = 1./(M-1) 8 | return M**2*h**2 + numpy.sum(0.25*( (u[1:,1:] - u[0:-1,0:-1])**2 + (u[1:,0:-1] - u[0:-1, 1:])**2)) 9 | 10 | 11 | 12 | # INITIAL VALUES 13 | M = 30 14 | h = 1./M 15 | u = numpy.zeros((M,M),dtype=float) 16 | u[0,:]= [numpy.sin(numpy.pi*j*h/2.) for j in range(M)] 17 | u[-1,:] = [ numpy.exp(numpy.pi/2) * numpy.sin(numpy.pi * j * h / 2.) for j in range(M)] 18 | u[:,0]= 0 19 | u[:,-1]= [ numpy.exp(i*h*numpy.pi/2.) for i in range(M)] 20 | 21 | # trace the function evaluation and store it in cg 22 | cg = algopy.CGraph() 23 | Fu = algopy.Function(u) 24 | Fy = O_tilde(Fu) 25 | cg.trace_off() 26 | cg.independentFunctionList = [Fu] 27 | cg.dependentFunctionList = [Fy] 28 | 29 | 30 | def dO_tilde(u): 31 | # use ALGOPY to compute the gradient 32 | g = cg.gradient([u])[0] 33 | 34 | # on the edge the analytical solution is fixed 35 | # so search direction must be zero on the boundary 36 | 37 | g[:,0] = 0 38 | g[0,:] = 0 39 | g[:,-1] = 0 40 | g[-1,:] = 0 41 | return g 42 | 43 | 44 | def projected_gradients(x0, ffcn,dffcn, box_constraints, beta = 0.5, delta = 10**-3, epsilon = 10**-2, max_iter = 1000, line_search_max_iter = 100): 45 | """ 46 | INPUT: box_constraints [L,U], where L (resp. U) vector or matrix with the lower (resp. upper) bounds 47 | """ 48 | x = x0.copy() 49 | L = numpy.array(box_constraints[0]) 50 | U = numpy.array(box_constraints[1]) 51 | def pgn(s): 52 | a = 1.* (x>L) 53 | b = 1.*(abs(x-L) <0.00001) 54 | c = 1.*(s>0) 55 | d = numpy.where( a + (b*c)) 56 | return numpy.sum(s[d]) 57 | 58 | def P(x, s, alpha): 59 | x_alpha = x + alpha * s 60 | a = x_alpha-L 61 | b = U - x_alpha 62 | return x_alpha - 1.*(a<0) * a + b * 1. * (b<0) 63 | 64 | 65 | s = - dffcn(x) 66 | k = 0 67 | while pgn(s)>epsilon and k<= max_iter: 68 | k +=1 69 | s = - dffcn(x) 70 | for m in range(line_search_max_iter): 71 | #print 'm=',m 72 | alpha = beta**m 73 | x_alpha = P(x,s,alpha) 74 | if ffcn( x_alpha ) - ffcn(x) <= - delta * numpy.sum(s* (x_alpha - x)): 75 | break 76 | x_old = x.copy() 77 | x = x_alpha 78 | 79 | return x_old,s 80 | 81 | 82 | # Setup of the optimization 83 | 84 | # X AND Y PARTITION 85 | x_grid = numpy.linspace(0,1,M) 86 | y_grid = numpy.linspace(0,1,M) 87 | 88 | # BOX CONSTRAINTS 89 | lo = 2.5 90 | L = numpy.zeros((M,M),dtype=float) 91 | 92 | for n in range(M): 93 | for m in range(M): 94 | L[n,m] = 2.5 * ( (x_grid[n]-0.5)**2 + (y_grid[m]-0.5)**2 <= 1./16) 95 | 96 | U = 100*numpy.ones((M,M),dtype=float) 97 | 98 | Z,s = projected_gradients(u,O_tilde,dO_tilde,[L,U]) 99 | 100 | 101 | # # Plot with MAYAVI 102 | x = y = list(range(numpy.shape(Z)[0])) 103 | 104 | try: 105 | import enthought.mayavi.mlab as mlab 106 | except: 107 | import mayavi.mlab as mlab 108 | mlab.figure() 109 | mlab.view(azimuth=130) 110 | s = mlab.surf(x, y, Z, representation='wireframe', warp_scale='auto', line_width=1.) 111 | 112 | 113 | mlab.savefig('./mayavi_3D_plot.png') 114 | # mlab.show() 115 | 116 | 117 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/implicit_euler.py: -------------------------------------------------------------------------------- 1 | import numpy; from numpy import sin,cos; from algopy import UTPM, zeros 2 | D,P = 4,1 3 | x = UTPM(numpy.zeros((D,P,2))) 4 | x.data[0,:,0] = 1 5 | p = UTPM(numpy.zeros((D,P))) 6 | p.data[0,:] = 3; p.data[1,:] = 1 7 | 8 | def f(t, x, p): 9 | retval = x.copy() 10 | retval[0] = x[1] 11 | retval[1] = -p* x[0] 12 | return retval 13 | 14 | def implicit_euler(f_fcn, x0, ts, p): 15 | """ implicit euler with fixed stepsizes, using Newton's method to solve 16 | the occuring implicit system of nonlinear equations 17 | """ 18 | 19 | def F_fcn(x_new, x, t_new, t, p): 20 | """ implicit function to solve: 0 = F(x_new, x, t_new, t_old)""" 21 | return (t_new - t) * f_fcn(t_new, x_new, p) - x_new + x 22 | 23 | def J_fcn(x_new, x, t_new, t, p): 24 | """ computes the Jacobian of F_fcn 25 | all inputs are double arrays 26 | """ 27 | y = UTPM(numpy.zeros((D,N,N))) 28 | y.data[0,:] = x_new 29 | y.data[1,:,:] = numpy.eye(N) 30 | F = F_fcn(y, x, t_new, t, p) 31 | return F.data[1,:,:].T 32 | 33 | 34 | x = x0.copy() 35 | D,P,N = x.data.shape 36 | x_new = x.copy() 37 | 38 | x_list = [x.data.copy() ] 39 | for nts in range(ts.size-1): 40 | h = ts[nts+1] - ts[nts] 41 | x_new.data[0,...] = x.data[0,...] 42 | x_new.data[1:,...] = 0 43 | 44 | # compute the Jacobian at x 45 | J = J_fcn(x_new.data[0,0], x.data[0,0], ts[nts+1], ts[nts], p.data[0,0]) 46 | 47 | # d=0: apply Newton's method to solve 0 = F_fcn(x_new, x, t_new, t) 48 | step = numpy.inf 49 | while step > 10**-10: 50 | delta_x = numpy.linalg.solve(J, F_fcn(x_new.data[0,0], x.data[0,0], ts[nts+1], ts[nts], p.data[0,0])) 51 | x_new.data[0,0] -= delta_x 52 | step = numpy.linalg.norm(delta_x) 53 | 54 | # d>0: compute higher order coefficients 55 | J = J_fcn(x_new.data[0,0], x.data[0,0], ts[nts+1], ts[nts], p.data[0,0]) 56 | for d in range(1,D): 57 | F = F_fcn(x_new, x, ts[nts+1], ts[nts], p) 58 | x_new.data[d,0] = -numpy.linalg.solve(J, F.data[d,0]) 59 | 60 | x.data[...] = x_new.data[...] 61 | x_list.append(x.data.copy()) 62 | 63 | return numpy.array(x_list) 64 | 65 | 66 | # compute AD solution 67 | ts = numpy.linspace(0,2*numpy.pi,1000) 68 | xs = implicit_euler(f, x, ts, p) 69 | 70 | # print xs 71 | # analytical solution 72 | def x_analytical(t,p): 73 | return numpy.cos(numpy.sqrt(p)*t) 74 | 75 | def x_p_analytical(t,p): 76 | return -0.5*numpy.sin(numpy.sqrt(p)*t)*p**(-0.5)*t 77 | 78 | print(xs.shape) 79 | import matplotlib.pyplot as pyplot; import os 80 | pyplot.plot(ts, xs[:,0,0,0], ',k-', label = r'$x(t)$') 81 | pyplot.plot(ts, x_analytical(ts,p.data[0,0]), 'k-.', label = r'analytic $x(t)$') 82 | pyplot.plot(ts, xs[:,1,0,0], ',r-', label = r'$x_p(t)$') 83 | pyplot.plot(ts, x_p_analytical(ts,p.data[0,0]), 'r-.', label = r'analytic $x_p(t)$') 84 | pyplot.plot(ts, xs[:,2,0,0], ',b-', label = r'$x_{pp}(t)$') 85 | pyplot.plot(ts, xs[:,3,0,0], ',m-', label = r'$x_{ppp}(t)$') 86 | 87 | pyplot.title('analytical and implicit Euler solution') 88 | pyplot.xlabel('time $t$') 89 | pyplot.legend(loc='best') 90 | pyplot.grid() 91 | pyplot.savefig(os.path.join(os.path.dirname(os.path.realpath(__file__)),'implicit_euler.png')) 92 | # pyplot.show() 93 | 94 | -------------------------------------------------------------------------------- /experimental/performance_tests/adolc_implementation_for_speed_comparison/main.cpp: -------------------------------------------------------------------------------- 1 | #include "adolc/adolc.h" 2 | #include 3 | #include 4 | 5 | 6 | struct timeval tv; 7 | int mtime(void){ 8 | gettimeofday(&tv,NULL); 9 | return (int)(tv.tv_sec*1000 + (tv.tv_usec / 1000)); 10 | } 11 | 12 | 13 | template 14 | Tdouble f(Tdouble *x,int N){ 15 | Tdouble tmp = 1.; 16 | for(int n = 0; n != N; ++n){ 17 | tmp *=x[n]; 18 | } 19 | return tmp; 20 | } 21 | 22 | int main( int argc, char *argv ){ 23 | vector Ns(10); 24 | 25 | 26 | const int N = 100; 27 | const int P = N*(N+1)/2; 28 | const int D = 3; 29 | const int M = 1; 30 | double x[N]; 31 | double y; 32 | for(int n = 0; n!=N; ++n){ 33 | x[n] = n+2; 34 | } 35 | printf("x=\n"); 36 | for(int n = 0; n!=N; ++n){ 37 | printf("%f ",x[n]); 38 | } 39 | printf("\n"); 40 | // printf("speelpenning(x,N)= %f\n",speelpenning(x,N)); 41 | 42 | int start_time; 43 | int end_time; 44 | start_time = mtime(); 45 | adouble ax[N]; 46 | adouble ay; 47 | 48 | trace_on(0); 49 | ay=1.; 50 | for(int n = 0; n < N; ++n ){ 51 | ax[n]<<=x[n]; 52 | ay *= ax[n]; 53 | } 54 | ay>>=y; 55 | trace_off(); 56 | 57 | 58 | // /* hos_forward */ 59 | // double **X; 60 | // X = myalloc2(N,D); 61 | // X[0][0]=1.; 62 | // X[1][0]=1.; 63 | // for(int n=0; n!=N; ++n){ 64 | // for(int d=0; d!=D; ++d){ 65 | // printf("%f",X[n][d]); 66 | // } 67 | // printf("\n"); 68 | // } 69 | // double **Y; 70 | // Y = myalloc(1,D); 71 | // hos_forward(0,M,N,D,0,x,X,&y,Y); 72 | // for(int m=0; m!=M; ++m){ 73 | // for(int d=0; d!=D; ++d){ 74 | // printf("%f ",Y[m][d]); 75 | // } 76 | // printf("\n"); 77 | // } 78 | 79 | 80 | 81 | /* hov_forward */ 82 | double ***X; 83 | X = myalloc3(N,P,D); 84 | 85 | 86 | // for(int n = 0; n!=N; ++n){ 87 | // for(int p = 0; p!=P; ++p){ 88 | // for(int d = 0; d != D; ++d){ 89 | // X[n][p][d] = 0.; 90 | // if(d==0){ 91 | // X[n][p][d] = (n==p); 92 | // } 93 | // } 94 | // } 95 | // } 96 | // 97 | // printf("X=\n"); 98 | // for(int n = 0; n!=N; ++n){ 99 | // for(int p = 0; p!=P; ++p){ 100 | // for(int d = 0; d != D; ++d){ 101 | // printf("%f ",X[n][p][d]); 102 | // } 103 | // printf("\n"); 104 | // } 105 | // printf("\n"); 106 | // } 107 | 108 | double ***Y; 109 | Y = myalloc3(1,P,D); 110 | 111 | // zos_forward(0,1,N,0,x,&y); 112 | // printf("y= %f\n",y); 113 | hov_forward(0,1,N,D,P,x,X,&y,Y); 114 | 115 | // printf("Y=\n"); 116 | // for(int m = 0; m != M; ++m){ 117 | // for(int p = 0; p!=N; ++p){ 118 | // for(int d = 0; d != D; ++d){ 119 | // printf("%f ",Y[0][p][d]); 120 | // } 121 | // printf("\n"); 122 | // } 123 | // printf("\n"); 124 | // } 125 | end_time = mtime(); 126 | double run_time = (end_time-start_time)/1000.; 127 | printf("required time = %0.6f\n",run_time); 128 | 129 | /* reverse mode speed */ 130 | // double** H; 131 | // H = myalloc2(N,N); 132 | // 133 | // start_time = mtime(); 134 | // hessian(0,N,x,H); 135 | // end_time = mtime(); 136 | // double run_time = (end_time-start_time)/1000.; 137 | // 138 | // printf("required time = %0.6f\n",run_time); 139 | 140 | // for(int n1=0; n1!=N; ++n1){ 141 | // for(int n2=0; n2!=N; ++n2){ 142 | // printf("%f ", H[n1][n2]); 143 | // } 144 | // cout<`_. 11 | 12 | The goal is to compute derivatives of the form :math:`\frac{d x(t)}{d p}`, 13 | where :math:`x(t) \equiv x(t; x_0, p) \in \mathbb R^{N_x}` is solution of the 14 | ordinary differential equation 15 | 16 | .. math:: 17 | \dot x(t) = f(t, x, p) \\ 18 | x(0) = x_0(p) \;, 19 | 20 | where the initial values :math:`x(0)` is a function :math:`x_0(p)` depending on 21 | some parameter :math:`p \in \mathbb R^{N_p}`. 22 | 23 | 24 | Consider the following code that computes :math:`\frac{d x(t)}{d p}` of the 25 | harmonic oscillator described by the ODE 26 | 27 | .. math:: 28 | \dot x(t) = \begin{pmatrix} x_2 \\ -p x_1 \end{pmatrix} \;. 29 | 30 | 31 | Explict Euler 32 | ~~~~~~~~~~~~~~ 33 | 34 | To illustrate the idea, we use the 35 | explict Euler integration scheme. 36 | 37 | .. literalinclude:: explicit_euler.py 38 | 39 | .. image:: explicit_euler.png 40 | :align: center 41 | :scale: 100 42 | 43 | Implicit Euler 44 | ~~~~~~~~~~~~~~ 45 | 46 | Since in practice often implicit integration schemes (stiff ODEs) are necessary, 47 | we illustrate the approach at the example of the implicit Euler integration scheme: 48 | 49 | 50 | The ODE is discretized in time as 51 | 52 | .. math:: 53 | x_{k+1} - x_k = (t_{k+1} - t_k) f(t_{k+1}, x_{k+1}, p) \\ 54 | 0 = F(x_{k+1}, x_k, t_{k+1}, t_k, p) = (t_{k+1} - t_k) f(t_{k+1}, x_{k+1}, p) - x_{k+1} + x_k 55 | 56 | The task at hand is to solve this implicit function in UTP arithmetic, i.e. given 57 | :math:`0 = F(x,y)`, where :math:`x` is input and :math:`y` is output, solve 58 | 59 | .. math:: 60 | 0 = F([x]_D, [y]_D) \mod T^D \;. 61 | 62 | There are several possibilities to achieve this goal. For instance one can first 63 | solve the nominal problem :math:`0 = F(x,y)` then successively compute the higher-order 64 | coefficients :math:`y_d, d=1,\dots,D-1`. To soution of the nominal problem can be 65 | done by applying Newton's method, i.e. given an initial guess for :math:`y` compute 66 | an update :math:`\delta y`, i.e. iterate 67 | 68 | .. math:: 69 | \delta y = - (F_x(x,y))^{-1} F(x,y) \\ 70 | y = y + \delta y 71 | 72 | Once :math:`y` is known one can find the higher-order coefficients by using Newton-Hensel lifting 73 | 74 | .. math:: 75 | y_d T^d = -(F_x(x,y))^{-1} F([y]_{d}, [x]_{d+1}) \mod T^{d+1} \;. 76 | 77 | The complete procedure is shown in the following code: 78 | 79 | .. literalinclude:: implicit_euler.py 80 | 81 | 82 | The generated plot shows the numerically computed trajectory 83 | and the analytically derived solutions. One can see that the numerical trajectory 84 | of :math:`\frac{d x(t)}{d p}` is close to the analytical solution. More elaborate 85 | ODE integrators would yield better results. 86 | 87 | 88 | 89 | 90 | .. image:: implicit_euler.png 91 | :align: center 92 | :scale: 100 93 | 94 | .. [Eberhard99] Automatic Differentiation of Numerical Integration Algorithms, http://www.jstor.org/pss/2585052 95 | 96 | 97 | -------------------------------------------------------------------------------- /documentation/sphinx/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | set SPHINXBUILD=sphinx-build 6 | set BUILDDIR=_build 7 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 8 | if NOT "%PAPER%" == "" ( 9 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 10 | ) 11 | 12 | if "%1" == "" goto help 13 | 14 | if "%1" == "help" ( 15 | :help 16 | echo.Please use `make ^` where ^ is one of 17 | echo. html to make standalone HTML files 18 | echo. dirhtml to make HTML files named index.html in directories 19 | echo. pickle to make pickle files 20 | echo. json to make JSON files 21 | echo. htmlhelp to make HTML files and a HTML help project 22 | echo. qthelp to make HTML files and a qthelp project 23 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 24 | echo. changes to make an overview over all changed/added/deprecated items 25 | echo. linkcheck to check all external links for integrity 26 | echo. doctest to run all doctests embedded in the documentation if enabled 27 | goto end 28 | ) 29 | 30 | if "%1" == "clean" ( 31 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 32 | del /q /s %BUILDDIR%\* 33 | goto end 34 | ) 35 | 36 | if "%1" == "html" ( 37 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 38 | echo. 39 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 40 | goto end 41 | ) 42 | 43 | if "%1" == "dirhtml" ( 44 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 45 | echo. 46 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 47 | goto end 48 | ) 49 | 50 | if "%1" == "pickle" ( 51 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 52 | echo. 53 | echo.Build finished; now you can process the pickle files. 54 | goto end 55 | ) 56 | 57 | if "%1" == "json" ( 58 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 59 | echo. 60 | echo.Build finished; now you can process the JSON files. 61 | goto end 62 | ) 63 | 64 | if "%1" == "htmlhelp" ( 65 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 66 | echo. 67 | echo.Build finished; now you can run HTML Help Workshop with the ^ 68 | .hhp project file in %BUILDDIR%/htmlhelp. 69 | goto end 70 | ) 71 | 72 | if "%1" == "qthelp" ( 73 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 74 | echo. 75 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 76 | .qhcp project file in %BUILDDIR%/qthelp, like this: 77 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\algopy.qhcp 78 | echo.To view the help file: 79 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\algopy.ghc 80 | goto end 81 | ) 82 | 83 | if "%1" == "latex" ( 84 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 85 | echo. 86 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 87 | goto end 88 | ) 89 | 90 | if "%1" == "changes" ( 91 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 92 | echo. 93 | echo.The overview file is in %BUILDDIR%/changes. 94 | goto end 95 | ) 96 | 97 | if "%1" == "linkcheck" ( 98 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 99 | echo. 100 | echo.Link check complete; look for any errors in the above output ^ 101 | or in %BUILDDIR%/linkcheck/output.txt. 102 | goto end 103 | ) 104 | 105 | if "%1" == "doctest" ( 106 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 107 | echo. 108 | echo.Testing of doctests in the sources finished, look at the ^ 109 | results in %BUILDDIR%/doctest/output.txt. 110 | goto end 111 | ) 112 | 113 | :end 114 | -------------------------------------------------------------------------------- /documentation/sphinx/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " pickle to make pickle files" 22 | @echo " json to make JSON files" 23 | @echo " htmlhelp to make HTML files and a HTML help project" 24 | @echo " qthelp to make HTML files and a qthelp project" 25 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 26 | @echo " changes to make an overview of all changed/added/deprecated items" 27 | @echo " linkcheck to check all external links for integrity" 28 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 29 | 30 | clean: 31 | -rm -rf $(BUILDDIR)/* 32 | 33 | runscripts: 34 | python examples_tracer.py 35 | python examples/series_expansion.py 36 | python examples/explicit_euler.py 37 | python examples/implicit_euler.py 38 | python examples/posterior_log_probability.py 39 | 40 | html: 41 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 42 | @echo 43 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 44 | 45 | dirhtml: 46 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 47 | @echo 48 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 49 | 50 | pickle: 51 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 52 | @echo 53 | @echo "Build finished; now you can process the pickle files." 54 | 55 | json: 56 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 57 | @echo 58 | @echo "Build finished; now you can process the JSON files." 59 | 60 | htmlhelp: 61 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 62 | @echo 63 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 64 | ".hhp project file in $(BUILDDIR)/htmlhelp." 65 | 66 | qthelp: 67 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 68 | @echo 69 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 70 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 71 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/algopy.qhcp" 72 | @echo "To view the help file:" 73 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/algopy.qhc" 74 | 75 | latex: 76 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 77 | @echo 78 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 79 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 80 | "run these through (pdf)latex." 81 | 82 | changes: 83 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 84 | @echo 85 | @echo "The overview file is in $(BUILDDIR)/changes." 86 | 87 | linkcheck: 88 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 89 | @echo 90 | @echo "Link check complete; look for any errors in the above output " \ 91 | "or in $(BUILDDIR)/linkcheck/output.txt." 92 | 93 | doctest: 94 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 95 | @echo "Testing of doctests in the sources finished, look at the " \ 96 | "results in $(BUILDDIR)/doctest/output.txt." 97 | -------------------------------------------------------------------------------- /documentation/sphinx/examples/expm_identities.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test higher order derivatives of expm using some identities. 3 | 4 | These tests only deal with special cases of expm. 5 | They require the matrix to be symmetric. 6 | They only test higher order derivatives with respect to a single 7 | scaling parameter, as opposed to parameters that affect the matrix 8 | in more complicated ways. 9 | """ 10 | 11 | from numpy.testing import * 12 | import numpy 13 | 14 | from algopy import * 15 | from algopy.linalg import * 16 | 17 | # All of the imports are copied from test_examples.py 18 | # and they put algopy functions like sum and exp in the top level namespace. 19 | 20 | 21 | def expm_tS_v1(t, S): 22 | """ 23 | Compute expm(t*S) using AlgoPy default expm. 24 | t: truncated univariate Taylor polynomial 25 | S: symmetric numpy matrix 26 | """ 27 | 28 | # Compute the expm using the default Taylor-aware implementation. 29 | # As of the time of writing this comment, this default implementation 30 | # uses a fixed-order Pade approximation. 31 | return expm(t * S) 32 | 33 | 34 | def expm_tS_v2(t, S): 35 | """ 36 | Compute expm(t*S) using AlgoPy eigendecomposition and an identity. 37 | t: truncated univariate Taylor polynomial 38 | S: symmetric numpy matrix 39 | """ 40 | 41 | # Compute the eigendecomposition using a Taylor-aware eigh. 42 | L, Q = eigh(t * S) 43 | 44 | return dot(Q * exp(L), Q.T) 45 | 46 | 47 | def expm_tS_v3(t, S): 48 | """ 49 | Compute expm(t*S) using LAPACK eigendecomposition and an identity. 50 | t: truncated univariate Taylor polynomial 51 | S: symmetric numpy matrix 52 | """ 53 | 54 | # Compute the eigendecomposition using a Taylor-naive eigh. 55 | L, Q = numpy.linalg.eigh(S) 56 | 57 | return dot(Q * exp(t * L), Q.T) 58 | 59 | 60 | def create_random_symmetric_matrix(): 61 | S_asym = numpy.random.rand(4, 4) 62 | return S_asym + S_asym.T 63 | 64 | 65 | class Test_ExpmScaledSymmetric(TestCase): 66 | 67 | def test_d0(self): 68 | S = create_random_symmetric_matrix() 69 | t0 = 0.123 70 | t = t0 71 | raw_v1 = expm_tS_v1(t, S) 72 | raw_v2 = expm_tS_v2(t, S) 73 | raw_v3 = expm_tS_v3(t, S) 74 | assert_allclose(raw_v1, raw_v2) 75 | assert_allclose(raw_v1, raw_v3) 76 | 77 | def test_d1(self): 78 | S = create_random_symmetric_matrix() 79 | t0 = 0.123 80 | t_grad = UTPM.init_jacobian(t0) 81 | raw_v1 = expm_tS_v1(t_grad, S) 82 | raw_v2 = expm_tS_v2(t_grad, S) 83 | raw_v3 = expm_tS_v3(t_grad, S) 84 | assert_allclose(raw_v1.data, raw_v2.data) 85 | assert_allclose(raw_v1.data, raw_v3.data) 86 | grad_v1 = UTPM.extract_jacobian(sum(raw_v1)) 87 | grad_v2 = UTPM.extract_jacobian(sum(raw_v2)) 88 | grad_v3 = UTPM.extract_jacobian(sum(raw_v3)) 89 | assert_allclose(grad_v1, grad_v2) 90 | assert_allclose(grad_v1, grad_v3) 91 | 92 | def test_d2(self): 93 | S = create_random_symmetric_matrix() 94 | t0 = 0.123 95 | t_hess = UTPM.init_hessian(t0) 96 | raw_v1 = expm_tS_v1(t_hess, S) 97 | raw_v2 = expm_tS_v2(t_hess, S) 98 | raw_v3 = expm_tS_v3(t_hess, S) 99 | assert_allclose(raw_v1.data, raw_v2.data) 100 | assert_allclose(raw_v1.data, raw_v3.data) 101 | hess_v1 = UTPM.extract_hessian(1, sum(raw_v1)) 102 | hess_v2 = UTPM.extract_hessian(1, sum(raw_v2)) 103 | hess_v3 = UTPM.extract_hessian(1, sum(raw_v3)) 104 | assert_allclose(hess_v1, hess_v2) 105 | assert_allclose(hess_v1, hess_v3) 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /experimental/ctps/tests/test_ctps.py: -------------------------------------------------------------------------------- 1 | from numpy.testing import * 2 | import numpy 3 | 4 | from algopy.ctps import * 5 | 6 | class Test_CTPS_operations(TestCase): 7 | def test_add(self): 8 | x1 = numpy.random.rand() 9 | x2 = numpy.random.rand() 10 | 11 | ax1 = CTPS(numpy.array([x1,1,0,0],dtype=float)) 12 | ax2 = CTPS(numpy.array([x2,0,0,0],dtype=float)) 13 | 14 | ay = ax1 + ax2 15 | assert_array_almost_equal(ay.data, ax1.data + ax2.data) 16 | 17 | def test_sub(self): 18 | x1 = numpy.random.rand() 19 | x2 = numpy.random.rand() 20 | 21 | ax1 = CTPS(numpy.array([x1,1,0,0],dtype=float)) 22 | ax2 = CTPS(numpy.array([x2,0,0,0],dtype=float)) 23 | 24 | ay = ax1 - ax2 25 | assert_array_almost_equal(ay.data, ax1.data - ax2.data) 26 | 27 | def test_mul(self): 28 | x1 = numpy.random.rand() 29 | x2 = numpy.random.rand() 30 | 31 | ax1 = CTPS(numpy.array([x1,1,0,0],dtype=float)) 32 | ax2 = CTPS(numpy.array([x2,0,2,0],dtype=float)) 33 | 34 | ay = ax1 * ax2 35 | assert_array_almost_equal([x1*x2, x2, 2*x1, 2], ay.data) 36 | 37 | def test_div(self): 38 | x1 = numpy.random.rand() 39 | x2 = numpy.random.rand() 40 | 41 | ax1 = CTPS(numpy.array([x1,1,0,0],dtype=float)) 42 | ax2 = CTPS(numpy.array([x2,0,2,0],dtype=float)) 43 | 44 | ay = ax1 / ax2 45 | assert_array_almost_equal([x1/x2, 1./x2, -2*x1/x2**2, -2./x2**2], ay.data) 46 | 47 | 48 | def test_scalar_add(self): 49 | x1 = numpy.random.rand() 50 | x2 = numpy.random.rand() 51 | 52 | ax1 = CTPS(numpy.array([x1,1,0,0],dtype=float)) 53 | ay = ax1 + x2 54 | assert_array_almost_equal(ay.data, ax1.data + [x2,0,0,0]) 55 | 56 | def test_scalar_sub(self): 57 | x1 = numpy.random.rand() 58 | x2 = numpy.random.rand() 59 | 60 | ax1 = CTPS(numpy.array([x1,1,0,0],dtype=float)) 61 | 62 | ay = ax1 - x2 63 | assert_array_almost_equal(ay.data, ax1.data - [x2,0,0,0]) 64 | 65 | def test_scalar_mul(self): 66 | x1 = numpy.random.rand() 67 | x2 = numpy.random.rand() 68 | 69 | ax1 = CTPS(numpy.array([x1,1,0,0],dtype=float)) 70 | 71 | ay = ax1 * x2 72 | assert_array_almost_equal( ay.data, [x1*x2, x2, 0 , 0]) 73 | 74 | def test_div(self): 75 | x1 = numpy.random.rand() 76 | x2 = numpy.random.rand() 77 | 78 | ax1 = CTPS(numpy.array([x1,1,0,0],dtype=float)) 79 | 80 | ay = ax1 / x2 81 | assert_array_almost_equal([x1/x2, 1./x2, 0, 0], ay.data) 82 | 83 | 84 | def test_simple_hessian(self): 85 | """ 86 | test function: 87 | f: R^4 -> R 88 | x -> y = f(x) = prod(x) 89 | 90 | u = x[:2] 91 | v = x[2:] 92 | 93 | goal: Compute d/du d/dv f(u,v) 94 | 95 | 96 | """ 97 | x1 = numpy.random.rand() 98 | x2 = numpy.random.rand() 99 | x3 = numpy.random.rand() 100 | x4 = numpy.random.rand() 101 | 102 | # compute d/dx1 d/x3 f 103 | ax1 = CTPS(numpy.array([x1,1,0,0],dtype=float)) 104 | ax2 = CTPS(numpy.array([x2,0,0,0],dtype=float)) 105 | ax3 = CTPS(numpy.array([x3,0,1,0],dtype=float)) 106 | ax4 = CTPS(numpy.array([x4,0,0,0],dtype=float)) 107 | 108 | ay = ax1 * ax2 * ax3 * ax4 109 | 110 | assert_almost_equal(ay.data[2**2 - 1], x2*x4, decimal = 3) 111 | 112 | 113 | 114 | if __name__ == "__main__": 115 | run_module_suite() 116 | -------------------------------------------------------------------------------- /experimental/tests/trash/unit_test_with_sympy_x_as_matrix_input.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ####################################################### 3 | # This is a Unit Test t that makes use of the Python # 4 | # Module Sympy. # 5 | ####################################################### 6 | 7 | import sympy as sym 8 | from numpy import array, zeros, ones, shape 9 | from numpy.random import random 10 | from numpy.linalg import norm 11 | 12 | N = 2 13 | D = 2 14 | M = N + 3 15 | 16 | xs = array([[sym.Symbol('x%d%d'%(n,d)) for d in range(D)] for n in range(N)]) 17 | # computing the function f: R^(NxD) -> R symbolically 18 | fs = 0 19 | for n in range(1,N): 20 | for m in range(n): 21 | tmp = 0 22 | for d in range(D): 23 | tmp += (xs[n,d] - xs[m,d])**2 24 | tmp = sym.sqrt(tmp) 25 | fs += 1/tmp 26 | 27 | # computing the gradient symbolically 28 | dfs = array([[sym.diff(fs, xs[n,d]) for d in range(D)] for n in range(N)]) 29 | 30 | # computing the Hessian symbolically 31 | ddfs = array([[[[ sym.diff(dfs[m,e], xs[n,d]) for d in range(D)] for n in range(N)] for e in range(D) ] for m in range(N)]) 32 | 33 | 34 | # function f 35 | def f(x): 36 | retval = 0. 37 | for n in range(1,N): 38 | for m in range(n): 39 | retval += 1./ norm(x[n,:] - x[m,:],2) 40 | return retval 41 | 42 | def df(x): 43 | g = zeros(shape(x),dtype=float) 44 | for n in range(N): 45 | for d in range(D): 46 | for m in range(N): 47 | if n != m: 48 | g[n,d] -= (x[n,d] - x[m,d])/norm(x[n,:]-x[m,:])**3 49 | return g 50 | 51 | def ddf(x): 52 | N,D = shape(x) 53 | H = zeros((N,D,N,D),dtype=float) 54 | for n in range(N): 55 | for d in range(D): 56 | for m in range(N): 57 | for e in range(D): 58 | for l in range(N): 59 | if l==n: 60 | continue 61 | H[n,d,m,e] -= (( (m==n) * (d==e) - (m==l)*(d==e) ) - 3* (x[n,d] - x[l,d])/norm(x[n,:]-x[l,:])**2 * ( (n==m) - (m==l))*( x[n,e] - x[l,e]))/norm(x[n,:] - x[l,:])**3 62 | return H 63 | 64 | 65 | def sym_f(x): 66 | symdict = dict() 67 | for n in range(N): 68 | for d in range(D): 69 | symdict[xs[n,d]] = x[n,d] 70 | return fs.subs(symdict).evalf() 71 | 72 | def sym_df(x): 73 | symdict = dict() 74 | for n in range(N): 75 | for d in range(D): 76 | symdict[xs[n,d]] = x[n,d] 77 | return array([[dfs[n,d].subs(symdict).evalf() for d in range(D)] for n in range(N)]) 78 | 79 | def sym_ddf(x): 80 | symdict = dict() 81 | for n in range(N): 82 | for d in range(D): 83 | symdict[xs[n,d]] = x[n,d] 84 | return array([[[[ ddfs[m,e,n,d].subs(symdict).evalf() for d in range(D)] for n in range(N)] for e in range(D)] for m in range(N)],dtype=float) 85 | 86 | # def ad_df(x): 87 | # return gradient(f,x) 88 | 89 | # def ad_ddf(x): 90 | # return hessian(f,x) 91 | 92 | 93 | # point at which the derivatives should be evaluated 94 | x = random((N,D)) 95 | 96 | 97 | print('\n\n') 98 | print('Sympy function = Ad function check (should be almost zero)') 99 | print(f(x) - sym_f(x)) 100 | 101 | print('\n\n') 102 | print('Sympy vs Hand Derived Gradient check (should be almost zero)') 103 | print(df(x) - sym_df(x)) 104 | print('Sympy vs Ad Derive Gradient check (should be almost zero)') 105 | print(ad_df(x) - sym_df(x)) 106 | 107 | print('\n\n') 108 | print('Sympy vs Hand Derived Hessian check (should be almost zero)') 109 | print(ddf(x) - sym_ddf(x)) 110 | print('Sympy vs Ad Derive Hessian check (should be almost zero)') 111 | print(ad_ddf(x) - sym_ddf(x)) 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /documentation/sphinx/getting_started.rst: -------------------------------------------------------------------------------- 1 | Getting Started 2 | =============== 3 | 4 | Consider the situation where the entries of a matrix :math:`A \equiv A(x)\in \mathbb R^{M \times N}` is computed 5 | by a computer program, where :math:`x \in \mathbb R^{N_x}`. To give an explicit example we consider 6 | 7 | .. math:: 8 | A(x) = \begin{pmatrix} 9 | \sin(x_1)^2 + x_2 & x_1 \\ 10 | e^{x_1/x_2} & x_3 \\ 11 | \log(x_1 + x_3*x_2) & 0 \\ 12 | \end{pmatrix} 13 | 14 | In a second step it is desired to compute 15 | 16 | .. math:: 17 | \Phi(x) = \max( \lambda( (A(x)^T A(x))^{-1}) \;, 18 | 19 | where :math:`\lambda(C)` computes all eigenvalues of the matrix :math:`C` and 20 | :math:`\max` returns the largest of the eigenvalues. The matrix inversion is not 21 | really necessary since one could as well invert the smallest eigenvalue. It is 22 | used here simply to make the point that its easy to concatenate matrix functions 23 | together. 24 | 25 | We are interested in the numerical value of the gradient 26 | 27 | .. math:: 28 | \nabla_x \Phi(x) 29 | 30 | at :math:`x=(3,5,7)^T`. At first we look at the forward mode of AD. E.g. we want to compute 31 | :math:`\frac{\partial \Phi}{\partial x_1}`. 32 | 33 | The corresponding code is:: 34 | 35 | >>> import numpy 36 | >>> from algopy import UTPM, eigh, inv, dot 37 | >>> 38 | >>> x = UTPM(numpy.zeros((2,1,3))) 39 | >>> x.data[0,0] = [3,5,7] 40 | >>> x.data[1,0] = [1,0,0] 41 | >>> 42 | >>> A = UTPM(numpy.zeros((2,1,3,2))) 43 | >>> A[0,0] = numpy.sin(x[0])**2 + x[1] 44 | >>> A[0,1] = x[0] 45 | >>> A[1,0] = numpy.exp(x[0]/x[1]) 46 | >>> A[1,1] = x[2] 47 | >>> A[2,0] = numpy.log(x[0] + x[2]*x[1]) 48 | >>> 49 | >>> print 'A =', A 50 | A = [[[[ 5.01991486 3. ] 51 | [ 1.8221188 7. ] 52 | [ 3.63758616 0. ]]] 53 | 54 | 55 | [[[-0.2794155 1. ] 56 | [ 0.36442376 0. ] 57 | [ 0.02631579 0. ]]]] 58 | >>> 59 | >>> y = eigh(inv(dot(A.T, A)))[0][-1] 60 | >>> 61 | >>> print 'Phi(x) = ', y.data[0] 62 | Phi(x) = [ 0.04784897] 63 | >>> print 'd/dx_1 Phi(x) = ', y.data[1] 64 | d/dx_1 Phi(x) = [ 0.01173805] 65 | 66 | 67 | The output of `print 'A =', A` is the contents of `A.data` with shape 68 | `A.data.shape = (2,1,3,2)`. The first block corresponds to `A.data[0]` 69 | and is simply the normal function evaluation. In the block `A.data[1]` 70 | one has the partial derivatives :math:`\frac{\partial A}{\partial x_1}`. 71 | Similarly for `y.data[0]` which is the normal function evaluation and 72 | `y.data[1]` is the partial derivative :math:`\frac{\partial \Phi}{\partial x_1}`. 73 | To compute the complete gradient one could repeat the above procedure but setting:: 74 | 75 | x.data[1] = [0,1,0] 76 | 77 | resp:: 78 | 79 | x.data[1] = [0,0,1] 80 | 81 | To reduce overhead, ALGOPY offers the possibility to propagate `P` directions 82 | at once. It also allows to compute higher-order derivatives. I.e. compute not only 83 | the zeroth and first Taylor coefficients but the first `D` coefficients. 84 | Then the program would look like:: 85 | 86 | >>> import numpy 87 | >>> from algopy import UTPM, eigh, inv, dot 88 | >>> 89 | >>> D,P,Nx,M,N = 2,3,3,3,2 90 | >>> 91 | >>> x = UTPM(numpy.zeros((D,P,Nx))) 92 | >>> x.data[0,:] = [3,5,7] 93 | >>> x.data[1,:] = numpy.eye(Nx) 94 | >>> 95 | >>> A = UTPM(numpy.zeros((D,P,M,N))) 96 | >>> A[0,0] = numpy.sin(x[0])**2 + x[1] 97 | >>> A[0,1] = x[0] 98 | >>> A[1,0] = numpy.exp(x[0]/x[1]) 99 | >>> A[1,1] = x[2] 100 | >>> A[2,0] = numpy.log(x[0] + x[2]*x[1]) 101 | >>> 102 | >>> y = eigh(inv(dot(A.T, A)))[0][-1] 103 | >>> 104 | >>> print 'Phi(x) = ', y.data[0] 105 | Phi(x) = [ 0.04784897 0.04784897 0.04784897] 106 | >>> print 'd/dx_1 Phi(x) = ', y.data[1] 107 | d/dx_1 Phi(x) = [ 0.01173805 -0.01228258 -0.00893191] 108 | -------------------------------------------------------------------------------- /documentation/sphinx/runtime_comparison.py: -------------------------------------------------------------------------------- 1 | from numpy.testing import * 2 | import numpy 3 | from algopy import * 4 | import adolc 5 | import time 6 | reps = 100 7 | N,P,D = 100,1,2 8 | def f(x): 9 | return numpy.sum([x[i] for i in range(N)]) 10 | 11 | # trace with ALGOPY 12 | start_time = time.time() 13 | cg = CGraph() 14 | x = Function(UTPM(numpy.random.rand(1,1,N))) 15 | y = f(x) 16 | cg.trace_off() 17 | cg.independentFunctionList = [x] 18 | cg.dependentFunctionList = [y] 19 | end_time = time.time() 20 | time_trace_algopy = end_time - start_time 21 | 22 | # trace with PYADOLC 23 | start_time = time.time() 24 | adolc.trace_on(1) 25 | x = adolc.adouble(numpy.random.rand(N)) 26 | adolc.independent(x) 27 | y = f(x) 28 | adolc.dependent(y) 29 | adolc.trace_off() 30 | end_time = time.time() 31 | time_trace_adolc = end_time - start_time 32 | 33 | # trace with PYADOLC.cgraph 34 | from adolc.cgraph import AdolcProgram 35 | start_time = time.time() 36 | ap = AdolcProgram() 37 | ap.trace_on(2) 38 | x = adolc.adouble(numpy.random.rand(N)) 39 | ap.independent(x) 40 | y = f(x) 41 | ap.dependent(y) 42 | ap.trace_off() 43 | end_time = time.time() 44 | time_trace_cgraph = end_time - start_time 45 | 46 | # time ALGOPY hos_forward 47 | x = UTP(numpy.random.rand(D,N)) 48 | start_time = time.time() 49 | for rep in range(reps): 50 | cg.pushforward([x]) 51 | end_time = time.time() 52 | time_hos_forward_algopy = end_time - start_time 53 | 54 | # time PYADOLC hos_forward 55 | x = numpy.random.rand(N) 56 | V = numpy.random.rand(N,D-1) 57 | 58 | start_time = time.time() 59 | for rep in range(reps): 60 | adolc.hos_forward(1,x , V, keep=0) 61 | end_time = time.time() 62 | time_hos_forward_adolc = end_time - start_time 63 | 64 | # time PYADOLC.cgraph hos_forward 65 | x = numpy.random.rand(N) 66 | V = numpy.random.rand(N,1,D-1) 67 | for rep in range(reps): 68 | ap.forward([x],[V]) 69 | end_time = time.time() 70 | time_hos_forward_cgraph = end_time - start_time 71 | 72 | # time ALGOPY hov_forward 73 | x = UTPM(numpy.random.rand(D,P,N)) 74 | start_time = time.time() 75 | for rep in range(reps): 76 | cg.pushforward([x]) 77 | end_time = time.time() 78 | time_hov_forward_algopy = end_time - start_time 79 | 80 | # time PYADOLC hov_forward 81 | x = numpy.random.rand(N) 82 | V = numpy.random.rand(N,P,D-1) 83 | 84 | start_time = time.time() 85 | for rep in range(reps): 86 | adolc.hov_forward(1,x, V) 87 | end_time = time.time() 88 | time_hov_forward_adolc = end_time - start_time 89 | 90 | # time PYADOLC.cgraph hos_forward 91 | x = numpy.random.rand(N) 92 | V = numpy.random.rand(N,P,D-1) 93 | for rep in range(reps): 94 | ap.forward([x],[V]) 95 | end_time = time.time() 96 | time_hov_forward_cgraph = end_time - start_time 97 | 98 | # time ALGOPY hov_reverse 99 | ybar = UTPM(numpy.random.rand(D,P)) 100 | start_time = time.time() 101 | for rep in range(reps): 102 | cg.pullback([ybar]) 103 | end_time = time.time() 104 | time_hov_reverse_algopy = end_time - start_time 105 | 106 | # time PYADOLC hov_reverse 107 | W = numpy.random.rand(1,1,D) 108 | start_time = time.time() 109 | V = numpy.random.rand(N,P,D-1) 110 | for rep in range(reps): 111 | for p in range(P): 112 | adolc.hos_forward(1,x, V[:,p,:], keep=D) 113 | adolc.hov_ti_reverse(1, W) 114 | end_time = time.time() 115 | time_hov_reverse_adolc = end_time - start_time 116 | 117 | # time PYADOLC.cgraph hov_reverse 118 | W = numpy.random.rand(1, 1,P,D) 119 | for rep in range(reps): 120 | ap.reverse([W]) 121 | end_time = time.time() 122 | time_hov_reverse_cgraph = end_time - start_time 123 | 124 | print('----------------') 125 | print(time_trace_algopy) 126 | print(time_trace_adolc) 127 | print(time_trace_cgraph) 128 | print('----------------') 129 | print(time_hos_forward_algopy) 130 | print(time_hos_forward_adolc) 131 | print(time_hos_forward_cgraph) 132 | print('----------------') 133 | print(time_hov_forward_algopy) 134 | print(time_hov_forward_adolc) 135 | print(time_hov_forward_cgraph) 136 | print('----------------') 137 | print(time_hov_reverse_algopy) 138 | print(time_hov_reverse_adolc) 139 | print(time_hov_reverse_cgraph) 140 | -------------------------------------------------------------------------------- /algopy/base_type.py: -------------------------------------------------------------------------------- 1 | """ 2 | This implements an abstrace base class Ring . 3 | 4 | Rationale: 5 | 6 | Goal is to separate the datatype specification from the algorithms and containers for the following reasons: 7 | 8 | 1) It allows to directly use the algorithms *without* overhead. E.g. calling mul(z.data, x.data, y.data) 9 | has much less overhead than z = x.__mul__(y). data is to be kept as close as possible to 10 | machine primitives. E.g. data is array or tuple of arrays. 11 | 2) Potential reuse of an algorithm in several datatypes. 12 | 3) Relatively easy to connect high performance algorithms with a very highlevel abstract description. 13 | For instance, most programming languages allow calling C-functions. Therefore, the algorithms 14 | should be given as void fcn(int A, double B, ...) 15 | 16 | For instance, the datatype is a truncated Taylor polynomial R[t]/ of the class Foo. 17 | The underlying container is a simple array of doubles. 18 | 19 | """ 20 | 21 | import numpy 22 | 23 | 24 | 25 | 26 | class Ring(object): 27 | """ 28 | 29 | An abstract base class in an attempt to follow the DRY principle. 30 | It implements the algebraic class of a ring as defined on 31 | http://en.wikipedia.org/wiki/Ring_%28mathematics%29 32 | 33 | The idea is that the set is described in data and the operations +,* etc. 34 | are implemented as functions that operate on the data. 35 | 36 | E.g. the factor ring of natural numbers modulo 4, x.data = 3 y.data = 2 37 | then z = add(x,y) is implemented as 38 | 39 | def add(x,y): 40 | return self.__class__((x.data*y.data)%4) 41 | 42 | and one obtains z.data = 1 43 | 44 | Warning: 45 | Since this class is only of little value it may be deprecated in the future. 46 | """ 47 | data = NotImplementedError() 48 | 49 | def totype(self, x): 50 | """ 51 | tries to convert x to an object of the class 52 | 53 | works for : scalar x, numpy.ndarray x 54 | 55 | Remark: 56 | at the moment, scalar x expanded as Ring with the same degree as self though. 57 | The reason is a missing implementation that works for graded rings of different degree. 58 | Once such implementations exist, this function should be adapted. 59 | 60 | """ 61 | if numpy.isscalar(x): 62 | xdata = self.__class__.__zeros_like__(self.data) 63 | self.__class__.__scalar_to_data__(xdata, x) 64 | return self.__class__(xdata) 65 | 66 | elif isinstance(x, numpy.ndarray): 67 | raise NotImplementedError('sorry, not implemented just yet') 68 | 69 | elif not isinstance(x, self.__class__): 70 | raise NotImplementedError('Cannot convert x\n type(x) = %s but expected type(x) = %s'%(str(type(x)))) 71 | 72 | else: 73 | return x 74 | 75 | 76 | def __add__(self, rhs): 77 | rhs = self.totype(rhs) 78 | retval = self.__class__(self.__class__.__zeros_like__(self.data)) 79 | self.__class__.add(retval.data, self.data, rhs.data) 80 | return retval 81 | 82 | def __sub__(self, rhs): 83 | rhs = self.totype(rhs) 84 | retval = self.__class__(self.__class__.__zeros_like__(self.data)) 85 | self.__class__.sub(retval.data, self.data, rhs.data) 86 | return retval 87 | 88 | def __mul__(self,rhs): 89 | rhs = self.totype(rhs) 90 | retval = self.__class__(self.__class__.__zeros_like__(self.data)) 91 | self.__class__.mul(retval.data, self.data, rhs.data) 92 | return retval 93 | 94 | def __truediv__(self,rhs): 95 | rhs = self.totype(rhs) 96 | retval = self.__class__(self.__class__.__zeros_like__(self.data)) 97 | self.__class__.div(retval.data, self.data, rhs.data) 98 | return retval 99 | 100 | def __radd__(self, lhs): 101 | return self + lhs 102 | 103 | def __rmul__(self, lhs): 104 | return self * lhs 105 | 106 | def zeros_like(self): 107 | return self.__class__(self.__class__.__zeros_like__(self.data)) 108 | 109 | def __str__(self): 110 | return str(self.data) 111 | 112 | -------------------------------------------------------------------------------- /experimental/performance_tests/UTPS_vs_UTPM.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | sys.path.append('../') 4 | from pylab import * 5 | from prettyplotting import * 6 | from adolc import * 7 | import adolc 8 | from algopy import * 9 | from numpy import * 10 | from time import * 11 | 12 | def myqr(in_A): 13 | # input checks 14 | Ndim = ndim(in_A) 15 | assert Ndim == 2 16 | N,M = shape(in_A) 17 | assert N==M 18 | 19 | # algorithm 20 | R = in_A.copy() 21 | if type(R[0,0]) == adouble: 22 | QT = array([[adouble(n==m) for m in range(N)] for n in range(N) ]) 23 | else: 24 | QT = eye(N) 25 | 26 | for n in range(N): 27 | for m in range(n+1,N): 28 | a = R[n,n] 29 | b = R[m,n] 30 | r = sqrt(a**2 + b**2) 31 | c = a/r 32 | s = b/r 33 | 34 | for k in range(N): 35 | Rnk = R[n,k] 36 | 37 | R[n,k] = c*Rnk + s*R[m,k] 38 | R[m,k] =-s*Rnk + c*R[m,k]; 39 | 40 | QTnk = QT[n,k] 41 | QT[n,k] = c*QTnk + s*QT[m,k] 42 | QT[m,k] =-s*QTnk + c*QT[m,k]; 43 | #print 'QT:\n',QT 44 | #print 'R:\n',R 45 | #print '-------------' 46 | 47 | return QT,R 48 | 49 | def inv(in_A): 50 | QT,R = myqr(in_A) 51 | N = shape(in_A)[0] 52 | 53 | for n in range(N-1,-1,-1): 54 | Rnn = R[n,n] 55 | R[n,:] /= Rnn 56 | QT[n,:] /= Rnn 57 | for m in range(n+1,N): 58 | Rnm = R[n,m] 59 | R[n,m] = 0 60 | QT[n,:] -= QT[m,:]*Rnm 61 | 62 | return QT,R 63 | 64 | def print_loc(aA): 65 | print('---') 66 | for n in range(N): 67 | print('[', end=' ') 68 | for m in range(N): 69 | print(aA[n,m].loc," ", end=' ') 70 | print(']') 71 | 72 | if __name__ == "__main__": 73 | from numpy.random import random 74 | 75 | Ns = list(range(1,22)) 76 | adolc_times = [] 77 | adolc_taping_times = [] 78 | adolc_num_operations = [] 79 | adolc_num_locations = [] 80 | 81 | algopy_times = [] 82 | 83 | 84 | for N in Ns: 85 | print('N=',N) 86 | A = random((N,N)) 87 | #A = array([ 88 | #[0.018 ,0.0085 ,0.017 ,0.017], 89 | #[0.02 ,0.0042 ,0.0072 ,0.016], 90 | #[0.006 ,0.012 ,0.01 ,0.014], 91 | #[0.0078 ,0.011 ,0.02 ,0.02]], dtype= float64) 92 | 93 | # with ADOL-C 94 | # ----------- 95 | N = shape(A)[0] 96 | 97 | t_start = time() 98 | aA = array([[adouble(A[n,m]) for m in range(N)] for n in range(N) ]) 99 | 100 | trace_on(0) 101 | for n in range(N): 102 | for m in range(N): 103 | independent(aA[n,m]) 104 | aC = inv(aA)[0] 105 | ay = trace(aC) 106 | dependent(ay) 107 | trace_off() 108 | 109 | adolc_num_locations.append(tapestats(0)['NUM_LOCATIONS']) 110 | adolc_num_operations.append(tapestats(0)['NUM_OPERATIONS']) 111 | 112 | t_end = time() 113 | adolc_taping_times.append(t_end-t_start) 114 | t_start = time() 115 | H1 = adolc.hessian(0,A.ravel()) 116 | t_end = time() 117 | print('adolc needs %0.6f seconds'%(t_end - t_start)) 118 | adolc_times.append(t_end-t_start) 119 | 120 | # with ALGOPY 121 | 122 | t_start = time() 123 | cg = CGraph() 124 | 125 | FA = zeros((2,N**2,N,N)) 126 | for n in range(N): 127 | for m in range(N): 128 | FA[0,n*N+m,:,:] = A[:,:] 129 | FA[1,n*N+m,n,m] = 1. 130 | 131 | 132 | FA = Mtc(FA) 133 | FA = Function(FA) 134 | FC = FA.inv() 135 | Fy = FC.trace() 136 | cg.independentFunctionList = [ FA ] 137 | cg.dependentFunctionList = [ Fy ] 138 | 139 | ybar = zeros((2,N**2,1,1)) 140 | ybar[0,:,0,0] = 1. 141 | 142 | cg.reverse([Mtc(ybar)]) 143 | 144 | # put everything in a Hessian 145 | H2 = zeros((N**2,N**2)) 146 | for n in range(N**2): 147 | H2[n,:] = FA.xbar.TC[1,n,:].ravel() 148 | 149 | t_end = time() 150 | print('algopy needs %0.6f seconds'%(t_end - t_start)) 151 | algopy_times.append(t_end-t_start) 152 | 153 | #print H1 - H2 154 | 155 | figure() 156 | semilogy(Ns,adolc_taping_times,'ko', label='pyadolc: taping') 157 | semilogy(Ns,adolc_times,'k.', label='pyadolc: hessian computation') 158 | semilogy(Ns,algopy_times,'kd', label='algopy: hessian computation') 159 | legend(loc=4) 160 | title('UTPS vs UTPM for Hessian Computation of $y =$tr$X^{-1}$') 161 | xlabel('matrix size $N$') 162 | ylabel('runtime $t$ in seconds') 163 | 164 | savefig('utps_vs_utpm.png') 165 | savefig('utps_vs_utpm.eps') 166 | 167 | figure() 168 | semilogy(Ns,adolc_num_operations,'k.', label='Number of Operations') 169 | semilogy(Ns,adolc_num_locations,'ko', label='Number of Locations') 170 | 171 | title('Memory needed by PyADOLC') 172 | xlabel('matrix size $N$') 173 | ylabel('size') 174 | 175 | savefig('pyadolc_locs_and_ops.png') 176 | savefig('pyadolc_locs_and_ops.eps') 177 | legend(loc=4) 178 | 179 | 180 | show() -------------------------------------------------------------------------------- /documentation/sphinx/examples/polarization.rst: -------------------------------------------------------------------------------- 1 | Polarization Identities for Mixed Partial Derivatives 2 | ----------------------------------------------------- 3 | 4 | This is an advanced tutorial. It explains how functions containing derivatives of 5 | other functions can be evaluated using univariate Taylor polynomial arithmetic 6 | by use of polarization identities. 7 | The described technique could be implemented as convenience functions just as 8 | UTPM.init_jacobian, UTPM.init_hessian, etc. 9 | 10 | Consider the function 11 | 12 | .. math:: 13 | F: \Bbb{R}^N \to & \Bbb{R}^M \\ 14 | x \mapsto& y = F(x) = \begin{pmatrix} x_1 x_2 \\ x_2 x_3 \\ x_1 - x_2 \end{pmatrix} 15 | 16 | where :math:`(N,N)=(3,3)` and we want to compute 17 | 18 | .. math:: 19 | \nabla_x \Phi(J(x)) =& \mathrm{tr} ( J(x)^T J(x)) \;, 20 | 21 | where :math:`J(x) = {\partial F \over \partial x}(x)`. 22 | In univariate Taylor polynomial arithmetic it is possible to compute the :math:`i`-th 23 | column of :math:`J(x)` using :math:`{\partial \over \partial T} F(x + e_i)|_{T=0}`. 24 | Also, it is possible to compute the :math:`j`-th element of the gradient 25 | :math:`\nabla_x \Phi(J(x))` using 26 | :math:`{\partial \over \partial T} \Phi( J(x + e_j)) |_{T=0}`. 27 | That means, in total it is necessary to compute 28 | 29 | .. math:: 30 | {\partial \Phi \over \partial x_j} (J(x)) =& \left. {\partial \over \partial T_2} \Phi \left( 31 | \begin{pmatrix} 32 | {\partial \over \partial T_1} F(x + e_1 T_1 + e_jT_2), 33 | \dots, 34 | {\partial \over \partial T_1} F(x + e_N T_1 + e_jT_2) \\ 35 | \end{pmatrix} 36 | \right) \right|_{T_1 = T_2 =0} \\ 37 | =& \left. \sum_{m=1}^M \sum_{n=1}^N {\partial \Phi \over \partial J_{mn}} \right|_{J = J(x)} 38 | {\partial^2 \over \partial T_2 \partial T_1} F_m(x + e_n T_1 + e_jT_2) 39 | 40 | In this form it seems to be necessary to propagate a multivariate Taylor polynomial 41 | :math:`x + e_1 T_1 + e_jT_2` through :math:`F`. However, one can use a 42 | polarization identity like 43 | 44 | .. math:: 45 | \left. {\partial^2 \over \partial T_1 \partial T_2} F(x + e_i T_1 + e_jT_2) 46 | \right|_{T_1 = T_2 = 0} 47 | = \left. {1 \over 4} {\partial^2 \over \partial T^2} \left( F(x + (e_i + e_j) T) - F(x + (e_i - e_j)T) \right) \right|_{T=0} 48 | 49 | to cast the problem back to a problem where twice the number of univariate Taylor 50 | polynomials have to be propagated. 51 | In total we need to cycle over :math:`e_i`, for :math:`i =1,2,\dots,N` 52 | and :math:`e_j`, for :math:`j = 1,2,\dots,N`, which can both be written as columns/rows 53 | of the identity matrix. 54 | Stacking :math:`e_i + e_j` and :math:`e_i - e_j` for all possible choices one obtains 55 | the univariate directions:: 56 | 57 | dirs = 58 | [[ 2. 0. 0.] 59 | [ 0. 0. 0.] 60 | [ 1. 1. 0.] 61 | [ 1. -1. 0.] 62 | [ 1. 0. 1.] 63 | [ 1. 0. -1.] 64 | [ 1. 1. 0.] 65 | [-1. 1. 0.] 66 | [ 0. 2. 0.] 67 | [ 0. 0. 0.] 68 | [ 0. 1. 1.] 69 | [ 0. 1. -1.] 70 | [ 1. 0. 1.] 71 | [-1. 0. 1.] 72 | [ 0. 1. 1.] 73 | [ 0. -1. 1.] 74 | [ 0. 0. 2.] 75 | [ 0. 0. 0.]] 76 | 77 | As one can see, some directions are zero and could be avoided. 78 | Without this obvious improvement, there are :math:`P = 2N^2=18` directions. 79 | Now to how the above example can be computed in AlgoPy. 80 | 81 | .. literalinclude:: polarization.py 82 | :lines: 0- 83 | 84 | As output we obtain:: 85 | 86 | $ python polarization.py 87 | dirs = 88 | [[ 2. 0. 0.] 89 | [ 0. 0. 0.] 90 | [ 1. 1. 0.] 91 | [ 1. -1. 0.] 92 | [ 1. 0. 1.] 93 | [ 1. 0. -1.] 94 | [ 1. 1. 0.] 95 | [-1. 1. 0.] 96 | [ 0. 2. 0.] 97 | [ 0. 0. 0.] 98 | [ 0. 1. 1.] 99 | [ 0. 1. -1.] 100 | [ 1. 0. 1.] 101 | [-1. 0. 1.] 102 | [ 0. 1. 1.] 103 | [ 0. -1. 1.] 104 | [ 0. 0. 2.] 105 | [ 0. 0. 0.]] 106 | Phi= [[ 20. 20. 20.] 107 | [ 2. 8. 6.]] 108 | gradient of Phi = [ 2. 8. 6.] 109 | 110 | which is the correct value. 111 | 112 | 113 | 114 | 115 | One should note that actually in this special case, when all elements 116 | :math:`{\partial \Phi \over \partial x_j}` have to be propagated 117 | it is advantageous to use another polarization identity which requires to 118 | propagate only as many directions as the Hessian has distinct elements. 119 | However, the polarization identity from above is more flexible since it allows to 120 | compute mixed partial derivatives more directly. 121 | 122 | 123 | 124 | --------------------------------------------------------------------------------