├── short-review-methods.pdf ├── 2018Handout_BIP_AlgUQ_Latz.pdf ├── README.md ├── IS_t2N.m ├── IS_N2t.m ├── KalmanFilter.m ├── KalmanFilter_SmallNoiseLimit.m ├── MHMCMC_t.m └── MHMCMC_multimodal.m /short-review-methods.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/latz-io/bayesian_inversion/HEAD/short-review-methods.pdf -------------------------------------------------------------------------------- /2018Handout_BIP_AlgUQ_Latz.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/latz-io/bayesian_inversion/HEAD/2018Handout_BIP_AlgUQ_Latz.pdf -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bayesian_inversion 2 | A collection of algorithms used in Bayesian Statistics and particularly in Bayesian Inverse Problems 3 | -------------------------------------------------------------------------------- /IS_t2N.m: -------------------------------------------------------------------------------- 1 | % Importance Sampling: Works. 2 | % Simulate the 2nd Moment of a normal distribution given a t-distributed 3 | % samples 4 | % Jonas Latz, M.Sc. 5 | % Lehrstuhl f?r Numerische Mathematik 6 | % Fakult?t f?r Mathematik 7 | % Technische Universit?t M?nchen 8 | % jonas.latz@tum.de 9 | % 2017 - 10 | 11 | %% Configuration 12 | 13 | k = 3.5; 14 | true = 1; 15 | 16 | %Number of Simulations 17 | N = 100000 18 | 19 | %% Sample 20 | 21 | X = trnd(k,[N,1]); 22 | Y= X.^2; 23 | w = normpdf(X,0,1)./tpdf(X,3); 24 | 25 | w = w/sum(w); 26 | Est = sum(Y.*w); 27 | %% Plot 28 | % Plot requires the function histwv.m 29 | % (Copyright (c) 2016, Brent All rights reserved.) 30 | % that can be downloaded: 31 | % https://de.mathworks.com/matlabcentral/fileexchange/58450-histwv-v--w--min--max--bins-/content/histwv.m 32 | 33 | figure(4) 34 | X_vals = min(X):0.5:max(X); 35 | [histw, intervals] = histwv(X',w',min(X),max(X),length(X_vals)); 36 | bar(X_vals,histw) 37 | hold on 38 | x = min(X):0.05:max(X); 39 | y = tpdf(x,k); 40 | plot(x,y,'Linewidth',1.2); 41 | hold off 42 | 43 | 44 | 45 | %% Return the relative error 46 | rel_error = abs(Est-true)/true 47 | -------------------------------------------------------------------------------- /IS_N2t.m: -------------------------------------------------------------------------------- 1 | % Importance Sampling: Breaks. 2 | % Simulate the 2nd Moment of a student's t-distribution given a normal 3 | % proposals 4 | % Jonas Latz, M.Sc. 5 | % Lehrstuhl f?r Numerische Mathematik 6 | % Fakult?t f?r Mathematik 7 | % Technische Universit?t M?nchen 8 | % jonas.latz@tum.de 9 | % 2017 - 10 | 11 | %% Configurations 12 | 13 | % Degrees of Freedom 14 | %k = 3.5; 15 | k = 200000; 16 | 17 | % Exact 2nd Moment 18 | true = k/(k-2); 19 | 20 | %Number of Simulations 21 | N = 100000; 22 | 23 | %% Sample 24 | X = normrnd(0,1,[N,1]); 25 | Y = X.^2; 26 | w = tpdf(X,k)./normpdf(X,0,1); 27 | Est = sum(Y.*w)/sum(w); 28 | 29 | %% Normalise the weights (just used to plot the the Dist) 30 | Norm = sum(w); 31 | w = w/Norm; 32 | 33 | %% Plot 34 | % Plot requires the function histwv.m 35 | % (Copyright (c) 2016, Brent All rights reserved.) 36 | % that can be downloaded: 37 | % https://de.mathworks.com/matlabcentral/fileexchange/58450-histwv-v--w--min--max--bins-/content/histwv.m 38 | figure(5) 39 | X_vals = min(X):0.5:max(X); 40 | [histw, intervals] = histwv(X',w',min(X),max(X),length(X_vals)); 41 | bar(X_vals,histw) 42 | hold on 43 | x = min(X):0.05:max(X); 44 | y = tpdf(x,k); 45 | plot(x,y); 46 | hold off 47 | 48 | 49 | 50 | %% Return the relative error 51 | rel_error = abs(Est-true)/true -------------------------------------------------------------------------------- /KalmanFilter.m: -------------------------------------------------------------------------------- 1 | % Kalman Filter in 1D for a time-dependent linear inverse problem. 2 | % Jonas Latz, M.Sc. 3 | % Lehrstuhl f?r Numerische Mathematik 4 | % Fakult?t f?r Mathematik 5 | % Technische Universit?t M?nchen 6 | % jonas.latz@tum.de 7 | % 2017 - 8 | 9 | % Time steps 10 | n = 1; 11 | t = n*10; 12 | 13 | % Linear forward operator 14 | G = fliplr((1:1:t)/(t+1)); 15 | G = diag(G); 16 | 17 | %Likelihood is centred Gaussian with Variance = gamma2. 18 | gamma2 = 2; 19 | 20 | % Generate Data 21 | true = 4; %true value = 2. 22 | x_true = true*ones(t,1); 23 | data = G*x_true + normrnd(0,sqrt(gamma2),t,1); 24 | 25 | % The intermediate distributions are Gaussian and can thus be 26 | % represented by their mean and variances. 27 | mu = zeros(t,1); 28 | sigma = zeros(t,1); 29 | 30 | % Prior: 31 | mu(1) = -2; 32 | sigma(1) = 25; 33 | 34 | 35 | % Plot the intermediate distributions. 36 | x = -20:0.1:20; 37 | y = zeros(t,size(x,2)); 38 | 39 | y(1,:) = normpdf(x, mu(1), sqrt(sigma(1))); 40 | figure(2) 41 | plot(x,y(1,:)); 42 | hold on; 43 | for i = 2:t 44 | mu(i) = (mu(i-1)/sigma(i-1) + (data(i)/G(i,i))/(gamma2/G(i,i)^2))/(1/sigma(i-1) + (G(i,i)^2)/gamma2); 45 | sigma(i) = 1/(1/sigma(i-1) + G(i,i)^2/gamma2); 46 | y(i,:) = normpdf(x, mu(i), sqrt(sigma(i))); 47 | if mod(i,n)== 0 48 | plot(x,y(i,:)); %Plot only 10 intermediate distributions. 49 | end 50 | end 51 | plot(true*ones(size(0:0.01:max(max(y)))),0:0.01:max(max(y))); 52 | legend('1','2','3','4','5','6','7','8','9','10','u=2'); 53 | 54 | hold off; 55 | 56 | 57 | %plot the sigmas 58 | figure(3) 59 | plot(sigma) 60 | 61 | 62 | -------------------------------------------------------------------------------- /KalmanFilter_SmallNoiseLimit.m: -------------------------------------------------------------------------------- 1 | % Kalman Filter in 1D for a time-dependent linear inverse problem. 2 | % Jonas Latz, M.Sc. 3 | % Lehrstuhl f?r Numerische Mathematik 4 | % Fakult?t f?r Mathematik 5 | % Technische Universit?t M?nchen 6 | % jonas.latz@tum.de 7 | % 2017 - 8 | 9 | % Time steps 10 | n = 1; 11 | t = n*10; 12 | 13 | % Linear forward operator 14 | G = fliplr((1:1:t)/(t+1)); 15 | G = diag(G); 16 | 17 | %Likelihood is centred Gaussian with Variance = gamma2. 18 | gamma2 = 1./(1:1:t); 19 | 20 | % Generate Data 21 | true = 4; %true value = 2. 22 | x_true = true*ones(t,1); 23 | data = G*x_true + mvnrnd(zeros(length(gamma2),1),diag(gamma2))'; 24 | 25 | % The intermediate distributions are Gaussian and can thus be 26 | % represented by their mean and variances. 27 | mu = zeros(t,1); 28 | sigma = zeros(t,1); 29 | 30 | % Prior: 31 | mu(1) = -2; 32 | sigma(1) = 5; 33 | 34 | 35 | % Plot the intermediate distributions. 36 | x = -20:0.1:20; 37 | y = zeros(t,size(x,2)); 38 | 39 | y(1,:) = normpdf(x, mu(1), sqrt(sigma(1))); 40 | figure(2) 41 | plot(x,y(1,:)); 42 | hold on; 43 | for i = 2:t 44 | mu(i) = (mu(i-1)/sigma(i-1) + (data(i)/G(i,i))/(gamma2(i)/G(i,i)^2))/(1/sigma(i-1) + (G(i,i)^2)/gamma2(i)); 45 | sigma(i) = 1/(1/sigma(i-1) + G(i,i)^2/gamma2(i)); 46 | y(i,:) = normpdf(x, mu(i), sqrt(sigma(i))); 47 | if mod(i,n)== 0 48 | plot(x,y(i,:)); %Plot only 10 intermediate distributions. 49 | end 50 | end 51 | plot(true*ones(size(0:0.01:0.5)),0:0.01:0.5); 52 | legend('1','2','3','4','5','6','7','8','9','10','u=2'); 53 | 54 | hold off; 55 | 56 | %plot the sigmas 57 | figure(3) 58 | plot(sigma) 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /MHMCMC_t.m: -------------------------------------------------------------------------------- 1 | % Random-Walk-Metropolis-Hastings MCMC in 1D to sample from a student's t 2 | % distribution 3 | % Jonas Latz, M.Sc. 4 | % Lehrstuhl f?r Numerische Mathematik 5 | % Fakult?t f?r Mathematik 6 | % Technische Universit?t M?nchen 7 | % jonas.latz@tum.de 8 | % 2017 - 9 | 10 | %% Configurations 11 | 12 | % Degrees of Freedom 13 | n = 3.5; 14 | 15 | % Initial state 16 | x_start = 0; 17 | 18 | % Number of simulations 19 | N = 100000; 20 | 21 | 22 | %% Some Initialisations 23 | 24 | X = zeros(1,N); 25 | a = zeros(1,N); 26 | accept = zeros(1,N); 27 | X(1) = x_start; 28 | a(1) = 1; 29 | accept(1)=1; 30 | 31 | %% MCMC-Run 32 | 33 | for k=2:N 34 | % Proposal 35 | X_prop = X(k-1) + normrnd(0,0.85); 36 | 37 | % Acceptance Probability 38 | a(k) =min(1,(tpdf(X_prop,n)/tpdf(X(k-1),n))); 39 | 40 | % Accept?! 41 | if rand < a(k) 42 | X(k) = X_prop; 43 | accept(k) = 1; 44 | else 45 | X(k) = X(k-1); 46 | end 47 | 48 | end 49 | 50 | %% Plot the sample path, histogram, and correct pdf. 51 | 52 | FigHandle = figure(21); 53 | set(FigHandle, 'Position', [50, 50, 800, 400]); 54 | subplot(1,2,1) 55 | plot(X) 56 | 57 | subplot(1,2,2) 58 | h=histogram(X(-10 < X <10),'Normalization','probability'); 59 | h.NumBins = 40; 60 | hold on 61 | y = -10:0.1:10; 62 | f = tpdf(y,n); 63 | plot(y,f,'LineWidth',1.5) 64 | hold off 65 | 66 | %% Derive the amount of accepted steps and estimate the 67 | % acceptance probability 68 | 69 | rel_accept = sum(accept)/N 70 | estim_accept_prob = sum(a)/N 71 | 72 | %% Estimate the Second Moment 73 | 74 | % Exact 2nd Moment 75 | Estimate = sum(X.^2)/N; 76 | true = n/(n-2); 77 | rel_error = abs(Estimate-true)/true -------------------------------------------------------------------------------- /MHMCMC_multimodal.m: -------------------------------------------------------------------------------- 1 | % Random-Walk-Metropolis-Hastings MCMC in 1D to sample from a Multimodal 2 | % Probability distribution 3 | % Jonas Latz, M.Sc. 4 | % Lehrstuhl f?r Numerische Mathematik 5 | % Fakult?t f?r Mathematik 6 | % Technische Universit?t M?nchen 7 | % jonas.latz@tum.de 8 | % 2017 - 9 | 10 | %% Configurations 11 | 12 | % Standarddeviation of the proposal 13 | 14 | %propsigma = 0.1; % Breaks 15 | propsigma = 5; % Works 16 | 17 | % Initial state 18 | x_start = 0; 19 | 20 | % Number of generated samples 21 | N = 100000; 22 | 23 | %% Some Initialisations 24 | 25 | X = zeros(1,N); 26 | a = zeros(1,N); 27 | accept = zeros(1,N); 28 | X(1) = x_start; 29 | a(1) = 1; 30 | accept(1)=1; 31 | 32 | 33 | %% PDF of the Multimodal Distribution 34 | 35 | multimod = @(X)(0.25*normpdf(X,-5,0.5)+0.35*normpdf(X,8,1)+0.23*normpdf(X,15,0.4)+0.17*normpdf(X,0,0.7)); 36 | 37 | 38 | %% MCMC-run 39 | 40 | for k=2:N 41 | % Proposal 42 | X_prop = X(k-1) + normrnd(0,propsigma); 43 | 44 | % Acceptance probability 45 | a(k) =min(1,(multimod(X_prop))/(multimod(X(k-1)))); 46 | 47 | % Accept?! 48 | if rand < a(k) 49 | X(k) = X_prop; 50 | accept(k) = 1; 51 | else 52 | X(k) = X(k-1); 53 | end 54 | 55 | end 56 | 57 | %% Plot the sample path, histogram, and correct pdf. 58 | 59 | FigHandle = figure(22); 60 | set(FigHandle, 'Position', [50, 600, 800, 400]); 61 | 62 | subplot(1,2,1) 63 | plot(X) 64 | 65 | subplot(1,2,2) 66 | h=histogram(X,'Normalization','probability'); 67 | h.NumBins = 30; 68 | hold on 69 | y = -10:0.1:20; 70 | f = multimod(y); 71 | plot(y,f,'LineWidth',1.5); 72 | hold off 73 | 74 | %% Derive the amount of accepted steps and estimate the 75 | % acceptance probability 76 | 77 | rel_accept = sum(accept)/N; 78 | estim_accept_prob = sum(a)/N; 79 | 80 | %% Return the relative Error of the Mean of the distribution 81 | 82 | Estim = sum(X)/N; 83 | 84 | % The expected value could be derived analytically. It is done numerically 85 | % for practical reasons. 86 | 87 | true = integral(@(x)multimod(x).*x,-Inf,Inf,'AbsTol',1e-8); 88 | 89 | rel_error = abs(Estim-true)/true --------------------------------------------------------------------------------