├── .gitattributes ├── GLMnet.f ├── README.md ├── ShowLandDistMap.m ├── TrendSeasonalFit_COLDLine.m ├── autoPara.m ├── autoRobustFit.m ├── autoTSFit.m ├── autoTSPred.m ├── autoTmask.m ├── cold_animation.gif ├── envihdrread.m ├── glmnetMex.F ├── glmnetMex.dll ├── glmnetMex.matlabR13.F ├── glmnetMex.mexa64 ├── glmnetMex.mexglx ├── glmnetMex.mexmaci ├── glmnetMex.mexmaci64 ├── glmnetMex.mexw32 ├── glmnetMex.mexw64 ├── glmnetSetL.m ├── glmnet_fast.m ├── label_dist_type.m ├── main_COLD.m ├── read_envihdr.m ├── robustfit_cor.m ├── rs_imwrite_bands.m ├── scanRequiredToolboxes.m ├── statrobustfit_cor.m └── update_cft.m /.gitattributes: -------------------------------------------------------------------------------- 1 | *.f linguist-language=Matlab 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # COLD 2 | Algorithm for COntinuous monitoring of Land Disturbance (COLD) using Landsat time series. It can detect many kinds of land disturbance continuously as new images are collected and provide historical land disturbance maps retrospectively. 3 | Please contact Zhe Zhu (zhe@uconn.edu) at Department of Natural Resources and the Environment, University of Connecticut if you have any questions. 4 | 5 | To stack the Landsat data into BIP format as the the input of the COLD algorithem, please check out the **autoPrepareDataARD.m** (if Landsat ARD) or **autoPrepareDataESPA.m** (if ESPA surface reflectance) in the [CCDC package](https://github.com/GERSL/CCDC). 6 | 7 | If no full MATLAB licenses, you may run **scanRequiredToolboxes.m** for displaying a list of all Matlab toolboxes that are required for executing this algorithm. 8 | 9 | **Reference dataset** are available at [this Google Drive](https://drive.google.com/file/d/1y53vCqY_K9W7R7jONFT6iI-FyH2tz_gR/view?usp=sharing), which include the reference data created by [TimeSync](https://timesync.forestry.oregonstate.edu) as well as the time series of Landsat surface and TOA reflectance. 10 | 11 | Please cite the following paper: 12 | Zhu, Z., Zhang, J., Yang, Z., Aljaddani, A.H., Cohen, W.B., Qiu, S. and Zhou, C., Continuous monitoring of land disturbance based on Landsat time series. Remote Sensing of Environment (2019). https://doi.org/10.1016/j.rse.2019.03.009 13 | 14 | ![plot](https://github.com/GERSL/COLD/blob/master/cold_animation.gif?raw=true) 15 | -------------------------------------------------------------------------------- /ShowLandDistMap.m: -------------------------------------------------------------------------------- 1 | function ShowLandDistMap(start_year,end_year) 2 | % This function is used to provde disturbance maps for each year 3 | % Version 1.00 No disturbance class in the cover map (03/29/2018) 4 | % vr = varead('COLD_log.txt','Version'); 5 | 6 | % get image parameters automatically 7 | imf = dir('L*'); % folder names 8 | 9 | % filter for Landsat folders 10 | imf = regexpi({imf.name}, 'L(T5|T4|E7|C8|ND)(\w*)', 'match'); 11 | imf = [imf{:}]; 12 | imf = vertcat(imf{:}); 13 | % name of the first stacked image 14 | filename = dir([imf(1,:),'/','L*stack']); 15 | % read in ENVI hdr 16 | info = read_envihdr([imf(1,:),'/',filename.name,'.hdr']); 17 | % provide values from info 18 | nrows = info.lines; 19 | ncols = info.samples; 20 | nbands = info.bands; 21 | % get current directory 22 | l_dir = pwd; 23 | 24 | % INPUTS: 25 | all_yrs = start_year:end_year;% all of years for producing maps 26 | % dimension and projection of the image 27 | jiDim = [ncols,nrows]; 28 | % max number of maps 29 | max_n = length(all_yrs); 30 | % slope threshold 31 | t_min = -200; % 0.02 change in surf ref 32 | 33 | % produce disturbance map 34 | LandDistMap = 9999*ones(nrows,ncols,max_n,'uint16'); % disturbance magnitude 35 | % most recent accumulated map 36 | AccuYearMap = 9999*ones(nrows,ncols,'uint16'); % disturbance magnitude 37 | % most recent accumulated map 38 | AccuTypeMap = 9999*ones(nrows,ncols,'uint8'); % disturbance magnitude 39 | 40 | % make Predict folder for storing predict images 41 | n_map = 'CCDCMap'; 42 | if isempty(dir(n_map)) 43 | mkdir(n_map); 44 | end 45 | 46 | % cd to the folder for storing recored structure 47 | % cd(v_input.name_rst); 48 | n_str = 'TSFitMap'; 49 | imf = dir([n_str,'/record_change*']); % folder names 50 | num_line = size(imf,1); 51 | 52 | for line = 1:num_line 53 | 54 | % show processing status 55 | if line/num_line < 1 56 | fprintf('Processing %.2f percent\r',100*(line/num_line)); 57 | else 58 | fprintf('Processing %.2f percent\n',100*(line/num_line)); 59 | end 60 | 61 | % load one line of time series models 62 | load([n_str,'/',imf(line).name]); %#ok 63 | 64 | % postions 65 | pos = [rec_cg.pos]; 66 | 67 | % continue if there is no model available 68 | l_pos = length(pos); 69 | if l_pos == 0 70 | continue 71 | end 72 | 73 | % break time 74 | t_break = [rec_cg.t_break]; 75 | % change probability 76 | change_prob = [rec_cg.change_prob]; 77 | % change vector magnitude 78 | mag = [rec_cg.magnitude]; 79 | % reshape magnitude 80 | mag = reshape(mag,nbands-1,[]); 81 | % coefficients 82 | coefs = [rec_cg.coefs]; 83 | coefs = reshape(coefs,8,nbands-1,[]); 84 | 85 | 86 | for i = 1:l_pos - 1 87 | % get row and col 88 | [I,J] = ind2sub(jiDim,pos(i)); 89 | 90 | % initialize pixels have at least one model 91 | if sum(LandDistMap(J,I,:) == 9999) == max_n 92 | % write doy to ChangeMap 93 | LandDistMap(J,I,:) = 0; 94 | AccuYearMap(J,I,:) = 0; 95 | end 96 | 97 | if change_prob(i) == 1 98 | [break_type,break_year,break_doy] = label_dist_type(coefs(:,:,i),t_break(i),t_min,mag(:,i),coefs(:,:,i+1)); 99 | % [break_type,break_year,break_doy] = label_dist_type(tst(j).coefs,tst(j).t_break,-200,tst(j).magnitude,tst(j+1).coefs); 100 | 101 | if break_type > 1 % land distrubance 102 | % get the band number for abrupt disturbance 103 | n_band = all_yrs == break_year; 104 | % magnitude of disturbance 105 | LandDistMap(J,I,n_band) = break_doy; 106 | % update accumulated by year map 107 | AccuYearMap(J,I) = break_year; 108 | end 109 | 110 | % update accumulated type map 111 | AccuTypeMap(J,I) = break_type; 112 | end 113 | end 114 | end 115 | 116 | rs_imwrite_bands(LandDistMap,[l_dir,'/',n_map,'/LandDistMap'],info,all_yrs); 117 | rs_imwrite_bands(AccuYearMap,[l_dir,'/',n_map,'/AccuYearMap'],info,start_year*100000+end_year); 118 | rs_imwrite_bands(AccuTypeMap,[l_dir,'/',n_map,'/AccuTypeMap'],info,'Disturb Type'); 119 | 120 | % % Scence or ARD 121 | % if zc >=1 && zc <= 60 122 | % % write ENVI files 123 | % enviwrite_bands([l_dir,'/',n_map,'/LandDistMap',char(vr)],LandDistMap,'uint16',res,jiUL,'bsq',zc,all_yrs); 124 | % enviwrite_bands([l_dir,'/',n_map,'/AccuYearMap',char(vr)],AccuYearMap,'uint16',res,jiUL,'bsq',start_year*100000+end_year); 125 | % else 126 | % % confirmed change 127 | % ARD_enviwrite_bands([l_dir,'/',n_map,'/LandDistMap',char(vr)],LandDistMap,'uint16','bsq',all_yrs); 128 | % ARD_enviwrite_bands([l_dir,'/',n_map,'/AccuYearMap',char(vr)],AccuYearMap,'uint16','bsq',start_year*100000+end_year); 129 | % end -------------------------------------------------------------------------------- /TrendSeasonalFit_COLDLine.m: -------------------------------------------------------------------------------- 1 | function rec_cg = TrendSeasonalFit_COLDLine(dir_l,n_rst,ncols,nrows,T_cg,Tmax_cg,conse,num_c,nbands,B_detect) 2 | % CCDC-COLD 13.4 version - Zhe Zhu, University of Connecticut, Storrs 3 | % Continuous Change Detection (CCD) designed for Continuous mOnitoring of Land Distubance (COLD) 4 | %% Revisions: $ Date: 9/03/2020 $ Copyright: Zhe Zhu 5 | % Version 13.04 Update model for every three observations (09/03/2020) 6 | % Version 13.03 Modified model intitializatin test (10/29/2018) 7 | % Version 13.02 Add included angle to exlcude false positive change (10/05/2018) 8 | % Version 13.01 Do not need clear observations more than 25 percent (03/28/2018) 9 | %% Version 13.0: Optimized parameters for monitoring disturbance - COLD (03/20/2018) 10 | % Version 12.36 Reduce 75 percent of memory needed for line_t (11/22/2017) 11 | % Version 12.35 Adjust T_cg based on conse (10/03/2017) 12 | % Version 12.34 Add data density requirement (09/20/2017) 13 | % Version 12.33 Adjust conse based on delta time (09/19/2017) 14 | % Version 12.32 Fix bud for not enough data (09/17/2017) 15 | % Version 12.31 Read input varibles in txt file (07/05/2016) 16 | % Version 12.30 Fixed a bug for pixels without minimum observations (11/20/2015) 17 | % Version 12.29 Modified fit for perennial snow and Fmask failed pixels (09/20/2015) 18 | % Version 12.29 Do not fit disturbed time period (09/18/2015) 19 | % Version 12.28 Fixed a bug for missing values in land cover maps (09/16/2015) 20 | % Version 12.27 Fixed bugs for persistent snow and falied Fmask pixels (06/17/2015) 21 | % Version 12.26 Connected time for all models (05/28/2015) 22 | % Version 12.25 Bug fixed in snow percent (05/19/2015) 23 | % Version 12.24 Change T_const in Tmask (03/31/2015) 24 | % Version 12.23 Update iteratively before 24 observations (03/22/2015) 25 | % Version 12.22 Adjust mini RMSE based on temporal variability (03/22/2015) 26 | % Version 12.21 Add more categories and update i_start in the end (03/14/2015) 27 | % Version 12.20 Convert BT to from K to C before analysis (03/12/2015) 28 | % Version 12.19 Fit for permanent snow if is more than 75% (03/12/2015) 29 | % Version 12.18 No change detection if clear observation less than 25% (03/12/2015) 30 | % Version 12.17 Use median value for very simple model & change magnitude (02/24/2015) 31 | % Version 12.16 Finding changes in all water pixels (02/24/2015) 32 | % Version 12.15 Use the original multitemporal cloud mask (02/15/2015) 33 | % Version 12.14 Do not need example_img in images folder (02/09/2015) 34 | % Version 12.13: More infromation in "category" (11/10/2014) 35 | % This version (12.13) is used for the third round of the LCMS project. 36 | % Command: TrendSeasonalFit_v12Plot(N_row,N_col,min=0.5,T_cg=0.99,n_times=3,conse=6,B_detect=2:6) 37 | % Version 12.12: Fit for pixels where Fmask fails (11/09/2014) 38 | % Version 12.11: Bug fixed in num_fc (11/09/2014) 39 | % Version 12.10: Better multietmporal cloud detection at the beginning (11/06/2014) 40 | % Version 12.9: Detect change only for land pixel (water/snow speical case) (10/31/2014) 41 | % Version 12.8: Speed up by reducing time for RMSE and model computing (10/17/2014) 42 | % Version 12.7: mini rmse should be larger than 10% of the mean (10/13/2014) 43 | % Version 12.6: Fit model again when there are a 33.3% more data (10/08/2014) 44 | % Version 12.5: Use subset of bands (2-6) for detecting surface change (10/01/2014) 45 | % Version 12.4: Only apply multitemporal cloud masking during model initialization (09/29/2014) 46 | % Version 12.3: Use subset of bands (3-5) to balance change in diferent dimensions (09/01/2014) 47 | % This version (12.3) is used for the second round of the LCMS project. 48 | % Command: TrendSeasonalFit_v12Plot(N_row,N_col,min=1,T_cg=0.99,n_times=3,conse=5,B_detect=3:6) 49 | % Version 12.2: Bug fixed in model intialization (08/14/2014) 50 | % Version 12.1: Use subset of bands (3-6) to avoid atmosphere influences (08/04/2014) 51 | %% Version 12.0 Detecting change based on probability (07/19/2014) 52 | % Version 11.6: No need to change folder name & faster in speed (by Christ Holden 06/06/2014) 53 | % Version 11.5: Improved calculation of temporally adjusted RMSE (04/23/2014) 54 | % Version 11.4: Revise "rec_cg.category" to better seperate different fit processes (04/01/2014) 55 | % This version (11.4) is used for generating synthetic data for ACRE project and 56 | % detecting change for LCMS project. 57 | % Command: TrendSeasonalFit_v11Plot(N_row,N_col,min=1,T_cg=2,n_times=3,conse=6,B_detect=1:6) 58 | % Version 11.3: Add "rec_cg.magnitude" as indicator of change magnitude (04/01/2014) 59 | % Version 11.2: Change very simple fit with mean value for start and end of timeseries (04/01/2014) 60 | % Version 11.1: Do not need metadata in the image folder to run CCDC (03/25/2014) 61 | %% Version 11.0: Use change vector magnitude as threshold for detecting change (03/25/2014) 62 | % Version 10.13: Use subset of bands (1-6) to avoid atmosphere influences (01/31/2014) 63 | % Version 10.12: More accurate number of days per year "num_yrs" (01/30/2014) 64 | % Version 10.11: RMSE up% agriculture activity (9)dates with time series fit (01/26/2014) 65 | % Version 10.10: Update temperature extreme in recent studies (01/16/2014) 66 | % Version 10.9: Find break in max value in any of the band (01/08/2014) 67 | % Version 10.8: Add very simple fit with median value for start and end of timeseries (10/21/2013) 68 | % This version (10.8) is used for generating synthetic data for the LCMS project. 69 | % Command: TrendSeasonalFit_v10Plot('stack',N_row,N_col,mini=0.5,T_cg=3,n_times=3,conse=6,B_detect=2:6) 70 | % Version 10.7: Better multitemporal cloud detection (10/19/2013) 71 | % Version 10.6: Add "Tmax_cg" for last step noise removal (10/18/2013) 72 | % Version 10.5: Use subset of bands (2-6) to avoid atmosphere influences (10/18/2013) 73 | % Version 10.4: Let dynamic fitting for pixels at the beginning (09/23/2013) 74 | % Version 10.3: Able to detect change at the verying beginning (09/06/2013) 75 | % Version 10.2: Add mini years "mini_yrs" in model intialization (09/03/2013) 76 | % Version 10.1: Reduce time for calcuating "v_dif" (09/02/2013) 77 | %% Version 10.0: Fit for beginning and end of the time series (08/31/2013) 78 | % Version 9.9: Only fit more than 50% of Landat images overlap area (08/28/2013) 79 | % Version 9.8: Force model fit for persistent snow pixels (08/27/2013) 80 | % Version 9.7: Add "rec_cg.category" as indicator of fitting procudure (08/20/2013) 81 | % Add rec_cg.change_prob as indicator of change probability (08/20/2013) 82 | % Add rec_cg.num_obs ad indicator of number of observations (08/20/2013) 83 | % Version 9.6: Remove mininum rmse "mini" and minimum years "mini_yrs" (08/16/2013) 84 | % Version 9.5: Model gets more coefficients with more observations (08/16/2013) 85 | % Version 9.4: Bug fixed in calculating temporally adjusted rmse (08/01/2013) 86 | % Version 9.3: Fit curve again after one year (03/28/2013) 87 | % This version (9.3) is used for mapping land cover for the IDS project. 88 | % Command: TrendSeasonalFit_v9Plot('stack',N_row,N_col,T_cg=2,n_times=3,conse=4) 89 | % Version 9.2: Use "mini = T_const/T_cg" for small rmse cases (03/26/2013) 90 | % Version 9.1: Remove out of range pixels before time series analysis (02/09/2013) 91 | %% Version 9.0: Using 8 coefficients and lasso fit (02/01/2013) 92 | % Version 8.4: Use "max v_slope" instead of "average v_slope" (01/16/2013) 93 | % Version 8.3: Start initialization when "time_span" > 1 year (01/16/2013) 94 | % Version 8.2: Bug fix% agriculture activity (9)ed in not fitting models at the begining (01/16/2013) 95 | % Version 8.1: Bug fixed in counting "i" and "i_span"(01/13/2013) 96 | %% Version 8.0: Temporally changing RMSE (01/09/2013) 97 | %% Version 7.3: Continuous Change Detection and Classification (CCDC) (07/11/2012) 98 | % This version (7.3) is explained by Zhu, Z. & Woodcock, C.E., Continuous Change 99 | % Detection and Classification (CCDC) of land cover using all available 100 | % Landsat data, Remote Sensing of Environment (2014). 101 | % Command: TrendSeasonalFit_v7Plot('stack',N_row,N_col,T_cg=3,n_times=3,conse=3) 102 | %% Version 1.0: Continous Monitoring of Forest Disturbance Algorithm (CMFDA) (07/13/2010) 103 | % This version (1.0) is explained by Zhu, Z., Woodcock, C.E., Olofsson, P., 104 | % Continuous monitoring of forest disturbance using all available Landsat 105 | % data, Remote Sensing of Environment (2012). 106 | % 107 | %% Inputs: 108 | % stk_n='stack'; stack image name 109 | % ncols = 8021; % number of pixels processed per line 110 | % nrows=1; % the nrowsth lines 111 | % for example 1 2 3 4 5 112 | % 6 7 8 9 10 113 | % 114 | %% Outputs: 115 | % 116 | % rec_cg RECord information about all curves between ChanGes 117 | % rec_cg(i).t_start record the start of the ith curve fitting (julian_date) 118 | % rec_cg(i).t_end record the end of the ith curve fitting (julian_date) 119 | % rec_cg(i).t_break record the first observed break time (julian_date) 120 | % rec_cg(i).coefs record the coefficients of the ith curve 121 | % rec_cg(i).pos record the position of the ith pixel (pixel id) 122 | % rec_cg(i).magnitude record the change vector of all spectral bands 123 | % rec_cg(i).category record what fitting procudure and model is used 124 | % cateogry category 5x: persistent snow 4x: Fmask fails 125 | % cateogry category 3x: modified fit 2x: end fit 126 | % category category 1x: start fit x: normal procedure 127 | % cateogry category x1: mean value x4: simple model 128 | % category category x6: advanced model x8: full model 129 | % 130 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% defining variables 131 | %% Constants 132 | % maximum number of coefficient required 133 | % 2 for tri-modal; 2 for bi-modal; 2 for seasonality; 2 for linear; 134 | min_num_c = 4; 135 | mid_num_c = 6; 136 | max_num_c = 8; 137 | % number of clear observation / number of coefficients 138 | n_times = 3; 139 | % initialize NUM of Functional Curves 140 | num_fc = 0; 141 | % number of days per year 142 | num_yrs = 365.25; 143 | % number of bytes: int16 144 | num_byte = 2; 145 | % Band for multitemporal cloud/snow detection (Green) 146 | num_B1 = 2; 147 | % Band for multitemporal shadow/snow shadow detection (SWIR) 148 | num_B2 = 5; 149 | % Threshold for cloud, shadow, and snow detection. 150 | T_const = 4.42; 151 | % minimum year for model intialization 152 | mini_yrs = 1; 153 | % no change detection for permanent snow pixels 154 | t_sn = 0.75; 155 | % threshold (degree) of mean included angle 156 | nsign = 45; 157 | % Fmask fails threshold 158 | % t_clr = 0.25; 159 | % agriculture activity (9) 160 | % get num of total folders start with "L" 161 | imf = dir('L*'); % folder names 162 | % filter for Landsat folders 163 | imf = regexpi({imf.name}, 'L(T5|T4|E7|C8|ND)(\w*)', 'match'); 164 | imf = [imf{:}]; 165 | imf = vertcat(imf{:}); 166 | % sort according to yeardoy 167 | yeardoy = str2num(imf(:, 10:16)); 168 | [~, sort_order] = sort(yeardoy); 169 | imf = imf(sort_order, :); 170 | % number of folders start with "L" 171 | num_t = size(imf,1); 172 | % consecutive number 173 | def_conse = conse; 174 | 175 | % Tmasking of noise 176 | Tmax_cg = chi2inv(Tmax_cg,length(B_detect)); 177 | % adjust threshold based on chi-squared distribution 178 | def_pT_cg = T_cg; 179 | def_T_cg = chi2inv(def_pT_cg,length(B_detect)); 180 | 181 | % initialize the struct data of RECording of ChanGe (rec_cg) 182 | rec_cg = struct('t_start',[],'t_end',[],'t_break',[],'coefs',[],'rmse',[],... 183 | 'pos',[],'change_prob',[],'num_obs',[],'category',[],'magnitude',[]); 184 | 185 | % % mask for study area (1 fit, 0 no fit) 186 | % fit_mask = enviread('GZ_Mask'); 187 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%Get ready for Xs & Ys 188 | %% Read in Xs & Ysq 189 | % transforming to serial date number (0000 year) 190 | sdate = zeros(num_t,1); % Xs 191 | line_t = zeros(num_t,nbands*ncols); %Ys 192 | 193 | for i = 1:num_t 194 | im_dir = dir(imf(i, :)); 195 | im = ''; 196 | for f = 1:size(im_dir, 1) 197 | % use regular expression to match: 198 | % 'L(\w*)' Any word begining with L that has any following chars 199 | % stk_n includes stack name somewhere after L 200 | % '$' ends with the stack name (e.g., no .hdr, .aux.xml) 201 | if regexp(im_dir(f).name, ['L(\w*)', 'stack', '$']) == 1 202 | im = [imf(i, :), '/', im_dir(f).name]; 203 | break 204 | end 205 | end 206 | % Check to make sure we found something 207 | if strcmp(im, '') 208 | error('Could not find stack image for directory %s\n', imf(i)); 209 | end 210 | % Find date for folder imf(i) 211 | yr = str2num(imf(i, 10:13)); %#ok<*ST2NM> 212 | doy = str2num(imf(i, 14:16)); 213 | sdate(i) = datenum(yr, 1, 0) + doy; 214 | dummy_name = im; 215 | fid_t = fopen(dummy_name,'r'); % get file ids 216 | fseek(fid_t,num_byte*(nrows-1)*ncols*nbands,'bof'); 217 | line_t(i,:) = fread(fid_t,nbands*ncols,'int16','ieee-le'); % get Ys 218 | end 219 | fclose('all'); % close all files 220 | 221 | for i_ids = 1:ncols 222 | % get default conse & T_cg 223 | conse = def_conse; 224 | T_cg = def_T_cg; 225 | % mask data 226 | line_m = line_t(:,nbands*i_ids); 227 | 228 | % % Only run CCDC for places where more than 50% of images has data 229 | % idexist = line_m < 255; 230 | % overlap_pct = sum(idexist)/num_t; 231 | % if overlap_pct < 0.5 232 | % continue; 233 | % end 234 | 235 | % convert Kelvin to Celsius 236 | line_t(:,nbands*(i_ids-1)+7) = 10*(line_t(:,nbands*(i_ids-1)+7) - 2732); 237 | 238 | % clear pixel should have reflectance between 0 and 1 239 | % brightness temperature should between -93.2 to 70.7 celsius degree 240 | idrange = line_t(:,nbands*(i_ids-1)+1)>0&line_t(:,nbands*(i_ids-1)+1)<10000&... 241 | line_t(:,nbands*(i_ids-1)+2)>0&line_t(:,nbands*(i_ids-1)+2)<10000&... 242 | line_t(:,nbands*(i_ids-1)+3)>0&line_t(:,nbands*(i_ids-1)+3)<10000&... 243 | line_t(:,nbands*(i_ids-1)+4)>0&line_t(:,nbands*(i_ids-1)+4)<10000&... 244 | line_t(:,nbands*(i_ids-1)+5)>0&line_t(:,nbands*(i_ids-1)+5)<10000&... 245 | line_t(:,nbands*(i_ids-1)+6)>0&line_t(:,nbands*(i_ids-1)+6)<10000&... 246 | line_t(:,nbands*(i_ids-1)+7)>-9320&line_t(:,nbands*(i_ids-1)+7)<7070; 247 | 248 | % # of clear observatons 249 | idclr = line_m < 2; 250 | % # of all available observations 251 | % idall = line_m < 255; 252 | % clear observation percentage 253 | % clr_pct = sum(idclr)/sum(idall); 254 | % snow pixels 255 | idsn = line_m == 3; 256 | % percent of snow observations 257 | sn_pct = sum(idsn)/(sum(idclr)+sum(idsn)+0.01); 258 | 259 | % not enough clear observations for change detection 260 | if sum(idclr) < n_times*max_num_c 261 | % permanent snow pixels 262 | if sn_pct > t_sn 263 | % snow observations are "good" now 264 | idgood = idsn|idclr; 265 | % number of snow pixel within range 266 | n_sn = sum(idgood); 267 | 268 | if n_sn < n_times*min_num_c % not enough snow pixels 269 | continue 270 | else 271 | % Xs & Ys for computation 272 | clrx = sdate(idgood); 273 | % bands 1-5,7,6 274 | clry = line_t(idgood,(nbands*(i_ids-1)+1):(nbands*(i_ids-1)+nbands-1)); 275 | clry = double(clry); 276 | % find repeated ids 277 | [clrx,uniq_id,~] = unique(clrx); 278 | % mean of repeated values 279 | tmp_y = zeros(length(clrx),nbands-1); 280 | % get the mean values 281 | for i = 1:nbands-1 282 | tmp_y(:,i) = clry(uniq_id,i); 283 | end 284 | clry = tmp_y; 285 | 286 | % the first observation for TSFit 287 | i_start = 1; 288 | % identified and move on for the next curve 289 | num_fc = num_fc + 1; % NUM of Fitted Curves (num_fc) 290 | 291 | % defining computed variables 292 | fit_cft = zeros(max_num_c,nbands-1); 293 | % rmse for each band 294 | rmse = zeros(nbands-1,1); 295 | % snow qa = 50 296 | qa = 50; 297 | 298 | for i_B=1:nbands-1 299 | if i_B ~= nbands-1 % treat saturated and unsaturated pixels differently 300 | idgood = clry(:,i_B) < 10000; % saturate if ref > 1 or NBR NDVR > 1 301 | i_span = sum(idgood); 302 | if i_span < min_num_c*n_times % fill value for frequently saturated snow pixels 303 | fit_cft(1,i_B) = 10000; % give constant value 304 | else % fit for enough unsaturat snow pixels 305 | [fit_cft(:,i_B),rmse(i_B)]=autoTSFit(clrx(idgood),clry(idgood,i_B),min_num_c); 306 | end 307 | else % fit for temperature band 308 | idgood = clry(:,i_B)>-9320&clry(:,i_B)<7070; 309 | [fit_cft(:,i_B),rmse(i_B)]=autoTSFit(clrx(idgood),clry(idgood,i_B),min_num_c); 310 | end 311 | end 312 | 313 | % updating information at each iteration 314 | % record time of curve start 315 | rec_cg(num_fc).t_start=clrx(i_start); 316 | % record time of curve end 317 | rec_cg(num_fc).t_end=clrx(end); 318 | % record break time 319 | rec_cg(num_fc).t_break = 0; % no break at the moment 320 | % record postion of the pixel 321 | rec_cg(num_fc).pos = (nrows-1)*ncols+i_ids; 322 | % record fitted coefficients 323 | rec_cg(num_fc).coefs = fit_cft; 324 | % record rmse of the pixel 325 | rec_cg(num_fc).rmse = rmse; 326 | % record change probability 327 | rec_cg(num_fc).change_prob = 0; 328 | % record number of observations 329 | rec_cg(num_fc).num_obs = n_sn; 330 | % record fit category 331 | rec_cg(num_fc).category = qa + min_num_c; 332 | % record change magnitude 333 | rec_cg(num_fc).magnitude = zeros(1,nbands-1); 334 | end 335 | else % no change detection for clear observations 336 | % within physical range pixels 337 | idgood = idrange; 338 | 339 | % Xs & Ys for computation 340 | clrx = sdate(idgood); 341 | % bands 1-5,7,6 342 | clry = line_t(idgood,(nbands*(i_ids-1)+1):(nbands*(i_ids-1)+nbands-1)); 343 | clry = double(clry); 344 | % find repeated ids 345 | [clrx,uniq_id,~] = unique(clrx); 346 | % mean of repeated values 347 | tmp_y = zeros(length(clrx),nbands-1); 348 | % get the mean values 349 | for i = 1:nbands-1 350 | tmp_y(:,i) = clry(uniq_id,i); 351 | end 352 | clry = tmp_y; 353 | 354 | idclr = clry(:,num_B1) < median(clry(:,num_B1)) + 400; 355 | n_clr = sum(idclr); 356 | 357 | if n_clr < n_times*min_num_c % not enough clear pixels 358 | continue 359 | else 360 | % Xs & Ys for computation 361 | clrx = clrx(idclr); 362 | clry = clry(idclr,:); 363 | 364 | % the first observation for TSFit 365 | i_start = 1; 366 | % identified and move on for the next curve 367 | num_fc = num_fc + 1; % NUM of Fitted Curves (num_fc) 368 | 369 | % defining computed variables 370 | fit_cft = zeros(max_num_c,nbands-1); 371 | % rmse for each band 372 | rmse = zeros(nbands-1,1); 373 | % Fmask fail qa = 40 374 | qa = 40; 375 | 376 | for i_B = 1:nbands-1 377 | % fit basic model for all within range snow pixels 378 | [fit_cft(:,i_B),rmse(i_B)] = autoTSFit(clrx,clry(:,i_B),min_num_c); 379 | end 380 | 381 | % record time of curve start 382 | rec_cg(num_fc).t_start = clrx(i_start); 383 | % record time of curve end 384 | rec_cg(num_fc).t_end = clrx(end); 385 | % record break time 386 | rec_cg(num_fc).t_break = 0; 387 | % record postion of the pixel 388 | rec_cg(num_fc).pos = (nrows-1)*ncols+i_ids; 389 | % record fitted coefficients 390 | rec_cg(num_fc).coefs = fit_cft; 391 | % record rmse of the pixel 392 | rec_cg(num_fc).rmse = rmse; 393 | % record change probability 394 | rec_cg(num_fc).change_prob = 0; 395 | % record number of observations 396 | rec_cg(num_fc).num_obs = length(clrx); 397 | % record fit category 398 | rec_cg(num_fc).category = qa + min_num_c; 399 | % record change magnitude 400 | rec_cg(num_fc).magnitude = zeros(1,nbands-1); 401 | end 402 | end 403 | else % normal CCDC procedure 404 | 405 | % clear and within physical range pixels 406 | idgood = idclr & idrange; 407 | 408 | % Xs & Ys for computation 409 | clrx = sdate(idgood); 410 | % bands 1-5,7,6 411 | clry = line_t(idgood,(nbands*(i_ids-1)+1):(nbands*(i_ids-1)+nbands-1)); 412 | clry = double(clry); 413 | 414 | % find repeated ids 415 | [clrx,uniq_id,~] = unique(clrx); 416 | 417 | % continue if not enough clear pixels 418 | if length(clrx) < n_times*min_num_c+conse 419 | continue 420 | end 421 | 422 | % mean of repeated values 423 | tmp_y = zeros(length(clrx),nbands-1); 424 | % get the mean values 425 | for i = 1:nbands-1 426 | tmp_y(:,i) = clry(uniq_id,i); 427 | end 428 | clry = tmp_y; 429 | 430 | % caculate median variogram 431 | var_clry = clry(2:end,:)-clry(1:end-1,:); 432 | % var_clry = clry(3:end,:)-clry(1:end-2,:); 433 | adj_rmse = median(abs(var_clry),1); 434 | 435 | % adjust conse based on delta days 436 | var_clrx = median(clrx(2:end)-clrx(1:end-1)); 437 | adj_conse = round(def_conse*16/var_clrx); 438 | if adj_conse < def_conse 439 | adj_conse = def_conse; 440 | end 441 | conse = adj_conse; 442 | 443 | % adjust T_cg based on conse 444 | if conse > def_conse 445 | pT_cg = 1-(1-def_pT_cg)^(def_conse/conse); 446 | T_cg = chi2inv(pT_cg,length(B_detect)); 447 | end 448 | 449 | % start with the miminum requirement of clear obs 450 | i = n_times*min_num_c; 451 | 452 | % initializing variables 453 | % the first observation for TSFit 454 | i_start = 1; 455 | % record the start of the model initialization (0=>initial;1=>done) 456 | BL_train = 0; 457 | % identified and move on for the next curve 458 | num_fc = num_fc + 1; % NUM of Fitted Curves (num_fc) 459 | % record the num_fc at the beginning of each pixel 460 | rec_fc = num_fc; 461 | % initialize i_dense 462 | i_dense = 1; 463 | 464 | % while loop - process till the last clear observation - conse 465 | while i<= length(clrx)-conse 466 | % span of "i" 467 | i_span = i-i_start+1; 468 | % span of time (num of years) 469 | time_span = (clrx(i)-clrx(i_start))/num_yrs; 470 | % max time difference 471 | time_dif = max(clrx(i_start+1:i) - clrx(i_start:i-1)); 472 | 473 | % basic requrirements: 1) enough observations; 2) enough time 474 | if i_span >= n_times*min_num_c && time_span >= mini_yrs 475 | % initializing model 476 | if BL_train == 0 477 | % check max time difference 478 | if time_dif > num_yrs 479 | i = i+1; 480 | i_start = i_start+1; 481 | % i that is dense enough 482 | i_dense = i_start; 483 | continue 484 | end 485 | % Tmask: noise removal (good => 0 & noise => 1) 486 | blIDs = autoTmask(clrx(i_start:i+conse),clry(i_start:i+conse,[num_B1,num_B2]),... 487 | (clrx(i+conse)-clrx(i_start))/num_yrs,adj_rmse(num_B1),adj_rmse(num_B2),T_const); 488 | 489 | % IDs to be removed 490 | IDs = i_start:i+conse; 491 | rmIDs = IDs(blIDs(1:end-conse) == 1); 492 | 493 | % update i_span after noise removal 494 | i_span = sum(~blIDs(1:end-conse)); 495 | 496 | % check if there is enough observation 497 | if i_span < n_times*min_num_c 498 | % move forward to the i+1th clear observation 499 | i = i+1; 500 | % not enough clear observations 501 | continue; 502 | % check if there is enough time 503 | else 504 | % copy x & y 505 | cpx = clrx; 506 | cpy = clry; 507 | 508 | % remove noise pixels between i_start & i 509 | cpx(rmIDs) = []; 510 | cpy(rmIDs,:) = []; 511 | 512 | % record i before noise removal 513 | % This is very important as if model is not initialized 514 | % the multitemporal masking shall be done again instead 515 | % of removing outliers in every masking 516 | i_rec = i; 517 | 518 | % update i afer noise removal (i_start stays the same) 519 | i = i_start+i_span-1; 520 | % update span of time (num of years) 521 | time_span = (cpx(i)-cpx(i_start))/num_yrs; 522 | 523 | % check if there is enough time 524 | if time_span < mini_yrs 525 | % keep the original i 526 | i = i_rec; 527 | % move forward to the i+1th clear observation 528 | i = i+1; 529 | % not enough time 530 | continue; 531 | % Step 2: model fitting 532 | else 533 | % remove noise 534 | clrx = cpx; 535 | clry = cpy; 536 | 537 | % Step 2: model fitting 538 | % initialize model testing variables 539 | % defining computed variables 540 | fit_cft = zeros(max_num_c,nbands-1); 541 | % rmse for each band 542 | rmse = zeros(nbands-1,1); 543 | % value of differnce 544 | v_dif = zeros(nbands-1,1); 545 | % record the diference in all bands 546 | rec_v_dif = zeros(i-i_start+1,nbands-1); 547 | 548 | for i_B = 1:nbands-1 549 | % initial model fit 550 | [fit_cft(:,i_B),rmse(i_B),rec_v_dif(:,i_B)] = ... 551 | autoTSFit(clrx(i_start:i),clry(i_start:i,i_B),min_num_c); 552 | end 553 | 554 | % normalized to z-score 555 | for i_B = B_detect 556 | % minimum rmse 557 | mini_rmse = max(adj_rmse(i_B),rmse(i_B)); 558 | 559 | % compare the first clear obs 560 | v_start = rec_v_dif(1,i_B)/mini_rmse; 561 | % compare the last clear observation 562 | v_end = rec_v_dif(end,i_B)/mini_rmse; 563 | % anormalized slope values 564 | v_slope = fit_cft(2,i_B)*(clrx(i)-clrx(i_start))/mini_rmse; 565 | 566 | % differece in model intialization 567 | v_dif(i_B) = abs(v_slope) + max(abs(v_start),abs(v_end)); 568 | end 569 | v_dif = norm(v_dif(B_detect))^2; 570 | 571 | % find stable start for each curve 572 | if v_dif > T_cg 573 | % start from next clear obs 574 | i_start = i_start + 1; 575 | % move forward to the i+1th clear observation 576 | i = i + 1; 577 | % keep all data and move to the next obs 578 | continue 579 | else 580 | % model ready! 581 | BL_train = 1; 582 | % count difference of i for each iteration 583 | i_count = 0; 584 | 585 | % find the previous break point 586 | if num_fc == rec_fc 587 | % first curve 588 | i_break = 1; 589 | else 590 | % after the first curve 591 | i_break = find(clrx >= rec_cg(num_fc-1).t_break); 592 | i_break = i_break(1); 593 | end 594 | 595 | if i_start > i_break 596 | % model fit at the beginning of the time series 597 | for i_ini = i_start-1:-1:i_break 598 | if i_start - i_break < conse 599 | ini_conse = i_start - i_break; 600 | else 601 | ini_conse = conse; 602 | end 603 | % value of difference for conse obs 604 | v_dif = zeros(ini_conse,nbands-1); 605 | % record the magnitude of change 606 | v_dif_mag = v_dif; 607 | % chagne vector magnitude 608 | vec_mag = zeros(ini_conse,1); 609 | 610 | for i_conse = 1:ini_conse 611 | for i_B = 1:nbands-1 612 | % absolute difference 613 | v_dif_mag(i_conse,i_B) = clry(i_ini-i_conse+1,i_B)-autoTSPred(clrx(i_ini-i_conse+1),fit_cft(:,i_B)); 614 | % normalized to z-scores 615 | if sum(i_B == B_detect) 616 | % minimum rmse 617 | mini_rmse = max(adj_rmse(i_B),rmse(i_B)); 618 | 619 | % z-scores 620 | v_dif(i_conse,i_B) = v_dif_mag(i_conse,i_B)/mini_rmse; 621 | end 622 | end 623 | vec_mag(i_conse) = norm(v_dif(i_conse,B_detect))^2; 624 | end 625 | 626 | % get the vec sign 627 | % vec_sign = abs(sum(sign(v_dif(:,B_detect)),1)); 628 | max_angl = mean(angl(v_dif(:,B_detect))); 629 | 630 | % change detection 631 | if min(vec_mag) > T_cg && max_angl < nsign% change detected 632 | break 633 | elseif vec_mag(1) > Tmax_cg % false change 634 | % remove noise 635 | clrx(i_ini) = []; 636 | clry(i_ini,:) = []; 637 | i=i-1; % stay & check again after noise removal 638 | end 639 | 640 | % update new_i_start if i_ini is not a confirmed break 641 | i_start = i_ini; 642 | end 643 | end 644 | % only fit first curve if 1) have more than 645 | % conse obs 2) previous obs is less than a year 646 | if num_fc == rec_fc && i_start - i_dense >= conse 647 | % defining computed variables 648 | fit_cft = zeros(max_num_c,nbands-1); 649 | % rmse for each band 650 | rmse = zeros(nbands-1,1); 651 | % start fit qa = 10 652 | qa = 10; 653 | 654 | for i_B=1:nbands-1 655 | [fit_cft(:,i_B),rmse(i_B)] = ... 656 | autoTSFit(clrx(i_dense:i_start-1),clry(i_dense:i_start-1,i_B),min_num_c); 657 | end 658 | 659 | % record time of curve end 660 | rec_cg(num_fc).t_end = clrx(i_start-1); 661 | % record postion of the pixel 662 | rec_cg(num_fc).pos = (nrows-1)*ncols + i_ids; 663 | % record fitted coefficients 664 | rec_cg(num_fc).coefs = fit_cft; 665 | % record rmse of the pixel 666 | rec_cg(num_fc).rmse = rmse; 667 | % record break time 668 | rec_cg(num_fc).t_break = clrx(i_start); 669 | % record change probability 670 | rec_cg(num_fc).change_prob = 1; 671 | % record time of curve start 672 | rec_cg(num_fc).t_start = clrx(1); 673 | % record fit category 674 | rec_cg(num_fc).category = qa + min_num_c; 675 | % record number of observations 676 | rec_cg(num_fc).num_obs = i_start - i_dense; 677 | % record change magnitude 678 | rec_cg(num_fc).magnitude = - median(v_dif_mag,1); 679 | 680 | % identified and move on for the next functional curve 681 | num_fc = num_fc + 1; 682 | end 683 | end 684 | end 685 | end 686 | end 687 | 688 | % continuous monitoring started!!! 689 | if BL_train == 1 690 | % all IDs 691 | IDs = i_start:i; 692 | % span of "i" 693 | i_span = i-i_start+1; 694 | 695 | % determine the time series model 696 | update_num_c = update_cft(i_span,n_times,min_num_c,mid_num_c,max_num_c,num_c); 697 | 698 | % initial model fit when there are not many obs 699 | if i_count == 0 || i_span <= max_num_c*n_times 700 | % update i_count at each interation 701 | i_count = clrx(i)-clrx(i_start); 702 | 703 | % record previous end i ######Junxue 704 | pre_end = i; 705 | 706 | % defining computed variables 707 | fit_cft = zeros(max_num_c,nbands-1); 708 | % rmse for each band 709 | rmse = zeros(nbands-1,1); 710 | % record the diference in all bands 711 | rec_v_dif = zeros(length(IDs),nbands-1); 712 | % normal fit qa = 0 713 | qa = 0; 714 | 715 | for i_B=1:nbands-1 716 | [fit_cft(:,i_B),rmse(i_B),rec_v_dif(:,i_B)] = ... 717 | autoTSFit(clrx(IDs),clry(IDs,i_B),update_num_c); 718 | end 719 | 720 | % updating information for the first iteration 721 | % record time of curve start 722 | rec_cg(num_fc).t_start = clrx(i_start); 723 | % record time of curve end 724 | rec_cg(num_fc).t_end = clrx(i); 725 | % record break time 726 | rec_cg(num_fc).t_break = 0; % no break at the moment 727 | % record postion of the pixel 728 | rec_cg(num_fc).pos = (nrows-1)*ncols+i_ids; 729 | % record fitted coefficients 730 | rec_cg(num_fc).coefs = fit_cft; 731 | % record rmse of the pixel 732 | rec_cg(num_fc).rmse = rmse; 733 | % record change probability 734 | rec_cg(num_fc).change_prob = 0; 735 | % record number of observations 736 | rec_cg(num_fc).num_obs = i-i_start+1; 737 | % record fit category 738 | rec_cg(num_fc).category = qa + update_num_c; 739 | % record change magnitude 740 | rec_cg(num_fc).magnitude = zeros(1,nbands-1); 741 | 742 | % detect change 743 | % value of difference for conse obs 744 | v_dif = zeros(conse,nbands-1); 745 | % record the magnitude of change 746 | v_dif_mag = v_dif; 747 | vec_mag = zeros(conse,1); 748 | 749 | for i_conse = 1:conse 750 | for i_B = 1:nbands-1 751 | % absolute difference 752 | v_dif_mag(i_conse,i_B) = clry(i+i_conse,i_B)-autoTSPred(clrx(i+i_conse),fit_cft(:,i_B)); 753 | % normalized to z-scores 754 | if sum(i_B == B_detect) 755 | % minimum rmse 756 | mini_rmse = max(adj_rmse(i_B),rmse(i_B)); 757 | 758 | % z-scores 759 | v_dif(i_conse,i_B) = v_dif_mag(i_conse,i_B)/mini_rmse; 760 | end 761 | end 762 | vec_mag(i_conse) = norm(v_dif(i_conse,B_detect))^2; 763 | end 764 | % IDs that haven't updated 765 | IDsOld = IDs; 766 | else 767 | if i - pre_end >= 3%######Junxue %clrx(i)-clrx(i_start) >= num_yrs%1.33*i_count 768 | % update i_count at each interation 769 | i_count = clrx(i)-clrx(i_start); 770 | 771 | % record previous end i ######Junxue 772 | pre_end = i; 773 | 774 | % defining computed variables 775 | fit_cft = zeros(max_num_c,nbands-1); 776 | % rmse for each band 777 | rmse = zeros(nbands-1,1); 778 | % record the diference in all bands 779 | rec_v_dif = zeros(length(IDs),nbands-1); 780 | % normal fit qa = 0 781 | qa = 0; 782 | 783 | % % moving window with nyr = 5 start 784 | % nyr = 5; 785 | % id_nyr = find(clrx(IDs(end))-clrx(IDs) < nyr*num_yrs); 786 | % if ~isempty(id_nyr) && length(id_nyr) >= max_num_c*n_times 787 | % IDs = IDs(id_nyr(1):end); 788 | % end 789 | % % end of moving window 790 | 791 | for i_B = 1:nbands-1 792 | [fit_cft(:,i_B),rmse(i_B),rec_v_dif(:,i_B)] = ... 793 | autoTSFit(clrx(IDs),clry(IDs,i_B),update_num_c); 794 | end 795 | 796 | % record fitted coefficients 797 | rec_cg(num_fc).coefs = fit_cft; 798 | % record rmse of the pixel 799 | rec_cg(num_fc).rmse = rmse; 800 | % record number of observations 801 | rec_cg(num_fc).num_obs = i-i_start+1; 802 | % record fit category 803 | rec_cg(num_fc).category = qa + update_num_c; 804 | 805 | % IDs that haven't updated 806 | IDsOld = IDs; 807 | end 808 | 809 | % record time of curve end 810 | rec_cg(num_fc).t_end = clrx(i); 811 | 812 | % use fixed number for RMSE computing 813 | n_rmse = n_times*rec_cg(num_fc).category; 814 | tmpcg_rmse = zeros(nbands-1,1); 815 | % better days counting for RMSE calculating 816 | % relative days distance 817 | d_rt = clrx(IDsOld) - clrx(i+conse); 818 | d_yr = abs(round(d_rt/num_yrs)*num_yrs-d_rt); 819 | 820 | [~,sorted_indx] = sort(d_yr); 821 | sorted_indx = sorted_indx(1:n_rmse); 822 | 823 | for i_B = B_detect 824 | % temporally changing RMSE 825 | tmpcg_rmse(i_B) = norm(rec_v_dif(IDsOld(sorted_indx)-IDsOld(1)+1,i_B))/... 826 | sqrt(n_rmse-rec_cg(num_fc).category); 827 | end 828 | 829 | % move the ith col to i-1th col 830 | v_dif(1:conse-1,:) = v_dif(2:conse,:); 831 | % only compute the difference of last consecutive obs 832 | v_dif(conse,:) = 0; 833 | % move the ith col to i-1th col 834 | v_dif_mag(1:conse-1,:) = v_dif_mag(2:conse,:); 835 | % record the magnitude of change of the last conse obs 836 | v_dif_mag(conse,:) = 0; 837 | % move the ith col to i-1th col 838 | vec_mag(1:conse-1) = vec_mag(2:conse); 839 | % change vector magnitude 840 | vec_mag(conse) = 0; 841 | 842 | for i_B = 1:nbands-1 843 | % absolute difference 844 | v_dif_mag(conse,i_B) = clry(i+conse,i_B)-autoTSPred(clrx(i+conse),fit_cft(:,i_B)); 845 | % normalized to z-scores 846 | if sum(i_B == B_detect) 847 | % minimum rmse 848 | mini_rmse = max(adj_rmse(i_B),tmpcg_rmse(i_B)); 849 | 850 | % z-scores 851 | v_dif(conse,i_B) = v_dif_mag(conse,i_B)/mini_rmse; 852 | end 853 | end 854 | vec_mag(conse) = norm(v_dif(end,B_detect))^2; 855 | end 856 | 857 | % sign of change vector 858 | % vec_sign = abs(sum(sign(v_dif(:,B_detect)),1)); 859 | max_angl = mean(angl(v_dif(:,B_detect))); 860 | 861 | % change detection 862 | if min(vec_mag) > T_cg && max_angl < nsign% change detected 863 | % record break time 864 | rec_cg(num_fc).t_break = clrx(i+1); 865 | % record change probability 866 | rec_cg(num_fc).change_prob = 1; 867 | % record change magnitude 868 | rec_cg(num_fc).magnitude = median(v_dif_mag,1); 869 | 870 | % identified and move on for the next functional curve 871 | num_fc = num_fc + 1; 872 | % start from i+1 for the next functional curve 873 | i_start = i + 1; 874 | % start training again 875 | BL_train = 0; 876 | 877 | elseif vec_mag(1) > Tmax_cg % false change 878 | % remove noise 879 | clrx(i+1) = []; 880 | clry(i+1,:) = []; 881 | i=i-1; % stay & check again after noise removal 882 | end 883 | end % end of continuous monitoring 884 | end % end of checking basic requrirements 885 | 886 | % move forward to the i+1th clear observation 887 | i=i+1; 888 | end % end of while iterative 889 | 890 | % Two ways for processing the end of the time series 891 | if BL_train == 1 892 | % 1) if no break find at the end of the time series 893 | % define probability of change based on conse 894 | for i_conse = conse:-1:1 895 | % sign of change vector 896 | % vec_sign = abs(sum(sign(v_dif(i_conse:conse,B_detect)),1)); 897 | max_angl = mean(angl(v_dif(i_conse:conse,B_detect))); 898 | 899 | if vec_mag(i_conse) <= T_cg || max_angl >= nsign 900 | % the last stable id 901 | id_last = i_conse; 902 | break; 903 | end 904 | end 905 | 906 | % update change probability 907 | rec_cg(num_fc).change_prob = (conse-id_last)/conse; 908 | % update end time of the curve 909 | rec_cg(num_fc).t_end=clrx(end-conse+id_last); 910 | 911 | if conse > id_last % > 1 912 | % update time of the probable change 913 | rec_cg(num_fc).t_break = clrx(end-conse+id_last+1); 914 | % update magnitude of change 915 | rec_cg(num_fc).magnitude = median(v_dif_mag(id_last+1:conse,:),1); 916 | end 917 | 918 | elseif BL_train == 0 919 | % 2) if break find close to the end of the time series 920 | % Use [conse,min_num_c*n_times+conse) to fit curve 921 | 922 | if num_fc == rec_fc 923 | % first curve 924 | i_start = 1; 925 | else 926 | i_start = find(clrx >= rec_cg(num_fc-1).t_break); 927 | i_start = i_start(1); 928 | end 929 | 930 | % Tmask 931 | if length(clrx(i_start:end)) > conse 932 | blIDs = autoTmask(clrx(i_start:end),clry(i_start:end,[num_B1,num_B2]),... 933 | (clrx(end)-clrx(i_start))/num_yrs,adj_rmse(num_B1),adj_rmse(num_B2),T_const); 934 | 935 | % update i_span after noise removal 936 | i_span = sum(~blIDs); %#ok 937 | 938 | IDs = i_start:length(clrx); % all IDs 939 | rmIDs = IDs(blIDs(1:end-conse) == 1); % IDs to be removed 940 | 941 | % remove noise pixels between i_start & i 942 | clrx(rmIDs) = []; 943 | clry(rmIDs,:) = []; 944 | end 945 | 946 | % enough data 947 | if length(clrx(i_start:end)) >= conse 948 | % defining computed variables 949 | fit_cft = zeros(max_num_c,nbands-1); 950 | % rmse for each band 951 | rmse = zeros(nbands-1,1); 952 | % end of fit qa = 20 953 | qa = 20; 954 | 955 | for i_B = 1:nbands-1 956 | [fit_cft(:,i_B),rmse(i_B)] = ... 957 | autoTSFit(clrx(i_start:end),clry(i_start:end,i_B),min_num_c); 958 | end 959 | 960 | % record time of curve start 961 | rec_cg(num_fc).t_start = clrx(i_start); 962 | % record time of curve end 963 | rec_cg(num_fc).t_end=clrx(end); 964 | % record break time 965 | rec_cg(num_fc).t_break = 0; 966 | % record postion of the pixel 967 | rec_cg(num_fc).pos = (nrows-1)*ncols+i_ids; 968 | % record fitted coefficients 969 | rec_cg(num_fc).coefs = fit_cft; 970 | % record rmse of the pixel 971 | rec_cg(num_fc).rmse = rmse; 972 | % record change probability 973 | rec_cg(num_fc).change_prob = 0; 974 | % record number of observations 975 | rec_cg(num_fc).num_obs = length(clrx(i_start:end)); 976 | % record fit category 977 | rec_cg(num_fc).category = qa + min_num_c; 978 | % record change magnitude 979 | rec_cg(num_fc).magnitude = zeros(1,nbands-1); 980 | end 981 | end 982 | end % end of if sum(idgood) statement 983 | end % end of for i_ids loop 984 | % save([dir_l,'/',n_rst,'/','record_change',num2str(nrows)],'rec_cg'); 985 | % when saving matlab file, the computer is shut off. The matlab file may be broken. 986 | file_name_mat = ['record_change',num2str(nrows),'.mat']; 987 | file_name_mat_part = [file_name_mat, '.part']; 988 | % save it first as name with part 989 | save([dir_l,'/',n_rst,'/', file_name_mat_part],'rec_cg'); 990 | % rename it 991 | movefile([dir_l,'/',n_rst,'/',file_name_mat_part], [dir_l,'/',n_rst,'/',file_name_mat]); 992 | 993 | end % end of function 994 | 995 | % function to caculate included angle between ajacent pair of change vectors 996 | function y = angl(v_dif) 997 | 998 | [row,~] = size(v_dif); 999 | y = zeros(row-1,1); 1000 | 1001 | if row > 1 1002 | for i = 1:row-1 1003 | a = v_dif(i,:); 1004 | b = v_dif(i+1,:); 1005 | % y measures the opposite of cos(angle) 1006 | y(i) = acos(a*b'/(norm(a)*norm(b))); 1007 | end 1008 | else 1009 | y = 0; 1010 | end 1011 | 1012 | % convert angle from radiance to degree 1013 | y = y*180/pi; 1014 | 1015 | end 1016 | -------------------------------------------------------------------------------- /autoPara.m: -------------------------------------------------------------------------------- 1 | function [nrows,ncols,nbands,jiUL,resolu,zc,num_imgs] = autoPara(imf) 2 | % Matlab code for read paramters of files automatically 3 | % Inputs: 4 | % imf: folder names from dir 5 | 6 | % number of images 7 | num_imgs = size(imf,1); 8 | % filter for Landsat folders 9 | imf = regexpi({imf.name}, 'L(T5|T4|E7|C8|ND)(\w*)', 'match'); 10 | imf = [imf{:}]; 11 | imf = vertcat(imf{:}); 12 | % name of the first stacked image 13 | filename = dir([imf(1,:),'/','L*stack']); 14 | % read in dimension and zone number of the data 15 | [jiDim,jiUL,resolu,zc,nbands] = envihdrread([imf(1,:),'/',filename.name]); 16 | % number of nrows processed 17 | nrows = jiDim(2); 18 | % number of pixels procesed per line 19 | ncols = jiDim(1); 20 | end -------------------------------------------------------------------------------- /autoRobustFit.m: -------------------------------------------------------------------------------- 1 | function fit_cft=autoRobustFit(x,y,Nyr) 2 | % INPUTS: 3 | % x - Julian day [1; 2; 3]; 4 | % y - predicted reflectances [0.1; 0.2; 0.3]; 5 | % w - cofficient related with T=365d; 6 | % yr - number of years ceil([end(day)-start(day)]/365); 7 | % outfitx - fitted x values; 8 | 9 | % OUTPUTS: 10 | % outfity - fitted y values; 11 | % fit_cft - fitted coefficients; 12 | % General fitting model: f(x) = a0 + a1*cos(x*w/N) + b1*sin(x*w/N) + ... 13 | % a2*cos(x*w/1) + b2*sin(x*w/1) 14 | 15 | % check num of clr obs before using 16 | num_x = length(x); % number of clear pixels 17 | 18 | % build X 19 | X = zeros(num_x,4); 20 | % annual cycle 21 | w = 2*pi/365.25; % annual cycle 22 | X(:,1) = cos(w*x); 23 | X(:,2) = sin(w*x); 24 | % Nyr year cycle 25 | w = w/Nyr; 26 | X(:,3) = cos(w*x); 27 | X(:,4) = sin(w*x); 28 | 29 | % Robust fitting 30 | fit_cft = robustfit_cor(X,y); 31 | end 32 | -------------------------------------------------------------------------------- /autoTSFit.m: -------------------------------------------------------------------------------- 1 | function [fit_cft,rmse,v_dif]=autoTSFit(x,y,df) 2 | % Revisions: 3 | % v1.0 Using lasso for timeseries modeling (01/27/2013) 4 | % Auto Trends and Seasonal Fit between breaks 5 | % INPUTS: 6 | % x - Julian day [1; 2; 3]; 7 | % y - predicted reflectances [0.1; 0.2; 0.3]; 8 | % df - degree of freedom (num_c) 9 | % 10 | % OUTPUTS: 11 | % fit_cft - fitted coefficients; 12 | % General model TSModel: 13 | % f1(x) = a0 + b0*x (df = 2) 14 | % f2(x) = f1(x) + a1*cos(x*w) + b1*sin(x*w) (df = 4) 15 | % f3(x) = f2(x) + a2*cos(x*2w) + b2*sin(x*2w) (df = 6) 16 | % f4(x) = f3(x) + a3*cos(x*3w) + b3*sin(x*3w) (df = 8) 17 | 18 | n=length(x); % number of clear pixels 19 | % num_yrs = 365.25; % number of days per year 20 | w=2*pi/365.25; % num_yrs; % anual cycle 21 | % fit coefs 22 | fit_cft = zeros(8,1); 23 | 24 | % global lamda_lasso 25 | 26 | %% LASSO Fit 27 | % build X 28 | X = zeros(n,df-1); 29 | X(:,1) = x; 30 | 31 | if df >= 4 32 | X(:,2)=cos(w*x); 33 | X(:,3)=sin(w*x); 34 | end 35 | 36 | if df >= 6 37 | X(:,4)=cos(2*w*x); 38 | X(:,5)=sin(2*w*x); 39 | end 40 | 41 | if df >= 8 42 | X(:,6)=cos(3*w*x); 43 | X(:,7)=sin(3*w*x); 44 | end 45 | 46 | % lasso fit with lambda = 20 47 | fit = glmnet_fast(X,y,glmnetSetL(20)); 48 | 49 | % curr_cft=[fit.a0;fit.beta]; 50 | fit_cft(1:df) = [fit.a0;fit.beta]; % curr_cft; 51 | 52 | % %% OLS Fit 53 | % 54 | % % build X 55 | % X = zeros(n,df); 56 | % X(:,1) = ones(n,1); 57 | % X(:,2) = x; 58 | % 59 | % if df >= 4 60 | % X(:,3)=cos(w*x); 61 | % X(:,4)=sin(w*x); 62 | % end 63 | % 64 | % if df >= 6 65 | % X(:,5)=cos(2*w*x); 66 | % X(:,6)=sin(2*w*x); 67 | % end 68 | % 69 | % if df >= 8 70 | % X(:,7)=cos(3*w*x); 71 | % X(:,8)=sin(3*w*x); 72 | % end 73 | % 74 | % % curr_cft=[fit.a0;fit.beta]; 75 | % fit_cft(1:df) = X\y; % curr_cft; 76 | 77 | yhat=autoTSPred(x,fit_cft); 78 | % rmse=median(abs(y-yhat)); 79 | v_dif = y-yhat; 80 | rmse=norm(v_dif)/sqrt(n-df); 81 | % f(x) = a0 + b0*x + a1*cos(x*w) + b1*sin(x*w) (df = 4) 82 | end 83 | -------------------------------------------------------------------------------- /autoTSPred.m: -------------------------------------------------------------------------------- 1 | function outfity=autoTSPred(outfitx,fit_cft) 2 | % Auto Trends and Seasonal Predict 3 | % INPUTS: 4 | % outfitx - Julian day [1; 2; 3]; 5 | % fit_cft - fitted coefficients; 6 | % OUTPUTS: 7 | % outfity - predicted reflectances [0.1; 0.2; 0.3]; 8 | % General model TSModel: 9 | % f(x) = a0 + b0*x + a1*cos(x*w) + b1*sin(x*w) 10 | 11 | % num_yrs = 365.25; % number of days per year 12 | % w=2*pi/num_yrs; % anual cycle 13 | w = 2*pi/365.25; 14 | 15 | outfity=[ones(size(outfitx)),outfitx,...% overall ref + trending 16 | cos(w*outfitx),sin(w*outfitx),...% add seasonality 17 | cos(2*w*outfitx),sin(2*w*outfitx),...% add bimodal seasonality 18 | cos(3*w*outfitx),sin(3*w*outfitx)]*fit_cft; % add trimodal seasonality 19 | end 20 | -------------------------------------------------------------------------------- /autoTmask.m: -------------------------------------------------------------------------------- 1 | % function mask = autoTmask(j_date,b_ref,yr,T_const) 2 | function mask = autoTmask(j_date,b_ref,yr,var1,var2,T_const) 3 | %% Multitepmoral cloud, cloud shadow, & snow masks (global version) 4 | % read in data with 3 more consecutive clear obs & correct data 5 | % Inputs: 6 | % j_date: Julian date 7 | % b_ref: Band 2 & 4 reflectance 8 | % yr: number of years to mask 9 | % 10 | % Outputs: 11 | % y: corrected Band reflectance 12 | 13 | yr = ceil(yr); 14 | % yr = yr*2; 15 | w = 2*pi/365.25; % anual cycle 16 | 17 | TOA_B1_cft = autoRobustFit(j_date,b_ref(:,1),yr); % Band 2 18 | TOA_B2_cft = autoRobustFit(j_date,b_ref(:,2),yr); % Band 5 19 | 20 | % predict Band 1 ref 21 | pred_B1 = TOA_B1_cft(1)+TOA_B1_cft(2)*cos(j_date*w)+TOA_B1_cft(3)*sin(j_date*w)... 22 | +TOA_B1_cft(4)*cos(j_date*w/yr)+TOA_B1_cft(5)*sin(j_date*w/yr); 23 | 24 | % predict Band 2 ref 25 | pred_B2 = TOA_B2_cft(1)+TOA_B2_cft(2)*cos(j_date*w)+TOA_B2_cft(3)*sin(j_date*w)... 26 | +TOA_B2_cft(4)*cos(j_date*w/yr)+TOA_B2_cft(5)*sin(j_date*w/yr); 27 | 28 | % true or false ids 29 | DeltaB1 = b_ref(:,1)-pred_B1; % band 2 30 | DeltaB2 = b_ref(:,2)-pred_B2; % band 5 31 | 32 | mask = zeros(length(j_date),1); 33 | % mask(DeltaB1 > T_const*std(DeltaB1) | DeltaB2 < - T_const*std(DeltaB2)) = 1; 34 | mask(abs(DeltaB1) > T_const*var1 | abs(DeltaB2) > T_const*var2) = 1; 35 | % mask(abs(DeltaB1) > T_const | abs(DeltaB2) > T_const) = 1; 36 | end 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /cold_animation.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GERSL/COLD/043cbc1315a98c65444660e5317ccfeb20ffac18/cold_animation.gif -------------------------------------------------------------------------------- /envihdrread.m: -------------------------------------------------------------------------------- 1 | function [jiDim,jiUL,resolu,ZC,bands]=envihdrread(filename) 2 | % any ENVI file uint 8, int16, and uint16 3 | % [jiDim,jiUL,resolu,ZC]=envihdrread(filename) 4 | % Input file directory 5 | % Output 1) im = data 6 | % Output 2) jiDim dimension cols and rows 7 | % Output 3)jiUL UpperLeft coner of the pixel (x,y) 8 | % Output 4) resolution [x,y] 9 | % Output 5) ZC zone code 10 | 11 | filename_HDR=[filename,'.HDR']; 12 | filename_hdr=[filename,'.hdr']; 13 | 14 | fid_in1=fopen(filename_hdr,'r'); 15 | fid_in2=fopen(filename_HDR,'r'); 16 | 17 | if fid_in1~=-1 18 | fid_in=fid_in1; 19 | elseif fid_in2~=-1 20 | fid_in=fid_in2; 21 | else 22 | fprintf('Wrong ENVI header file!\n'); 23 | fprintf('%s\n',filename); % show envi hdr file 24 | return; 25 | end 26 | 27 | geo_char=fscanf(fid_in,'%c',inf); 28 | fclose(fid_in); 29 | 30 | geo_char=geo_char'; 31 | geo_str=strread(geo_char,'%s'); 32 | 33 | 34 | indx_samples = strmatch('samples',geo_str)+2; 35 | indx_lines = strmatch('lines',geo_str)+2; 36 | indx_bands = strmatch('bands',geo_str)+2; 37 | indx_datatype = strmatch('data',geo_str)+3; 38 | indx_interleave = strmatch('interleave',geo_str)+2; 39 | indx_xUL = strmatch('map',geo_str)+6; 40 | indx_yUL = strmatch('map',geo_str)+7; 41 | indx_xreso = strmatch('map',geo_str)+8; 42 | indx_yreso = strmatch('map',geo_str)+9; 43 | indx_zc = strmatch('map',geo_str)+10; 44 | 45 | % read input image hdr 46 | cols = str2double(geo_str(indx_samples)); 47 | rows = str2double(geo_str(indx_lines)); 48 | jiDim = [cols,rows]; 49 | 50 | bands = str2double(geo_str(indx_bands)); 51 | datatype = str2double(geo_str(indx_datatype)); 52 | interleave = char(geo_str(indx_interleave)); 53 | jiUL(1)=str2double(geo_str(indx_xUL)); 54 | jiUL(2)=str2double(geo_str(indx_yUL)); 55 | resolu(1)=str2double(geo_str(indx_xreso)); 56 | resolu(2)=str2double(geo_str(indx_yreso)); 57 | ZC=str2double(geo_str(indx_zc)); 58 | 59 | if datatype == 1 60 | in_type = 'uint8'; 61 | elseif datatype == 2 62 | in_type = 'int16'; 63 | elseif datatype == 3 64 | in_type = 'int32'; 65 | elseif datatype == 4 66 | in_type = 'single'; 67 | elseif datatype == 12 68 | in_type = 'uint16'; 69 | else 70 | error('Invalid read data type!'); 71 | end 72 | 73 | % % Define the data set. 74 | % % Read every other band of the data using the Band-Sequential format. 75 | % im = multibandread(filename, [rows cols bands], ... 76 | % [in_type,'=>',in_type], 0, interleave, 'ieee-le'); 77 | end 78 | 79 | -------------------------------------------------------------------------------- /glmnetMex.F: -------------------------------------------------------------------------------- 1 | #include "fintrf.h" 2 | C glmnetMex.F 3 | C 4 | C Lasso and elastic-net regularized generalized linear models 5 | 6 | C [a0,ca,ia,nin,rsq,alm,nlp,jerr] = ... 7 | C glmnetMex(parm,x,y,jd,vp,ne,nx,nlam,flmin,ulam,thr,isd,w,ka) 8 | C [a0,ca,ia,nin,dev,alm,nlp,jerr] = ... 9 | C glmnetMex(parm,x,y,jd,vp,ne,nx,nlam,flmin,ulam,thr,isd,nc,maxit,kopt) 10 | C 11 | C Extremely efficient procedures for fitting the entire lasso or 12 | C elastic-net regularization path for linear regression, logistic and 13 | C multinomial regression models. The algorithm uses cyclical coordinate 14 | C descent in a pathwise as described in the paper on the maintainer's 15 | C website. 16 | C 17 | C NOTES: This is a MEX-file wrapper of GLMnet.f for MATLAB. Should be called 18 | C only by glmnet.m. For details about input and output arguments, see 19 | C GLMnet.f. 20 | C 21 | C LICENSE: GPL-2 22 | C 23 | C DATE: 13 Jul 2009 24 | C 25 | C AUTHORS: 26 | C Algorithm designed by Jerome Friedman, Trevor Hastie and Rob Tibshirani 27 | C Fortran code written by Jerome Friedman 28 | C MATLAB wrapper written and maintained by Hui Jiang, jiangh@stanford.edu 29 | C Department of Statistics, Stanford University, Stanford, California, USA. 30 | C 31 | C REFERENCES: 32 | C Friedman, J., Hastie, T. and Tibshirani, R. (2009) 33 | C Regularization Paths for Generalized Linear Models via Coordinate Descent. 34 | C Journal of Statistical Software, 33(1), 2010 35 | C 36 | C EXAMPLE: 37 | C parm = 1.0; 38 | C x = [1 1; 2 2; 3 3]; 39 | C y = [1 3 2]'; 40 | C jd = 0; 41 | C vp = [1 1]'; 42 | C ne = 3; 43 | C nx = 2; 44 | C nlam = 100; 45 | C flmin = 0.0001; 46 | C ulam = 0; 47 | C thr = 1.0e-4; 48 | C isd = 1; 49 | C w = [1 1 1]'; 50 | C ka = 2; 51 | C [a0,ca,ia,nin,rsq,alm,nlp,jerr] = glmnetMex(parm,x,y,jd,vp,ne,nx,nlam,flmin,ulam,thr,isd,w,ka) 52 | C 53 | C DEVELOPMENT: 13 Jul 2009: Original version of glmnetMex.f written. 54 | C 55 | C----------------------------------------------------------------------- 56 | 57 | subroutine mexFunction(nlhs, plhs, nrhs, prhs) 58 | C----------------------------------------------------------------------- 59 | C (pointer) Replace integer by integer*8 on the DEC Alpha 60 | C 64-bit platform 61 | 62 | mwpointer plhs(*), prhs(*) 63 | mwpointer mxCreateDoubleMatrix, mxGetPr 64 | integer nlhs, nrhs 65 | mwsize mxGetM, mxGetN 66 | integer mxIsNumeric 67 | C----------------------------------------------------------------------- 68 | 69 | C Input 70 | real parm,flmin,thr 71 | integer ka,no,ni,nc,ne,nx,nlam,isd,maxit,kopt 72 | real, dimension (:), allocatable :: x,y,w,vp,ulam 73 | integer, dimension (:), allocatable :: ix,jx,jd 74 | 75 | C Output 76 | integer lmu,nlp,jerr 77 | real, dimension (:), allocatable :: a0,ca,alm,dev,rsq 78 | integer, dimension (:), allocatable :: ia,nin 79 | 80 | C Temporary 81 | mwpointer temp_pr 82 | mwsize temp_m, temp_n 83 | integer task 84 | 85 | C Check for proper number of arguments. 86 | if(nrhs .eq. 14 .and. nlhs .eq. 8) then 87 | task = 1; 88 | elseif(nrhs .eq. 15 .and. nlhs .eq. 8) then 89 | task = 2; 90 | else 91 | call mexErrMsgTxt('Incorrect number of arguments.') 92 | endif 93 | 94 | C Get input 95 | 96 | temp_pr = mxGetPr(prhs(1)) 97 | call getreal(temp_pr,parm,1) 98 | 99 | temp_pr = mxGetPr(prhs(2)) 100 | temp_m = mxGetM(prhs(2)) 101 | no = temp_m 102 | temp_n = mxGetN(prhs(2)) 103 | ni = temp_n 104 | allocate(x(1:no*ni)) 105 | call getreal(temp_pr,x,no*ni) 106 | 107 | temp_pr = mxGetPr(prhs(4)) 108 | temp_m = mxGetM(prhs(4)) 109 | temp_n = mxGetN(prhs(4)) 110 | allocate(jd(temp_m*temp_n)) 111 | call getinteger(temp_pr,jd,temp_m*temp_n) 112 | 113 | temp_pr = mxGetPr(prhs(5)) 114 | allocate(vp(1:ni)) 115 | call getreal(temp_pr,vp,ni) 116 | 117 | temp_pr = mxGetPr(prhs(6)) 118 | call getinteger(temp_pr,ne,1) 119 | 120 | temp_pr = mxGetPr(prhs(7)) 121 | call getinteger(temp_pr,nx,1) 122 | 123 | temp_pr = mxGetPr(prhs(8)) 124 | call getinteger(temp_pr,nlam,1) 125 | 126 | temp_pr = mxGetPr(prhs(9)) 127 | call getreal(temp_pr,flmin,1) 128 | 129 | temp_pr = mxGetPr(prhs(10)) 130 | temp_m = mxGetM(prhs(10)) 131 | temp_n = mxGetN(prhs(10)) 132 | allocate(ulam(1:temp_m * temp_n)) 133 | call getreal(temp_pr,ulam,temp_m * temp_n) 134 | 135 | temp_pr = mxGetPr(prhs(11)) 136 | call getreal(temp_pr,thr,1) 137 | 138 | temp_pr = mxGetPr(prhs(12)) 139 | call getinteger(temp_pr,isd,1) 140 | 141 | if (task .eq. 1) then 142 | temp_pr = mxGetPr(prhs(3)) 143 | allocate(y(1:no)) 144 | call getreal(temp_pr,y,no) 145 | 146 | temp_pr = mxGetPr(prhs(13)) 147 | allocate(w(1:no)) 148 | call getreal(temp_pr,w,no) 149 | 150 | temp_pr = mxGetPr(prhs(14)) 151 | call getinteger(temp_pr,ka,1) 152 | elseif (task .eq. 2) then 153 | temp_pr = mxGetPr(prhs(13)) 154 | call getinteger(temp_pr,nc,1) 155 | 156 | temp_pr = mxGetPr(prhs(14)) 157 | call getinteger(temp_pr,maxit,1) 158 | 159 | temp_pr = mxGetPr(prhs(15)) 160 | call getinteger(temp_pr,kopt,1) 161 | 162 | temp_pr = mxGetPr(prhs(3)) 163 | allocate(y(1:no*(max(2,nc)))) 164 | call getreal(temp_pr,y,no*(max(2,nc))) 165 | endif 166 | 167 | C Allocate memory for output 168 | allocate(ia(1:nx)) 169 | call zerointeger(ia,nx) 170 | allocate(nin(1:nlam)) 171 | call zerointeger(nin,nlam) 172 | allocate(alm(1:nlam)) 173 | call zeroreal(alm,nlam) 174 | if (task .eq. 1) then 175 | allocate(a0(1:nlam)) 176 | call zeroreal(a0,nlam) 177 | 178 | allocate(ca(1:nx*nlam)) 179 | call zeroreal(ca,nx*nlam) 180 | 181 | allocate(rsq(1:nlam)) 182 | call zeroreal(rsq,nlam) 183 | elseif (task .eq. 2) then 184 | allocate(a0(1:nc*nlam)) 185 | call zeroreal(a0,nc*nlam) 186 | 187 | allocate(ca(1:nx*nc*nlam)) 188 | call zeroreal(ca,nx*nc*nlam) 189 | 190 | allocate(dev(1:nlam)) 191 | call zeroreal(dev,nlam) 192 | endif 193 | 194 | C Call glmnet 195 | lmu = 0 196 | nlp = 0 197 | jerr = 0 198 | if (task .eq. 1) then 199 | call elnet(ka,parm,no,ni,x,y,w,jd,vp,ne,nx,nlam 200 | *,flmin,ulam,thr,isd,lmu,a0,ca,ia,nin,rsq,alm,nlp,jerr) 201 | elseif (task .eq. 2) then 202 | call lognet(parm,no,ni,nc,x,y,jd,vp,ne,nx,nlam,flmin 203 | *,ulam,thr,isd,maxit,kopt,lmu,a0,ca,ia,nin,dev,alm,nlp,jerr) 204 | endif 205 | 206 | C Prepare output 207 | plhs(3) = mxCreateDoubleMatrix(nx,1,0) 208 | temp_pr = mxGetPr(plhs(3)) 209 | call putinteger(ia,temp_pr,nx) 210 | 211 | plhs(4) = mxCreateDoubleMatrix(lmu,1,0) 212 | temp_pr = mxGetPr(plhs(4)) 213 | call putinteger(nin,temp_pr,lmu) 214 | 215 | plhs(6) = mxCreateDoubleMatrix(lmu,1,0) 216 | temp_pr = mxGetPr(plhs(6)) 217 | call putreal(alm,temp_pr,lmu) 218 | 219 | plhs(7) = mxCreateDoubleMatrix(1,1,0) 220 | temp_pr = mxGetPr(plhs(7)) 221 | call putinteger(nlp,temp_pr,1) 222 | 223 | plhs(8) = mxCreateDoubleMatrix(1,1,0) 224 | temp_pr = mxGetPr(plhs(8)) 225 | call putinteger(jerr,temp_pr,1) 226 | 227 | if (task .eq. 1) then 228 | plhs(1) = mxCreateDoubleMatrix(lmu,1,0) 229 | temp_pr = mxGetPr(plhs(1)) 230 | call putreal(a0,temp_pr,lmu) 231 | 232 | plhs(2) = mxCreateDoubleMatrix(nx,lmu,0) 233 | temp_pr = mxGetPr(plhs(2)) 234 | call putreal(ca,temp_pr,nx*lmu) 235 | 236 | plhs(5) = mxCreateDoubleMatrix(lmu,1,0) 237 | temp_pr = mxGetPr(plhs(5)) 238 | call putreal(rsq,temp_pr,lmu) 239 | elseif (task .eq. 2) then 240 | plhs(1) = mxCreateDoubleMatrix(nc,lmu,0) 241 | temp_pr = mxGetPr(plhs(1)) 242 | call putreal(a0,temp_pr,nc*lmu) 243 | 244 | plhs(2) = mxCreateDoubleMatrix(nx*nc,lmu,0) 245 | temp_pr = mxGetPr(plhs(2)) 246 | call putreal(ca,temp_pr,nx*nc*lmu) 247 | 248 | plhs(5) = mxCreateDoubleMatrix(lmu,1,0) 249 | temp_pr = mxGetPr(plhs(5)) 250 | call putreal(dev,temp_pr,lmu) 251 | endif 252 | 253 | C Deallocate memory 254 | deallocate(x) 255 | deallocate(y) 256 | deallocate(jd) 257 | deallocate(vp) 258 | deallocate(ulam) 259 | deallocate(a0) 260 | deallocate(ca) 261 | deallocate(ia) 262 | deallocate(nin) 263 | deallocate(alm) 264 | C For logistic elastic net 265 | if (task .eq. 1) then 266 | deallocate(w) 267 | deallocate(rsq) 268 | elseif (task .eq. 2) then 269 | deallocate(dev) 270 | endif 271 | 272 | return 273 | end 274 | 275 | C End of subroutine mexFunction 276 | 277 | subroutine real8toreal(x, y, size) 278 | integer size 279 | real*8 x(size) 280 | real y(size) 281 | do 10 i=1,size 282 | y(i)= x(i) 283 | 10 continue 284 | return 285 | end 286 | 287 | subroutine realtoreal8(x, y, size) 288 | integer size 289 | real x(size) 290 | real*8 y(size) 291 | do 20 i=1,size 292 | y(i)= x(i) 293 | 20 continue 294 | return 295 | end 296 | 297 | subroutine real8tointeger(x, y, size) 298 | integer size 299 | real*8 x(size) 300 | integer y(size) 301 | do 30 i=1,size 302 | y(i)= x(i) 303 | 30 continue 304 | return 305 | end 306 | 307 | subroutine integertoreal8(x, y, size) 308 | integer size 309 | integer x(size) 310 | real*8 y(size) 311 | do 40 i=1,size 312 | y(i)= x(i) 313 | 40 continue 314 | return 315 | end 316 | 317 | subroutine getreal(pr,x,size) 318 | mwpointer pr 319 | integer size 320 | real x(size) 321 | real*8, dimension (:), allocatable :: temp 322 | allocate(temp(1:size)) 323 | call mxCopyPtrToReal8(pr,temp,size) 324 | call real8toreal(temp,x,size) 325 | deallocate(temp) 326 | return 327 | end 328 | 329 | subroutine getinteger(pr,x,size) 330 | mwpointer pr 331 | integer size 332 | integer x(size) 333 | real*8, dimension (:), allocatable :: temp 334 | allocate(temp(1:size)) 335 | call mxCopyPtrToReal8(pr,temp,size) 336 | call real8tointeger(temp,x,size) 337 | deallocate(temp) 338 | return 339 | end 340 | 341 | subroutine putreal(x,pr,size) 342 | mwpointer pr 343 | integer size 344 | real x(size) 345 | real*8, dimension (:), allocatable :: temp 346 | allocate(temp(1:size)) 347 | call realtoreal8(x,temp,size) 348 | call mxCopyReal8ToPtr(temp,pr,size) 349 | deallocate(temp) 350 | return 351 | end 352 | 353 | subroutine putinteger(x,pr,size) 354 | mwpointer pr 355 | integer size 356 | integer x(size) 357 | real*8, dimension (:), allocatable :: temp 358 | allocate(temp(1:size)) 359 | call integertoreal8(x,temp,size) 360 | call mxCopyReal8ToPtr(temp,pr,size) 361 | deallocate(temp) 362 | return 363 | end 364 | 365 | subroutine zeroreal(x,size) 366 | integer size 367 | real x(size) 368 | do 90 i=1,size 369 | x(i) = 0 370 | 90 continue 371 | return 372 | end 373 | 374 | subroutine zerointeger(x,size) 375 | integer size 376 | integer x(size) 377 | do 100 i=1,size 378 | x(i) = 0 379 | 100 continue 380 | return 381 | end 382 | -------------------------------------------------------------------------------- /glmnetMex.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GERSL/COLD/043cbc1315a98c65444660e5317ccfeb20ffac18/glmnetMex.dll -------------------------------------------------------------------------------- /glmnetMex.matlabR13.F: -------------------------------------------------------------------------------- 1 | C glmnetMex.F 2 | C 3 | C Lasso and elastic-net regularized generalized linear models 4 | 5 | C [a0,ca,ia,nin,rsq,alm,nlp,jerr] = ... 6 | C glmnetMex(parm,x,y,jd,vp,ne,nx,nlam,flmin,ulam,thr,isd,w,ka) 7 | C [a0,ca,ia,nin,dev,alm,nlp,jerr] = ... 8 | C glmnetMex(parm,x,y,jd,vp,ne,nx,nlam,flmin,ulam,thr,isd,nc,maxit,kopt) 9 | C 10 | C Extremely efficient procedures for fitting the entire lasso or 11 | C elastic-net regularization path for linear regression, logistic and 12 | C multinomial regression models. The algorithm uses cyclical coordinate 13 | C descent in a pathwise as described in the paper on the maintainer's 14 | C website. 15 | C 16 | C NOTES: This is a MEX-file wrapper of GLMnet.f for MATLAB. Should be called 17 | C only by glmnet.m. For details about input and output arguments, see 18 | C GLMnet.f. 19 | C 20 | C LICENSE: GPL-2 21 | C 22 | C DATE: 13 Jul 2009 23 | C 24 | C AUTHORS: 25 | C Algorithm designed by Jerome Friedman, Trevor Hastie and Rob Tibshirani 26 | C Fortran code written by Jerome Friedman 27 | C MATLAB wrapper written and maintained by Hui Jiang, jiangh@stanford.edu 28 | C Department of Statistics, Stanford University, Stanford, California, USA. 29 | C 30 | C REFERENCES: 31 | C Friedman, J., Hastie, T. and Tibshirani, R. (2009) 32 | C Regularization Paths for Generalized Linear Models via Coordinate Descent. 33 | C To appear, Journal of Statistical Software 34 | C 35 | C EXAMPLE: 36 | C parm = 1.0; 37 | C x = [1 1; 2 2; 3 3]; 38 | C y = [1 3 2]'; 39 | C jd = 0; 40 | C vp = [1 1]'; 41 | C ne = 3; 42 | C nx = 2; 43 | C nlam = 100; 44 | C flmin = 0.0001; 45 | C ulam = 0; 46 | C thr = 1.0e-4; 47 | C isd = 1; 48 | C w = [1 1 1]'; 49 | C ka = 2; 50 | C [a0,ca,ia,nin,rsq,alm,nlp,jerr] = glmnetMex(parm,x,y,jd,vp,ne,nx,nlam,flmin,ulam,thr,isd,w,ka) 51 | C 52 | C DEVELOPMENT: 13 Jul 2009: Original version of glmnetMex.f written. 53 | C 54 | C----------------------------------------------------------------------- 55 | 56 | subroutine mexFunction(nlhs, plhs, nrhs, prhs) 57 | C----------------------------------------------------------------------- 58 | C (pointer) Replace integer by integer*8 on the DEC Alpha 59 | C 64-bit platform 60 | 61 | integer plhs(*), prhs(*) 62 | integer nlhs, nrhs 63 | C----------------------------------------------------------------------- 64 | 65 | C Input 66 | real parm,flmin,thr 67 | integer ka,no,ni,nc,ne,nx,nlam,isd,maxit,kopt 68 | real, dimension (:), allocatable :: x,y,w,vp,ulam 69 | integer, dimension (:), allocatable :: ix,jx,jd 70 | 71 | C Output 72 | integer lmu,nlp,jerr 73 | real, dimension (:), allocatable :: a0,ca,alm,dev,rsq 74 | integer, dimension (:), allocatable :: ia,nin 75 | 76 | C Temporary 77 | integer temp_pr 78 | integer temp_m, temp_n 79 | integer task 80 | 81 | C Check for proper number of arguments. 82 | if(nrhs .eq. 14 .and. nlhs .eq. 8) then 83 | task = 1; 84 | elseif(nrhs .eq. 15 .and. nlhs .eq. 8) then 85 | task = 2; 86 | else 87 | call mexErrMsgTxt('Incorrect number of arguments.') 88 | endif 89 | 90 | C Get input 91 | 92 | temp_pr = mxGetPr(prhs(1)) 93 | call getreal(temp_pr,parm,1) 94 | 95 | temp_pr = mxGetPr(prhs(2)) 96 | no = mxGetM(prhs(2)) 97 | ni = mxGetN(prhs(2)) 98 | allocate(x(1:no*ni)) 99 | call getreal(temp_pr,x,no*ni) 100 | 101 | temp_pr = mxGetPr(prhs(4)) 102 | temp_m = mxGetM(prhs(4)) 103 | temp_n = mxGetN(prhs(4)) 104 | allocate(jd(temp_m*temp_n)) 105 | call getinteger(temp_pr,jd,temp_m*temp_n) 106 | 107 | temp_pr = mxGetPr(prhs(5)) 108 | allocate(vp(1:ni)) 109 | call getreal(temp_pr,vp,ni) 110 | 111 | temp_pr = mxGetPr(prhs(6)) 112 | call getinteger(temp_pr,ne,1) 113 | 114 | temp_pr = mxGetPr(prhs(7)) 115 | call getinteger(temp_pr,nx,1) 116 | 117 | temp_pr = mxGetPr(prhs(8)) 118 | call getinteger(temp_pr,nlam,1) 119 | 120 | temp_pr = mxGetPr(prhs(9)) 121 | call getreal(temp_pr,flmin,1) 122 | 123 | temp_pr = mxGetPr(prhs(10)) 124 | temp_m = mxGetM(prhs(10)) 125 | temp_n = mxGetN(prhs(10)) 126 | allocate(ulam(1:temp_m * temp_n)) 127 | call getreal(temp_pr,ulam,temp_m * temp_n) 128 | 129 | temp_pr = mxGetPr(prhs(11)) 130 | call getreal(temp_pr,thr,1) 131 | 132 | temp_pr = mxGetPr(prhs(12)) 133 | call getinteger(temp_pr,isd,1) 134 | 135 | if (task .eq. 1) then 136 | temp_pr = mxGetPr(prhs(3)) 137 | allocate(y(1:no)) 138 | call getreal(temp_pr,y,no) 139 | 140 | temp_pr = mxGetPr(prhs(13)) 141 | allocate(w(1:no)) 142 | call getreal(temp_pr,w,no) 143 | 144 | temp_pr = mxGetPr(prhs(14)) 145 | call getinteger(temp_pr,ka,1) 146 | elseif (task .eq. 2) then 147 | temp_pr = mxGetPr(prhs(13)) 148 | call getinteger(temp_pr,nc,1) 149 | 150 | temp_pr = mxGetPr(prhs(14)) 151 | call getinteger(temp_pr,maxit,1) 152 | 153 | temp_pr = mxGetPr(prhs(15)) 154 | call getinteger(temp_pr,kopt,1) 155 | 156 | temp_pr = mxGetPr(prhs(3)) 157 | allocate(y(1:no*(max(2,nc)))) 158 | call getreal(temp_pr,y,no*(max(2,nc))) 159 | endif 160 | 161 | C Allocate memory for output 162 | allocate(ia(1:nx)) 163 | call zerointeger(ia,nx) 164 | allocate(nin(1:nlam)) 165 | call zerointeger(nin,nlam) 166 | allocate(alm(1:nlam)) 167 | call zeroreal(alm,nlam) 168 | if (task .eq. 1) then 169 | allocate(a0(1:nlam)) 170 | call zeroreal(a0,nlam) 171 | 172 | allocate(ca(1:nx*nlam)) 173 | call zeroreal(ca,nx*nlam) 174 | 175 | allocate(rsq(1:nlam)) 176 | call zeroreal(rsq,nlam) 177 | elseif (task .eq. 2) then 178 | allocate(a0(1:nc*nlam)) 179 | call zeroreal(a0,nc*nlam) 180 | 181 | allocate(ca(1:nx*nc*nlam)) 182 | call zeroreal(ca,nx*nc*nlam) 183 | 184 | allocate(dev(1:nlam)) 185 | call zeroreal(dev,nlam) 186 | endif 187 | 188 | C Call glmnet 189 | lmu = 0 190 | nlp = 0 191 | jerr = 0 192 | if (task .eq. 1) then 193 | call elnet(ka,parm,no,ni,x,y,w,jd,vp,ne,nx,nlam 194 | *,flmin,ulam,thr,isd,lmu,a0,ca,ia,nin,rsq,alm,nlp,jerr) 195 | elseif (task .eq. 2) then 196 | call lognet(parm,no,ni,nc,x,y,jd,vp,ne,nx,nlam,flmin 197 | *,ulam,thr,isd,maxit,kopt,lmu,a0,ca,ia,nin,dev,alm,nlp,jerr) 198 | endif 199 | 200 | C Prepare output 201 | plhs(3) = mxCreateDoubleMatrix(nx,1,0) 202 | temp_pr = mxGetPr(plhs(3)) 203 | call putinteger(ia,temp_pr,nx) 204 | 205 | plhs(4) = mxCreateDoubleMatrix(lmu,1,0) 206 | temp_pr = mxGetPr(plhs(4)) 207 | call putinteger(nin,temp_pr,lmu) 208 | 209 | plhs(6) = mxCreateDoubleMatrix(lmu,1,0) 210 | temp_pr = mxGetPr(plhs(6)) 211 | call putreal(alm,temp_pr,lmu) 212 | 213 | plhs(7) = mxCreateDoubleMatrix(1,1,0) 214 | temp_pr = mxGetPr(plhs(7)) 215 | call putinteger(nlp,temp_pr,1) 216 | 217 | plhs(8) = mxCreateDoubleMatrix(1,1,0) 218 | temp_pr = mxGetPr(plhs(8)) 219 | call putinteger(jerr,temp_pr,1) 220 | 221 | if (task .eq. 1) then 222 | plhs(1) = mxCreateDoubleMatrix(lmu,1,0) 223 | temp_pr = mxGetPr(plhs(1)) 224 | call putreal(a0,temp_pr,lmu) 225 | 226 | plhs(2) = mxCreateDoubleMatrix(nx,lmu,0) 227 | temp_pr = mxGetPr(plhs(2)) 228 | call putreal(ca,temp_pr,nx*lmu) 229 | 230 | plhs(5) = mxCreateDoubleMatrix(lmu,1,0) 231 | temp_pr = mxGetPr(plhs(5)) 232 | call putreal(rsq,temp_pr,lmu) 233 | elseif (task .eq. 2) then 234 | plhs(1) = mxCreateDoubleMatrix(nc,lmu,0) 235 | temp_pr = mxGetPr(plhs(1)) 236 | call putreal(a0,temp_pr,nc*lmu) 237 | 238 | plhs(2) = mxCreateDoubleMatrix(nx*nc,lmu,0) 239 | temp_pr = mxGetPr(plhs(2)) 240 | call putreal(ca,temp_pr,nx*nc*lmu) 241 | 242 | plhs(5) = mxCreateDoubleMatrix(lmu,1,0) 243 | temp_pr = mxGetPr(plhs(5)) 244 | call putreal(dev,temp_pr,lmu) 245 | endif 246 | 247 | C Deallocate memory 248 | deallocate(x) 249 | deallocate(y) 250 | deallocate(jd) 251 | deallocate(vp) 252 | deallocate(ulam) 253 | deallocate(a0) 254 | deallocate(ca) 255 | deallocate(ia) 256 | deallocate(nin) 257 | deallocate(alm) 258 | C For logistic elastic net 259 | if (task .eq. 1) then 260 | deallocate(w) 261 | deallocate(rsq) 262 | elseif (task .eq. 2) then 263 | deallocate(dev) 264 | endif 265 | 266 | return 267 | end 268 | 269 | C End of subroutine mexFunction 270 | 271 | subroutine real8toreal(x, y, size) 272 | integer size 273 | real*8 x(size) 274 | real y(size) 275 | do 10 i=1,size 276 | y(i)= x(i) 277 | 10 continue 278 | return 279 | end 280 | 281 | subroutine realtoreal8(x, y, size) 282 | integer size 283 | real x(size) 284 | real*8 y(size) 285 | do 20 i=1,size 286 | y(i)= x(i) 287 | 20 continue 288 | return 289 | end 290 | 291 | subroutine real8tointeger(x, y, size) 292 | integer size 293 | real*8 x(size) 294 | integer y(size) 295 | do 30 i=1,size 296 | y(i)= x(i) 297 | 30 continue 298 | return 299 | end 300 | 301 | subroutine integertoreal8(x, y, size) 302 | integer size 303 | integer x(size) 304 | real*8 y(size) 305 | do 40 i=1,size 306 | y(i)= x(i) 307 | 40 continue 308 | return 309 | end 310 | 311 | subroutine getreal(pr,x,size) 312 | integer pr,size 313 | real x(size) 314 | real*8, dimension (:), allocatable :: temp 315 | allocate(temp(1:size)) 316 | call mxCopyPtrToReal8(pr,temp,size) 317 | call real8toreal(temp,x,size) 318 | deallocate(temp) 319 | return 320 | end 321 | 322 | subroutine getinteger(pr,x,size) 323 | integer pr,size 324 | integer x(size) 325 | real*8, dimension (:), allocatable :: temp 326 | allocate(temp(1:size)) 327 | call mxCopyPtrToReal8(pr,temp,size) 328 | call real8tointeger(temp,x,size) 329 | deallocate(temp) 330 | return 331 | end 332 | 333 | subroutine putreal(x,pr,size) 334 | integer pr,size 335 | real x(size) 336 | real*8, dimension (:), allocatable :: temp 337 | allocate(temp(1:size)) 338 | call realtoreal8(x,temp,size) 339 | call mxCopyReal8ToPtr(temp,pr,size) 340 | deallocate(temp) 341 | return 342 | end 343 | 344 | subroutine putinteger(x,pr,size) 345 | integer pr,size 346 | integer x(size) 347 | real*8, dimension (:), allocatable :: temp 348 | allocate(temp(1:size)) 349 | call integertoreal8(x,temp,size) 350 | call mxCopyReal8ToPtr(temp,pr,size) 351 | deallocate(temp) 352 | return 353 | end 354 | 355 | subroutine zeroreal(x,size) 356 | integer size 357 | real x(size) 358 | do 90 i=1,size 359 | x(i) = 0 360 | 90 continue 361 | return 362 | end 363 | 364 | subroutine zerointeger(x,size) 365 | integer size 366 | integer x(size) 367 | do 100 i=1,size 368 | x(i) = 0 369 | 100 continue 370 | return 371 | end -------------------------------------------------------------------------------- /glmnetMex.mexa64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GERSL/COLD/043cbc1315a98c65444660e5317ccfeb20ffac18/glmnetMex.mexa64 -------------------------------------------------------------------------------- /glmnetMex.mexglx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GERSL/COLD/043cbc1315a98c65444660e5317ccfeb20ffac18/glmnetMex.mexglx -------------------------------------------------------------------------------- /glmnetMex.mexmaci: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GERSL/COLD/043cbc1315a98c65444660e5317ccfeb20ffac18/glmnetMex.mexmaci -------------------------------------------------------------------------------- /glmnetMex.mexmaci64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GERSL/COLD/043cbc1315a98c65444660e5317ccfeb20ffac18/glmnetMex.mexmaci64 -------------------------------------------------------------------------------- /glmnetMex.mexw32: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GERSL/COLD/043cbc1315a98c65444660e5317ccfeb20ffac18/glmnetMex.mexw32 -------------------------------------------------------------------------------- /glmnetMex.mexw64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GERSL/COLD/043cbc1315a98c65444660e5317ccfeb20ffac18/glmnetMex.mexw64 -------------------------------------------------------------------------------- /glmnetSetL.m: -------------------------------------------------------------------------------- 1 | 2 | function options = glmnetSetL(lambda) 3 | 4 | %-------------------------------------------------------------------------- 5 | % glmnetSet creates or alters an options structure for glmnet.m. 6 | %-------------------------------------------------------------------------- 7 | % options = glmnetSet; (with no input arguments) 8 | % creates a structure with all fields set to their default values. 9 | % Each field is an option (also called a parameter). 10 | % 11 | % glmnetSet (with no input or output arguments) 12 | % displays all options and their default values. 13 | % 14 | % options = glmnetSet(opts); 15 | % creates a structure with all fields set to their default values, 16 | % except valid fields in the structure "opts" replace the defaults. 17 | % 18 | % options.weights Observation weights. Can be total counts if responses 19 | % are proportion matrices. Default is 1 for each 20 | % observation. 21 | % options.alpha The elasticnet mixing parameter, with 0 < alpha <= 1. 22 | % The penalty is defined as 23 | % (1-alpha)/2(||beta||_2)^2+alpha||beta||_1. 24 | % Default is alpha = 1, which is the lasso penalty; 25 | % Currently alpha < 0.01 is not reliable, unless you 26 | % supply your own lambda sequence. 27 | % options.nlambda The number of lambda values - default is 100. 28 | % options.lambda_min Smallest value for lambda, as a fraction of 29 | % lambda_max, the (data derived) entry value (i.e., the 30 | % smallest value for which all coefficients are zero). 31 | % The default depends on the sample size nobs relative 32 | % to the number of variables nvars. If nobs > nvars, 33 | % the default is 0.0001, close to zero. If nobs < 34 | % nvars, the defaults is 0.05. A very small value of 35 | % lambda_min will lead to a saturated fit. This is 36 | % undefined for "binomial" and "multinomial" models, 37 | % and glmnet will exit gracefully when the percentage 38 | % deviance explained is almost 1. 39 | % options.lambda A user supplied lambda sequence. Typical usage is to 40 | % have the program compute its own lambda sequence 41 | % based on nlambda and lambda_min. Supplying a value of 42 | % lambda override this. Use with care - it is better to 43 | % supply a decreasing sequence of lambda values than a 44 | % single (small) value. 45 | % options.standardize Logical flag for variable standardization, prior to 46 | % fitting the model sequence. The coefficients are 47 | % always returned on the original scale. Default is 48 | % standardize = true. 49 | % options.thresh Convergence threshold for coordinate descent. Each 50 | % inner coordinate-descent loop continues until the 51 | % relative change in any coefficient is less than 52 | % thresh. Defaults value is 1E-4. 53 | % options.dfmax Limit the maximum number of variables in the model. 54 | % Useful for very large nvars, if a partial path is 55 | % desired. Default is nvars + 1. 56 | % options.pmax Limit the maximum number of variables ever to be 57 | % nonzero. Default is min(dfmax * 1.2, nvars). 58 | % options.exclude Indices of variables to be excluded from the model. 59 | % Default is none. Equivalent to an infinite penalty 60 | % factor (next item). 61 | % options.penalty_factor 62 | % Separate penalty factors can be applied to each 63 | % coefficient. This is a number that multiplies lambda 64 | % to allow differential shrinkage. Can be 0 for some 65 | % variables, which implies no shrinkage, and that 66 | % variable is always included in the model. Default is 67 | % 1 for all variables (and implicitly infinity for 68 | % variables listed in exclude). 69 | % options.maxit Maximum number of outer-loop iterations for 70 | % 'binomial' or 'multinomial' families. Default is 100. 71 | % options.HessianExact 72 | % Only applies to 'binomial' or 'multinomial' families. 73 | % If false (the default), an upper-bound approximation 74 | % is made to the hessian, which is not recalculated at 75 | % each outer loop. 76 | % options.type Two algorithm types are supported for (only) 77 | % family = 'gaussian'. The default type = 'covariance' 78 | % saves all inner-products ever computed, and can be 79 | % much faster than type = 'naive'. The latter can be 80 | % more efficient for p >> N situations. 81 | % 82 | % LICENSE: GPL-2 83 | % 84 | % DATE: 14 Jul 2009 85 | % 86 | % AUTHORS: 87 | % Algorithm was designed by Jerome Friedman, Trevor Hastie and Rob Tibshirani 88 | % Fortran code was written by Jerome Friedman 89 | % R wrapper (from which the MATLAB wrapper was adapted) was written by Trevor Hasite 90 | % MATLAB wrapper was written and maintained by Hui Jiang, jiangh@stanford.edu 91 | % Department of Statistics, Stanford University, Stanford, California, USA. 92 | % 93 | % REFERENCES: 94 | % Friedman, J., Hastie, T. and Tibshirani, R. (2009) 95 | % Regularization Paths for Generalized Linear Models via Coordinate Descent. 96 | % Journal of Statistical Software, 33(1), 2010 97 | % 98 | % SEE ALSO: 99 | % glmnet, glmnetPrint, glmnetPlot, glmnetPredict and glmnetCoef methods. 100 | % 101 | % EXAMPLES: 102 | % 103 | % DEVELOPMENT: 14 Jul 2009: Original version of glmnet.m written. 104 | 105 | % Set default options. 106 | options.weights = []; 107 | options.alpha = 1.0; 108 | % options.nlambda = 100; 109 | % options.lambda_min = 0; 110 | options.lambda = lambda; % optimum lambda for CCDC 111 | options.standardize = true; 112 | options.thresh = 1E-4; 113 | % options.dfmax = 0; 114 | % options.pmax = 0; 115 | % options.exclude = []; 116 | % options.penalty_factor = []; 117 | % options.maxit = 100; 118 | options.HessianExact = false; 119 | % options.type = 'covariance'; 120 | 121 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 122 | % End default options. 123 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 124 | 125 | % % Quick return if no user opts 126 | % if nargin == 0 || isempty(opts) 127 | % if nargout == 0 % Display options. 128 | % disp('pdco default options:') 129 | % disp( options ) 130 | % end 131 | % return 132 | % end 133 | % 134 | % % List of valid field names 135 | % vfields = fieldnames( options ); 136 | % 137 | % % Grab valid fields from user's opts 138 | % for i = 1:length(vfields) 139 | % field = vfields{i}; 140 | % if isfield( opts, field ); 141 | % options.(field) = opts.(field); 142 | % end 143 | % end 144 | -------------------------------------------------------------------------------- /glmnet_fast.m: -------------------------------------------------------------------------------- 1 | function fit = glmnet_fast(x, y, options) 2 | % Speed up version by Zhe Zhu (10/28/2014) 3 | %-------------------------------------------------------------------------- 4 | % glmnet.m: fit an elasticnet model path 5 | %-------------------------------------------------------------------------- 6 | % 7 | % DESCRIPTION: 8 | % Fit a regularization path for the elasticnet at a grid of values for 9 | % the regularization parameter lambda. Can deal with all shapes of data. 10 | % Fits linear, logistic and multinomial regression models. 11 | % 12 | % USAGE: 13 | % fit = glmnet(x, y) 14 | % fit = glmnet(x, y, family, options) 15 | % 16 | % EXTERNAL FUNCTIONS: 17 | % options = glmnetSet; provided with glmnet.m 18 | % 19 | % INPUT ARGUMENTS: 20 | % x Input matrix, of dimension nobs x nvars; each row is an 21 | % observation vector. Currently sparse format is NOT supported. 22 | % y Response variable. Quantitative for family = 23 | % 'gaussian'. For family = 'binomial' should be either a vector 24 | % of two levels, or a two-column matrix of counts or 25 | % proportions. For family = 'multinomial', can be either a 26 | % vector of nc>=2 levels, or a matrix with nc columns of counts 27 | % or proportions. 28 | % family Reponse type. (See above). Default is 'gaussian'. 29 | % options A structure that may be set and altered by glmnetSet (type 30 | % help glmnetSet). 31 | % 32 | % OUTPUT ARGUMENTS: 33 | % fit A structure. 34 | % fit.a0 Intercept sequence of length length(fit.lambda). 35 | % fit.beta For "elnet" and "lognet" models, a nvars x length(lambda) 36 | % matrix of coefficients. For "multnet", a list of nc such 37 | % matrices, one for each class. 38 | % fit.lambda The actual sequence of lambda values used. 39 | % fit.dev The fraction of (null) deviance explained (for "elnet", this 40 | % is the R-square). 41 | % fit.nulldev Null deviance (per observation). 42 | % fit.df The number of nonzero coefficients for each value of lambda. 43 | % For "multnet", this is the number of variables with a nonzero 44 | % coefficient for any class. 45 | % fit.dfmat For "multnet" only. A matrix consisting of the number of 46 | % nonzero coefficients per class. 47 | % fit.dim Dimension of coefficient matrix (ices). 48 | % fit.npasses Total passes over the data summed over all lambda values. 49 | % fit.jerr Error flag, for warnings and errors (largely for internal 50 | % debugging). 51 | % fit.class Type of regression - internal usage. 52 | % 53 | % DETAILS: 54 | % The sequence of models implied by lambda is fit by coordinate descent. 55 | % For family = 'gaussian' this is the lasso sequence if alpha = 1, else 56 | % it is the elasticnet sequence. For family = 'binomial' or family = 57 | % "multinomial", this is a lasso or elasticnet regularization path for 58 | % fitting the linear logistic or multinomial logistic regression paths. 59 | % Sometimes the sequence is truncated before options.nlambda values of 60 | % lambda have been used, because of instabilities in the logistic or 61 | % multinomial models near a saturated fit. glmnet(..., family = 62 | % 'binomial') fits a traditional logistic regression model for the 63 | % log-odds. glmnet(..., family = 'multinomial') fits a symmetric 64 | % multinomial model, where each class is represented by a linear model 65 | % (on the log-scale). The penalties take care of redundancies. A 66 | % two-class "multinomial" model will produce the same fit as the 67 | % corresponding "binomial" model, except the pair of coefficient 68 | % matrices will be equal in magnitude and opposite in sign, and half the 69 | % "binomial" values. Note that the objective function for 70 | % "gaussian" is 71 | % 1 / (2 * nobs) RSS + lambda * penalty 72 | % , and for the logistic models it is 73 | % 1 / nobs - loglik + lambda * penalty 74 | % 75 | % LICENSE: GPL-2 76 | % 77 | % DATE: 14 Jul 2009 78 | % 79 | % AUTHORS: 80 | % Algorithm was designed by Jerome Friedman, Trevor Hastie and Rob Tibshirani 81 | % Fortran code was written by Jerome Friedman 82 | % R wrapper (from which the MATLAB wrapper was adapted) was written by Trevor Hasite 83 | % MATLAB wrapper was written and maintained by Hui Jiang, jiangh@stanford.edu 84 | % Department of Statistics, Stanford University, Stanford, California, USA. 85 | % 86 | % REFERENCES: 87 | % Friedman, J., Hastie, T. and Tibshirani, R. (2009) 88 | % Regularization Paths for Generalized Linear Models via Coordinate Descent. 89 | % Journal of Statistical Software, 33(1), 2010 90 | % 91 | % SEE ALSO: 92 | % glmnetSet, glmnetPrint, glmnetPlot, glmnetPredict and glmnetCoef methods. 93 | % 94 | % EXAMPLES: 95 | % x=randn(100,20); 96 | % y=randn(100,1); 97 | % g2=randsample(2,100,true); 98 | % g4=randsample(4,100,true); 99 | % fit1=glmnet(x,y); 100 | % glmnetPrint(fit1); 101 | % glmnetCoef(fit1,0.01) % extract coefficients at a single value of lambda 102 | % glmnetPredict(fit1,'response',x(1:10,:),[0.01,0.005]') % make predictions 103 | % fit2=glmnet(x,g2,'binomial'); 104 | % fit3=glmnet(x,g4,'multinomial'); 105 | % 106 | % DEVELOPMENT: 107 | % 14 Jul 2009: Original version of glmnet.m written. 108 | % 26 Jan 2010: Fixed a bug in the description of y, pointed out by 109 | % Peter Rijnbeek from Erasmus University. 110 | % 09 Mar 2010: Fixed a bug of printing "ka = 2", pointed out by 111 | % Ramon Casanova from Wake Forest University. 112 | % 25 Mar 2010: Fixed a bug when p > n in multinomial fitting, pointed 113 | % out by Gerald Quon from University of Toronto 114 | % 25 Jul 2010: Check for input matrix format and size 115 | % 27 Sep 2010: Fixed a bug of undefined "df" in multinomial fitting, 116 | % pointed by Jeff Howbert from Insilicos. 117 | % 27 Jan 2013 Faster version made especially for CCDC (Zhe Zhu) 118 | 119 | % % Check input arguments 120 | % if nargin < 2 121 | % error('more input arguments needed.'); 122 | % end 123 | % 124 | % if nargin < 3 125 | % family = 'gaussian'; 126 | % end 127 | % 128 | % if nargin < 4 129 | % options = glmnetSet; 130 | % end 131 | 132 | % fixed family 133 | % family = 'gaussian'; 134 | % fixed options 135 | % options = glmnetSet; 136 | 137 | % Prepare parameters 138 | % nlam = options.nlambda; 139 | 140 | % if (issparse(x) || issparse(y)) 141 | % error('currently sparse matrix is NOT supported.'); 142 | % end 143 | % 144 | % if (~isa(x,'double') || ~isa(y,'double')) 145 | % error('only DOUBLE precision matrix is supported.'); 146 | % end 147 | 148 | [nobs,nvars] = size(x); 149 | 150 | % if (nobs <= 1) 151 | % error('at least two observations should be provided.'); 152 | % end 153 | 154 | weights = options.weights; 155 | if isempty(weights) 156 | weights = ones(nobs,1); 157 | end 158 | 159 | % maxit = options.maxit; 160 | 161 | % if strcmp(family, 'binomial') || strcmp(family, 'multinomial') 162 | % [noo,nc] = size(y); 163 | % kopt = double(options.HessianExact); 164 | % if noo ~= nobs 165 | % error('x and y have different number of rows'); 166 | % end 167 | % if nc == 1 168 | % classes = unique(y); 169 | % nc = length(classes); 170 | % indexes = eye(nc); 171 | % y = indexes(y,:); 172 | % end 173 | % if strcmp(family, 'binomial') 174 | % if nc > 2 175 | % error ('More than two classes; use multinomial family instead'); 176 | % end 177 | % nc = 1; % for calling multinet 178 | % end 179 | % if ~isempty(weights) 180 | % % check if any are zero 181 | % o = weights > 0; 182 | % if ~all(o) %subset the data 183 | % y = y(o,:); 184 | % x = x(o,:); 185 | % weights = weights(o); 186 | % nobs = sum(o); 187 | % end 188 | % [my,ny] = size(y); 189 | % y = y .* repmat(weights,1,ny); 190 | % end 191 | % % Compute the null deviance 192 | % prior = sum(y,1); 193 | % sumw = sum(sum(y)); 194 | % prior = prior / sumw; 195 | % nulldev = -2 * sum(sum(y .* (ones(nobs, 1) * log(prior)))) / sumw; 196 | % elseif strcmp(family, 'gaussian') 197 | % Compute the null deviance 198 | ybar = y' * weights/ sum(weights); 199 | nulldev = (y' - ybar).^2 * weights / sum(weights); 200 | % if strcmp(options.type, 'covariance') 201 | % ka = 1; 202 | % elseif strcmp(options.type, 'naive') 203 | % ka = 2; 204 | % else 205 | % error('unrecognized type'); 206 | % end 207 | ka = 1; 208 | % else 209 | % error('unrecognized family'); 210 | % end 211 | 212 | % ne = options.dfmax; 213 | % if ne == 0 214 | % ne = nvars + 1; 215 | % end 216 | ne = nvars + 1; 217 | 218 | % nx = options.pmax; 219 | % if nx == 0 220 | % nx = min(ne * 1.2, nvars); 221 | % end 222 | nx = min(ne * 1.2, nvars); 223 | 224 | % exclude = options.exclude; 225 | % if ~isempty(exclude) 226 | % exclude = unique(exclude); 227 | % % if ~all(exclude > 0 & exclude <= nvars) 228 | % % error('Some excluded variables out of range'); 229 | % % end 230 | % jd = [length(exclude); exclude]; 231 | % else 232 | % jd = 0; 233 | % end 234 | jd = 0; 235 | 236 | % vp = options.penalty_factor; 237 | % if isempty(vp) 238 | % vp = ones(nvars,1); 239 | % end 240 | vp = ones(nvars,1); 241 | 242 | isd = double(options.standardize); 243 | thresh = options.thresh; 244 | lambda = options.lambda; 245 | % lambda_min = options.lambda_min; 246 | % % if lambda_min == 0 247 | % if nobs < nvars 248 | % lambda_min = 5e-2; 249 | % else 250 | % lambda_min = 1e-4; 251 | % end 252 | % end 253 | % if isempty(lambda) 254 | % if (lambda_min >= 1) 255 | % error('lambda_min should be less than 1'); 256 | % end 257 | % flmin = lambda_min; 258 | % ulam = 0; 259 | % else 260 | flmin = 1.0; 261 | % if any(lambda < 0) 262 | % error ('lambdas should be non-negative'); 263 | % end 264 | ulam = -sort(-lambda); 265 | nlam = length(lambda); 266 | % end 267 | 268 | parm = options.alpha; 269 | 270 | % if strcmp(family, 'gaussian') 271 | [a0,ca,ia,nin,rsq,alm,nlp,jerr] = glmnetMex(parm,x,y,jd,vp,ne,nx,nlam,flmin,ulam,thresh,isd,weights,ka); 272 | % else 273 | % [a0,ca,ia,nin,dev,alm,nlp,jerr] = glmnetMex(parm,x,y,jd,vp,ne,nx,nlam,flmin,ulam,thresh,isd,nc,maxit,kopt); 274 | % end 275 | 276 | % Prepare output 277 | lmu = length(alm); 278 | ninmax = max(nin); 279 | lam = alm; 280 | if isempty(options.lambda) 281 | lam = fix_lam(lam); % first lambda is infinity; changed to entry point 282 | end 283 | % errmsg = err(jerr, maxit, nx); 284 | % if errmsg.n == 1 285 | % error(errmsg.msg); 286 | % elseif errmsg.n == -1 287 | % warning(errmsg.msg); 288 | % end 289 | 290 | % if strcmp(family, 'multinomial') 291 | % beta_list = {}; 292 | % a0 = a0 - repmat(mean(a0), nc, 1); 293 | % dfmat=a0; 294 | % dd=[nvars, lmu]; 295 | % if ninmax > 0 296 | % ca = reshape(ca, nx, nc, lmu); 297 | % ca = ca(1:ninmax,:,:); 298 | % ja = ia(1:ninmax); 299 | % [ja1,oja] = sort(ja); 300 | % df = any(abs(ca) > 0, 2); 301 | % df = sum(df, 1); 302 | % df = df(:); 303 | % for k=1:nc 304 | % ca1 = reshape(ca(:,k,:), ninmax, lmu); 305 | % cak = ca1(oja,:); 306 | % dfmat(k,:) = sum(sum(abs(cak) > 0)); 307 | % beta = zeros(nvars, lmu); 308 | % beta(ja1,:) = cak; 309 | % beta_list{k} = beta; 310 | % end 311 | % else 312 | % for k = 1:nc 313 | % dfmat(k,:) = zeros(1,lmu); 314 | % beta_list{k} = zeros(nvars, lmu); 315 | % end 316 | % df = zeros(1,lmu); 317 | % end 318 | % fit.a0 = a0; 319 | % fit.beta = beta_list; 320 | % fit.dev = dev; 321 | % fit.nulldev = nulldev; 322 | % fit.dfmat = dfmat; 323 | % fit.df = df'; 324 | % fit.lambda = lam; 325 | % fit.npasses = nlp; 326 | % fit.jerr = jerr; 327 | % fit.dim = dd; 328 | % fit.class = 'multnet'; 329 | % else 330 | dd=[nvars, lmu]; 331 | if ninmax > 0 332 | ca = ca(1:ninmax,:); 333 | df = sum(abs(ca) > 0, 1); 334 | ja = ia(1:ninmax); 335 | [ja1,oja] = sort(ja); 336 | beta = zeros(nvars, lmu); 337 | beta (ja1, :) = ca(oja,:); 338 | else 339 | beta = zeros(nvars,lmu); 340 | df = zeros(1,lmu); 341 | end 342 | 343 | % if strcmp(family, 'binomial') 344 | % a0 = -a0; 345 | % fit.a0 = a0; 346 | % fit.beta = -beta; %sign flips make 2 arget class 347 | % fit.dev = dev; 348 | % fit.nulldev = nulldev; 349 | % fit.df = df'; 350 | % fit.lambda = lam; 351 | % fit.npasses = nlp; 352 | % fit.jerr = jerr; 353 | % fit.dim = dd; 354 | % fit.class = 'lognet'; 355 | % else 356 | fit.a0 = a0; 357 | fit.beta = beta; 358 | fit.dev = rsq; 359 | fit.nulldev = nulldev; 360 | fit.df = df'; 361 | fit.lambda = lam; 362 | fit.npasses = nlp; 363 | fit.jerr = jerr; 364 | fit.dim = dd; 365 | fit.class = 'elnet'; 366 | % end 367 | % end 368 | 369 | %------------------------------------------------------------------ 370 | % End function glmnet 371 | %------------------------------------------------------------------ 372 | 373 | function new_lam = fix_lam(lam) 374 | 375 | new_lam = lam; 376 | llam=log(lam); 377 | new_lam(1)=exp(2*llam(2)-llam(3)); 378 | 379 | %------------------------------------------------------------------ 380 | % End private function fix_lam 381 | %------------------------------------------------------------------ 382 | 383 | % function output = err(n,maxit,pmax) 384 | % 385 | % if n==0 386 | % output.n=0; 387 | % output.msg=''; 388 | % elseif n>0 %fatal error 389 | % if n<7777 390 | % msg='Memory allocation error; contact package maintainer'; 391 | % elseif n==7777 392 | % msg='All used predictors have zero variance'; 393 | % elseif (8000 1.0 - 1.0e-5', n-9000); 397 | % elseif n==10000 398 | % msg='All penalty factors are <= 0'; 399 | % end 400 | % output.n=1; 401 | % output.msg=['in glmnet fortran code - %s',msg]; 402 | % elseif n<0 %non fatal error 403 | % if n > -10000 404 | % msg=sprintf('Convergence for %dth lambda value not reached after maxit=%d iterations; solutions for larger lambdas returned', -n, maxit); 405 | % elseif n < -10000 406 | % msg=sprintf('Number of nonzero coefficients along the path exceeds pmax=%d at %dth lambda value; solutions for larger lambdas returned', pmax, -n-10000); 407 | % end 408 | % output.n=-1; 409 | % output.msg=['from glmnet fortran code - ',msg]; 410 | % end 411 | % 412 | % %------------------------------------------------------------------ 413 | % % End private function err 414 | % %------------------------------------------------------------------ 415 | -------------------------------------------------------------------------------- /label_dist_type.m: -------------------------------------------------------------------------------- 1 | function [dist_type,dist_year,dist_doy]=label_dist_type(curr_cft,curr_time,t_c,vec,next_cft) 2 | % This function is used to provde distubance year and disturbance type 3 | % 1 => regrowth break 4 | % 2 => aforestation break 5 | % 3 => land disturbance 6 | % 7 | % Version 1.0: (Zhe Zhu 10/30/2018) 8 | % Modification: Identifications of Aforestation and Regrowth breaks are modified in Line #27. (Zhe and Shi, 21/08/2020) 9 | % 10 | %% get disurbance pixel 11 | % vec = obs - pred 12 | % only provide disturbance map 13 | % obs - pred 14 | nir = vec(4); 15 | c_nir = curr_cft(2,4); 16 | n_nir = next_cft(2,4); 17 | 18 | vis = vec(3); 19 | c_vis = curr_cft(2,3); 20 | n_vis = next_cft(2,3); 21 | 22 | swir = vec(5); 23 | c_swir = curr_cft(2,5); 24 | n_swir = next_cft(2,5); 25 | 26 | if nir > t_c && vis < -t_c && swir < -t_c 27 | if n_nir > abs(c_nir) && n_vis < -abs(c_vis) && n_swir < -abs(c_swir) 28 | dist_type = 2; % aforestation 29 | else 30 | dist_type = 1; % regrowth 31 | end 32 | else 33 | dist_type = 3; % land disturbance 34 | end 35 | 36 | dist_year = datevecmx(curr_time); 37 | dist_year = dist_year(1); 38 | dist_doy = curr_time - datenummx(dist_year,1,0); 39 | -------------------------------------------------------------------------------- /main_COLD.m: -------------------------------------------------------------------------------- 1 | function main_COLD(task,ntasks) 2 | % Matlab code for Continous Change Detection (for standalone version) 3 | % Inputs: 4 | % T_cg: change probability threshold 5 | % conse: number of consecutive observations 6 | % max_c: maximum number of coefficients used 7 | % task: The task CPU used 8 | % ntasks: number of CPUs used 9 | % profile on; 10 | % Get variable and path 11 | dir_l = pwd; %'/projectnb/landsat/projects/LCMS/4530/images/'; 12 | 13 | % %% Tuning variables for continuous change detection 14 | % % probability for detecting surface change 15 | % T_cg = 0.99; % (default) 16 | % % number of consecutive observation 17 | % conse = 6; % (default) 18 | % % number of maximum coefficients 19 | % max_c = 8; % (default) 20 | try inputs = textread('COLD_Parameters.txt'); %#ok 21 | % need at least three varibles 22 | % change threshold 23 | T_cg = inputs(1); % 0.99 24 | % number of consecutive obs 25 | conse = inputs(2); % 6 26 | % maximum number of coefficients 27 | max_c = inputs(3); % 8 28 | catch me %#ok 29 | % change threshold 30 | T_cg = 0.99; 31 | % number of consecutive obs 32 | conse = 6; 33 | % maximum number of coefficients 34 | max_c = 8; 35 | end 36 | %% Constants: 37 | % Bands for detection change 38 | B_detect = 2:6; 39 | % version of CCDC 40 | ccdc_v = 13.03; 41 | % Treshold of noise 42 | Tmax_cg = 1-1e-5; 43 | 44 | % get image parameters automatically 45 | imf=dir('L*'); % folder names 46 | imf(~[imf.isdir]) = []; % only directory 47 | [nrows,ncols,nbands,~,~,~,num_imgs] = autoPara(imf); 48 | 49 | % folder name of all CCDC results 50 | name_rst = 'TSFitMap'; 51 | % make TSFitMap folder for storing coefficients 52 | if isempty(dir(name_rst)) 53 | mkdir(name_rst); 54 | end 55 | 56 | % log for CCDC Change paramters and versions 57 | % report only for the first task 58 | if task == 1 59 | fileID = fopen('COLD_log.txt','w'); 60 | % write location of image stack 61 | fprintf(fileID,'Image location = %s\r\n',dir_l); 62 | % write number of images used 63 | fprintf(fileID,'Number of images = %d\r\n',num_imgs); 64 | % change probability threshold 65 | fprintf(fileID,'Change Probability Threshold = %.2f\r\n',T_cg); 66 | % # of consecutive observations 67 | fprintf(fileID,'Number of Consecutive Observations = %d\r\n',conse); 68 | % Maximum number of coefficients 69 | fprintf(fileID,'Maximum Number of Coefficients = %d\r\n',max_c); 70 | % CCDC Version 71 | % fprintf(fileID,'CCDC Change for ARD Version = %.2f\r\n',ccdc_v); 72 | fprintf(fileID,'COLD Land Disturbance Detection Algorithm with Analysis Ready Data Version = %.2f\r\n',ccdc_v); 73 | % updates 74 | fprintf(fileID,'******************************************************************************************************\r\n'); 75 | fprintf(fileID,'Revisions: $ Date: 10/30/2018 $ Copyright: Zhe Zhu\r\n'); 76 | fprintf(fileID,'Version 13.03 Modified model intitializatin test (10/29/2018)\r\n'); 77 | fprintf(fileID,'Version 13.02 Add included angle to exlcude false positive change (10/05/2018) \r\n'); 78 | fprintf(fileID,'Version 13.01 Do not need clear observations more than 25 percent (03/28/2018)\r\n'); 79 | fprintf(fileID,'Version 13.00 Optimied parameters for monitoring disturbance (03/20/2018)\r\n'); 80 | fprintf(fileID,'Version 12.36 Reduce 75 perecent of memory needed for line_t (11/22/2017)\r\n'); 81 | fprintf(fileID,'Version 12.35 Adjust T_cg based on conse (10/03/2017)\r\n'); 82 | fprintf(fileID,'Version 12.34 Add data density requirement (09/20/2017)\r\n'); 83 | fprintf(fileID,'Version 12.33 Adjust conse based on delta time (09/19/2017)\r\n'); 84 | fprintf(fileID,'Version 12.32 Fix bud for not enough data (09/17/2017)\r\n'); 85 | fprintf(fileID,'Version 12.31 Read input varibles in txt file (07/05/2016)\r\n'); 86 | fprintf(fileID,'Version 12.30 Fix a bug for pixels without minimum observations (11/20/2015)\r\n'); 87 | fprintf(fileID,'Version 12.29 Modify fit for perennial snow and Fmask failed pixels (09/20/2015)\r\n'); 88 | fprintf(fileID,'Version 12.28 Fix a bug for missing values in land cover maps (09/16/2015)\r\n'); 89 | fprintf(fileID,'Version 12.27 Fix a bug for persistent snow and falied Fmask pixels (06/17/2015)\r\n'); 90 | fprintf(fileID,'Version 12.26 Connect time for all models (05/28/2015)\r\n'); 91 | fprintf(fileID,'Version 12.25 Fix a bug in snow percent (05/19/2015)\r\n'); 92 | fprintf(fileID,'Version 12.24 Change T_const in Tmask (03/31/2015)\r\n'); 93 | fprintf(fileID,'Version 12.23 Update iteratively before 24 observations (03/22/2015)\r\n'); 94 | fprintf(fileID,'Version 12.22 Adjust mini RMSE based on temporal variability (03/22/2015)\r\n'); 95 | fprintf(fileID,'Version 12.21 Add more categories and update i_start in the end (03/14/2015)\r\n'); 96 | fprintf(fileID,'Version 12.20 Convert BT to from K to C before analysis (03/12/2015)\r\n'); 97 | fprintf(fileID,'Version 12.19 Fit for permanent snow if it is > 75 percent (03/12/2015)\r\n'); 98 | fprintf(fileID,'Version 12.18 No change detection if clear observation < 25 percent (03/12/2015)\r\n'); 99 | fprintf(fileID,'Version 12.17 Use median value for very simple model & change magnitude (02/24/2015)\r\n'); 100 | fprintf(fileID,'Version 12.16 Finding changes in all water pixels (02/24/2015)\r\n'); 101 | fprintf(fileID,'Version 12.15 Use the original multitemporal cloud mask (02/15/2015)\r\n'); 102 | fprintf(fileID,'Version 12.14 Do not need example_img in images folder (02/09/2015)\r\n'); 103 | fprintf(fileID,'Version 12.13 More infromation in "category" (11/10/2014)\r\n'); 104 | fprintf(fileID,'Version 12.12 Fit for pixels where Fmask fails (11/09/2014)\r\n'); 105 | fprintf(fileID,'Version 12.11 Fix a bug in num_fc (11/09/2014)\r\n'); 106 | fprintf(fileID,'Version 12.10 Better multietmporal cloud detection at the beginning (11/06/2014)\r\n'); 107 | fprintf(fileID,'Version 12.09 Detect change for land pixel (water/snow speical case) (10/31/2014)\r\n'); 108 | fprintf(fileID,'Version 12.08 Speed up by reducing time for RMSE and model computing (10/17/2014)\r\n'); 109 | fprintf(fileID,'Version 12.07 mini rmse should be larger than 10 percent of the mean (10/13/2014)\r\n'); 110 | fprintf(fileID,'Version 12.06 Fit model again when there are a 33 percent more in time (10/08/2014)\r\n'); 111 | fprintf(fileID,'Version 12.05 Use subset of bands (2-6) for detecting surface change (10/01/2014)\r\n'); 112 | fprintf(fileID,'Version 12.04 Only apply Tmask during model initialization (09/29/2014)\r\n'); 113 | fprintf(fileID,'Version 12.03 Use subset of bands (3-5) for detecting surface change (09/01/2014)\r\n'); 114 | fprintf(fileID,'Version 12.02 Fix a bug in model intialization (08/14/2014)\r\n'); 115 | fprintf(fileID,'Version 12.01 Use subset of bands (3-6) to reduce atmosphere influence (08/04/2014)\r\n'); 116 | fprintf(fileID,'Version 12.00 Detect change based on probability (07/19/2014)\r\n'); 117 | fprintf(fileID,'Version 11.06 No need to change folder name & faster in speed (06/06/2014)\r\n'); 118 | fprintf(fileID,'Version 11.05 Improve calculation of temporally adjusted RMSE (04/23/2014)\r\n'); 119 | fprintf(fileID,'Version 11.04 Revise "rec_cg.category" for different fit processes (04/01/2014)\r\n'); 120 | fprintf(fileID,'Version 11.03 Add "rec_cg.magnitude" as change magnitude indicator (04/01/2014)\r\n'); 121 | fprintf(fileID,'Version 11.02 Change very simple fit with mean value (04/01/2014)\r\n'); 122 | fprintf(fileID,'Version 11.01 Do not need metadata in the image folder to run CCDC (03/25/2014)\r\n'); 123 | fprintf(fileID,'Version 11.00 Use change vector magnitude as hreshold for change (03/25/2014)\r\n'); 124 | fprintf(fileID,'Version 10.13 Use subset of bands (1-6) to reduce atmosphere influence (01/31/2014)\r\n'); 125 | fprintf(fileID,'Version 10.12 More accurate number of days per year "num_yrs" (01/30/2014)\r\n'); 126 | fprintf(fileID,'Version 10.11 RMSE updates with time series fit (01/26/2014)\r\n'); 127 | fprintf(fileID,'Version 10.10 Update temperature extreme in recent studies (01/16/2014)\r\n'); 128 | fprintf(fileID,'Version 10.09 Find break in max value in any of the band (01/08/2014)\r\n'); 129 | fprintf(fileID,'Version 10.08 Add very simple fit (median) for start & end of timeseries (10/21/2013)\r\n'); 130 | fprintf(fileID,'Version 10.07 Better multitemporal cloud detection (10/19/2013)\r\n'); 131 | fprintf(fileID,'Version 10.06 Add "Tmax_cg" for last step noise removal (10/18/2013)\r\n'); 132 | fprintf(fileID,'Version 10.05 Use subset of bands (2-6) to avoid atmosphere influences (10/18/2013)\r\n'); 133 | fprintf(fileID,'Version 10.04 Let dynamic fitting for pixels at the beginning (09/23/2013)\r\n'); 134 | fprintf(fileID,'Version 10.03 Able to detect change at the verying beginning (09/06/2013)\r\n'); 135 | fprintf(fileID,'Version 10.02 Add mini years "mini_yrs" in model intialization (09/03/2013)\r\n'); 136 | fprintf(fileID,'Version 10.01 Reduce time for calcuating "v_dif" (09/02/2013)\r\n'); 137 | fprintf(fileID,'Version 10.00 Fit for beginning and end of the time series (08/31/2013)\r\n'); 138 | fprintf(fileID,'Version 9.09 Only fit more than 50 percent of Landat images overlap area (08/28/2013)\r\n'); 139 | fprintf(fileID,'Version 9.08 Force model fit for persistent snow pixels (08/27/2013)\r\n'); 140 | fprintf(fileID,'Version 9.07 Add "rec_cg.category", "rec_cg.change_prob", and "recc_cg.num_obs" (08/20/2013)\r\n'); 141 | fprintf(fileID,'Version 9.06 Remove mininum rmse "mini" and minimum years "mini_yrs" (08/16/2013)\r\n'); 142 | fprintf(fileID,'Version 9.05 Model gets more coefficients with more observations (08/16/2013)\r\n'); 143 | fprintf(fileID,'Version 9.04 Fix a bug in calculating temporally adjusted rmse (08/01/2013)\r\n'); 144 | fprintf(fileID,'Version 9.03 Fit curve again after one year (03/28/2013)\r\n'); 145 | fprintf(fileID,'Version 9.02 Use "mini = T_const/T_cg" for small rmse cases (03/26/2013)\r\n'); 146 | fprintf(fileID,'Version 9.01 Remove out of range pixels before time series analysis (02/09/2013)\r\n'); 147 | fprintf(fileID,'Version 9.00 Using 8 coefficients and lasso fit (02/01/2013)\r\n'); 148 | fprintf(fileID,'Version 8.04 Use "max v_slope" instead of "average v_slope" (01/16/2013)\r\n'); 149 | fprintf(fileID,'Version 8.03 Start initialization when "time_span" > 1 year (01/16/2013)\r\n'); 150 | fprintf(fileID,'Version 8.02 Fix a bug in not fitting models at the begining (01/16/2013)" (08/20/2013)\r\n'); 151 | fprintf(fileID,'Version 8.01 Fix a bug in counting "i" and "i_span"(01/13/2013)\r\n'); 152 | fprintf(fileID,'Version 8.00 Temporally changing RMSE (01/09/2013)\r\n'); 153 | fprintf(fileID,'Version 7.03 Continuous Change Detection and Classification (CCDC) (07/11/2012)\r\n'); 154 | fprintf(fileID,'Version 1.00 Continous Monitoring of Forest Disturbance Algorithm (CMFDA) (07/13/2010) \r\n'); 155 | fprintf(fileID,'******************************************************************************************************\r\n'); 156 | fclose(fileID); 157 | end 158 | 159 | % prepare the irows for idn_cpu for ALL rows 160 | irows = zeros(1,1); 161 | i = 0; 162 | while task + ntasks*i <= nrows % process all lines 163 | irows(i+1) = task + ntasks*i; 164 | i = i+1; 165 | end 166 | 167 | % % only calculate a few lines 168 | % % start with line_start 169 | % line_start = 3700; 170 | % line_end = 4000; 171 | % 172 | % irows(irows < line_start) = []; 173 | % irows(irows > line_end) = []; 174 | 175 | for i = 1:length(irows) 176 | % Check whether record_change already exist for row == i 177 | try 178 | % if successfully loaded => skip 179 | load([dir_l,'/',name_rst,'/','record_change',sprintf('%d',irows(i)),'.mat']); %#ok 180 | catch me %#ok 181 | % not exist or corrupt 182 | % fprintf('Processing the %dth row\n',irows(i)); 183 | TrendSeasonalFit_COLDLine(dir_l,name_rst,ncols,irows(i),T_cg,Tmax_cg,conse,max_c,nbands,B_detect); 184 | % TrendSeasonalFit_PRLine(dir_l,name_rst,ncols,irows(i),T_cg,Tmax_cg,conse,max_c,nbands,B_detect); 185 | % Continous Change Detection Done for a line of timeseries pixels 186 | % TrendSeasonalFit_v12_30ARDLine(dir_l,name_rst,ncols,irows(i),T_cg,Tmax_cg,conse,max_c,nbands,B_detect); 187 | % TrendSeasonalFit_v12_30Line(dir_l,name_rst,ncols,irows(i),T_cg,Tmax_cg,conse,max_c,nbands,B_detect); 188 | end 189 | end 190 | 191 | % profile viewer 192 | % exit 193 | end % end of function 194 | -------------------------------------------------------------------------------- /read_envihdr.m: -------------------------------------------------------------------------------- 1 | function info = read_envihdr(hdrfile) 2 | % READ_ENVIHDR read and return ENVI image file header information. 3 | % INFO = READ_ENVIHDR('HDR_FILE') reads the ASCII ENVI-generated image 4 | % header file and returns all the information in a structure of 5 | % parameters. 6 | % 7 | % Example: 8 | % >> info = read_envihdr('my_envi_image.hdr') 9 | % info = 10 | % description: [1x101 char] 11 | % samples: 658 12 | % lines: 749 13 | % bands: 3 14 | % header_offset: 0 15 | % file_type: 'ENVI Standard' 16 | % data_type: 4 17 | % interleave: 'bsq' 18 | % sensor_type: 'Unknown' 19 | % byte_order: 0 20 | % map_info: [1x1 struct] 21 | % projection_info: [1x102 char] 22 | % wavelength_units: 'Unknown' 23 | % pixel_size: [1x1 struct] 24 | % band_names: [1x154 char] 25 | % 26 | % NOTE: This function is used by ENVIREAD to import data. 27 | 28 | fid = fopen(hdrfile); 29 | while fid 30 | line = fgetl(fid); 31 | if line == -1 32 | break 33 | else 34 | eqsn = findstr(line,'='); 35 | if ~isempty(eqsn) 36 | param = strtrim(line(1:eqsn-1)); 37 | param(findstr(param,' ')) = '_'; 38 | value = strtrim(line(eqsn+1:end)); 39 | if isempty(str2num(value)) 40 | if ~isempty(findstr(value,'{')) && isempty(findstr(value,'}')) 41 | while isempty(findstr(value,'}')) 42 | line = fgetl(fid); 43 | value = [value,strtrim(line)]; 44 | end 45 | end 46 | eval(['info.',param,' = ''',value,''';']) 47 | else 48 | eval(['info.',param,' = ',value,';']) 49 | end 50 | end 51 | end 52 | end 53 | fclose all; 54 | end -------------------------------------------------------------------------------- /robustfit_cor.m: -------------------------------------------------------------------------------- 1 | function beta = robustfit_cor(X,y) 2 | %ROBUSTFIT Robust linear regression 3 | %Corrected 1.0 version (Zhe 04/06/2013) 4 | % B = ROBUSTFIT(X,Y) returns the vector B of regression coefficients, 5 | % obtained by performing robust regression to estimate the linear model 6 | % Y = Xb. X is an n-by-p matrix of predictor variables, and Y is an 7 | % n-by-1 vector of observations. The algorithm uses iteratively 8 | % reweighted least squares with the bisquare weighting function. By 9 | % default, ROBUSTFIT adds a column of ones to X, corresponding to a 10 | % constant term in the first element of B. Do not enter a column of ones 11 | % directly into the X matrix. 12 | % 13 | % The ROBUSTFIT function estimates the variance-covariance matrix of the 14 | % coefficient estimates as V=inv(X'*X)*STATS.S^2. The standard errors 15 | % and correlations are derived from V. 16 | % 17 | % ROBUSTFIT treats NaNs in X or Y as missing values, and removes them. 18 | % 19 | % Example: 20 | % x = (1:10)'; 21 | % y = 10 - 2*x + randn(10,1); y(10) = 0; 22 | % bls = regress(y,[ones(10,1) x]) 23 | % brob = robustfit(x,y) 24 | % scatter(x,y) 25 | % hold on 26 | % plot(x,brob(1)+brob(2)*x,'r-', x,bls(1)+bls(2)*x,'m:') 27 | % 28 | % See also REGRESS, ROBUSTDEMO. 29 | 30 | % References: 31 | % DuMouchel, W.H., and F.L. O'Brien (1989), "Integrating a robust 32 | % option into a multiple regression computing environment," 33 | % Computer Science and Statistics: Proceedings of the 21st 34 | % Symposium on the Interface, American Statistical Association. 35 | % Holland, P.W., and R.E. Welsch (1977), "Robust regression using 36 | % iteratively reweighted least-squares," Communications in 37 | % Statistics - Theory and Methods, v. A6, pp. 813-827. 38 | % Huber, P.J. (1981), Robust Statistics, New York: Wiley. 39 | % Street, J.O., R.J. Carroll, and D. Ruppert (1988), "A note on 40 | % computing robust regression estimates via iteratively 41 | % reweighted least squares," The American Statistician, v. 42, 42 | % pp. 152-154. 43 | 44 | wfun = @bisquare; 45 | tune = 4.685; 46 | 47 | % varargout=cell(1,max(1,nargout)); 48 | beta = statrobustfit_cor(X,y,wfun,tune); 49 | 50 | % --------- weight functions 51 | function w = bisquare(r) 52 | w = (abs(r)<1) .* (1 - r.^2).^2; 53 | -------------------------------------------------------------------------------- /rs_imwrite_bands.m: -------------------------------------------------------------------------------- 1 | function rs_imwrite_bands(varargin) 2 | % rs_imwrite - Write an ENVI image with differnt bands with names (Copy Right - Zhe Zhu, Sept. 4th 2018) 3 | % Examples 4 | % rs_imwrite(image,'filename',info,n_names); 5 | % Input 6 | % image: image matrix 7 | % filename: File names 8 | % info: Image properties and cartographic information for ENVI or GeoTIFF file 9 | % n_names : Band names 1985:2015 or 'disturb type'; 10 | 11 | if nargin < 4 12 | fprintf('Failed! Less Than Three Variables! \n'); 13 | return 14 | elseif nargin == 4 15 | % Create input variables 16 | image = varargin{1}; 17 | fname = varargin{2}; 18 | info = varargin{3}; 19 | n_names = varargin{4}; 20 | 21 | im_size=size(image); 22 | im_size(3)=size(image,3); 23 | 24 | % ENVI data format numbers 25 | d = [1 2 3 4 5 12]; 26 | 27 | % Check user input 28 | if ~ischar(fname) 29 | error('fname should be a char string'); 30 | end 31 | 32 | cl=class(image); 33 | switch cl 34 | case 'uint8' 35 | t = d(1); 36 | case 'int16' 37 | t = d(2); 38 | case 'int32' 39 | t = d(3); 40 | case 'single' 41 | t = d(4); 42 | case 'double' 43 | t = d(5); 44 | case 'uint16' 45 | t = d(6); 46 | otherwise 47 | error('Data type not recognized'); 48 | end 49 | 50 | % update info 51 | info.samples = im_size(2); 52 | info.lines = im_size(1); 53 | info.bands = im_size(3); 54 | info.data_type = t; 55 | 56 | try multibandwrite(image,fname,info.interleave); 57 | fprintf('Write ENVI Images %s',fname); 58 | fprintf((' . ')); 59 | catch 60 | fprintf('Images Failed! Not ENVI Variables! \n'); 61 | return 62 | end 63 | 64 | % Write header file 65 | fprintf((' . ')); 66 | fid = fopen(strcat(fname,'.hdr'),'w'); 67 | 68 | fprintf(fid,'%s \n','ENVI'); 69 | n_field = fieldnames(info); 70 | n_value = struct2cell(info); 71 | 72 | for i = 1:length(n_field)-1 73 | n_field{i} = strrep(n_field{i},'_',' '); 74 | if ischar(n_value{i}) 75 | fprintf(fid,'%s = %s \n',n_field{i},n_value{i}); 76 | else 77 | fprintf(fid,'%s = %d \n',n_field{i},n_value{i}); 78 | end 79 | end 80 | 81 | % band names 82 | fprintf(fid,'band names = {'); 83 | if ischar(n_names) 84 | fprintf(fid,'%s',n_names); 85 | else 86 | for i=1:length(n_names) 87 | str_n = num2str(n_names(i)); 88 | fprintf(fid,'%s',str_n); 89 | if i < length(n_names) 90 | fprintf(fid,','); 91 | end 92 | end 93 | end 94 | fprintf(fid,'}\n'); 95 | 96 | % elements={'samples =','lines =','bands =',... 97 | % 'header offset =','file type =', 'data type =',... 98 | % 'interleave =','sensor type =','byte order =',... 99 | % 'map info =','coordinate system string =','wavelength units =',... 100 | % 'band names ='}; 101 | 102 | % fprintf(fid,'%s \n','ENVI'); 103 | % fprintf(fid,'%s %s \n','description =',info.description); 104 | % fprintf(fid,'%s %d \n',elements{1,1},im_size(2)); 105 | % fprintf(fid,'%s %d \n',elements{1,2},im_size(1)); 106 | % fprintf(fid,'%s %d \n',elements{1,3},im_size(3)); 107 | % fprintf(fid,'%s %d \n',elements{1,4},info.header_offset); 108 | % fprintf(fid,'%s %s \n',elements{1,5},info.file_type); 109 | % fprintf(fid,'%s %d \n',elements{1,6},t); 110 | % fprintf(fid,'%s %s \n',elements{1,7},info.interleave); 111 | % fprintf(fid,'%s %s \n',elements{1,8},info.sensor_type); 112 | % fprintf(fid,'%s %d \n',elements{1,9},info.byte_order); 113 | % fprintf(fid,'%s %s \n',elements{1,10},info.map_info); 114 | % fprintf(fid,'%s %s \n',elements{1,11},info.coordinate_system_string); 115 | % fprintf(fid,'%s %s \n',elements{1,12},info.wavelength_units); 116 | % fprintf(fid,'%s %s \n',elements{1,13},info.band_names); 117 | 118 | fclose(fid); 119 | fprintf((' . \n')); 120 | else 121 | fprintf('Failed! Too Many Variables! \n'); 122 | return 123 | end -------------------------------------------------------------------------------- /scanRequiredToolboxes.m: -------------------------------------------------------------------------------- 1 | % This MATLAB file is to scan the toolboxes requried 2 | % 3 | % Learn more from https://www.mathworks.com/help/matlab/matlab_prog/identify-dependencies.html#responsive_offcanvas 4 | % 5 | % 6 | % Mar 5., 2021 7 | % GERS Lab, UCONN 8 | % 9 | 10 | mfiles = dir('*.m'); 11 | 12 | fprintf('toolboxes required for each MATLAB file:\r'); 13 | toolboxesRequired = []; 14 | for im = 1: length(mfiles) 15 | mfilename = mfiles(im).name; 16 | [fList,pList] = matlab.codetools.requiredFilesAndProducts(mfilename); 17 | toolboxesRequired = [toolboxesRequired, pList]; % append all toolboxs together 18 | 19 | % Display each item 20 | displayinfo = sprintf('(%02d/%02d) Finished scanningthe matlab file %s (requries', ... 21 | im, length(mfiles), mfilename); 22 | % Mutilple toolboxs requried for a single matlab file maybe 23 | for ip = 1: length(pList) 24 | switch ip 25 | case 1 % first 26 | displayinfo = [displayinfo, ' ', pList(ip).Name]; 27 | case length(pList) % end 28 | displayinfo = [displayinfo, ' and ', pList(ip).Name]; 29 | otherwise % other 30 | displayinfo = [displayinfo, ', ', pList(ip).Name]; 31 | end 32 | end 33 | displayinfo = [displayinfo, ')']; 34 | fprintf('%s\r', displayinfo); 35 | end 36 | 37 | 38 | fprintf('\r'); 39 | fprintf('Summary of toolboxes requried:\r'); 40 | [~, idx] = unique({toolboxesRequired.Name}); 41 | 42 | for it = 1: length(idx) 43 | fprintf('%d. %s\r', it, toolboxesRequired(idx(it)).Name); 44 | end 45 | -------------------------------------------------------------------------------- /statrobustfit_cor.m: -------------------------------------------------------------------------------- 1 | function b = statrobustfit_cor(X,y,wfun,tune) 2 | % STATROBUSTFIT Calculation function for ROBUSTFIT 3 | % Corrected 1.0 version (Zhe 04/06/2013) 4 | 5 | [n,p] = size(X); 6 | X = [ones(n,1) X]; 7 | p = p+1; 8 | 9 | % Find the least squares solution. 10 | [Q,R,perm] = qr(X,0); 11 | tol = abs(R(1)) * max(n,p) * eps(class(R)); 12 | xrank = sum(abs(diag(R)) > tol); 13 | if xrank==p 14 | b(perm,:) = R \ (Q'*y); 15 | else 16 | % Use only the non-degenerate parts of R and Q, but don't reduce 17 | % R because it is returned in stats and is expected to be of 18 | % full size. 19 | b(perm,:) = [R(1:xrank,1:xrank) \ (Q(:,1:xrank)'*y); zeros(p-xrank,1)]; 20 | perm = perm(1:xrank); 21 | end 22 | b0 = zeros(size(b)); 23 | 24 | % Adjust residuals using leverage, as advised by DuMouchel & O'Brien 25 | E = X(:,perm)/R(1:xrank,1:xrank); 26 | h = min(.9999, sum(E.*E,2)); 27 | adjfactor = 1 ./ sqrt(1-h); 28 | 29 | % If we get a perfect or near perfect fit, the whole idea of finding 30 | % outliers by comparing them to the residual standard deviation becomes 31 | % difficult. We'll deal with that by never allowing our estimate of the 32 | % standard deviation of the error term to get below a value that is a small 33 | % fraction of the standard deviation of the raw response values. 34 | tiny_s = 1e-6 * std(y); 35 | if tiny_s == 0 36 | tiny_s = 1; 37 | end 38 | 39 | % Perform iteratively reweighted least squares to get coefficient estimates 40 | D = sqrt(eps(class(X))); 41 | iter = 1; 42 | iterlim = 5;% default value is 50 (less iteration improves efficiency) 43 | wxrank = xrank; % rank of weighted version of x 44 | while((iter==0) || any(abs(b-b0) > D*max(abs(b),abs(b0)))) 45 | iter = iter+1; 46 | if (iter>iterlim) 47 | break; 48 | end 49 | 50 | % Compute residuals from previous fit, then compute scale estimate 51 | r = y - X*b; 52 | radj = r .* adjfactor; 53 | mad_s = madsigma(radj,wxrank);% oringinally "s" 54 | 55 | % Compute new weights from these residuals, then re-fit 56 | w = feval(wfun, radj/(max(mad_s,tiny_s)*tune));% oringinally "s" 57 | b0 = b; 58 | [b(perm),wxrank] = wfit(y,X(:,perm),w); 59 | end 60 | 61 | % ----------------------------- 62 | function [b,r] = wfit(y,x,w) 63 | %WFIT weighted least squares fit 64 | 65 | % Create weighted x and y 66 | n = size(x,2); 67 | sw = sqrt(w); 68 | yw = y .* sw; 69 | xw = x .* sw(:,ones(1,n)); 70 | 71 | % Computed weighted least squares results 72 | [b,r] = linsolve(xw,yw,struct('RECT',true)); 73 | 74 | % ----------------------------- 75 | function s = madsigma(r,p) 76 | %MADSIGMA Compute sigma estimate using MAD of residuals from 0 77 | rs = sort(abs(r)); 78 | s = median(rs(max(1,p):end)) / 0.6745; 79 | 80 | -------------------------------------------------------------------------------- /update_cft.m: -------------------------------------------------------------------------------- 1 | function update_num_c = update_cft(i_span,n_times,min_num_c,mid_num_c,max_num_c,num_c) 2 | 3 | % determine the time series model 4 | if i_span < mid_num_c*n_times 5 | % start with 4 coefficients model 6 | update_num_c = min(min_num_c,num_c); 7 | elseif i_span < max_num_c*n_times 8 | % start with 6 coefficients model 9 | update_num_c = min(mid_num_c,num_c); 10 | else 11 | % start with 8 coefficients model 12 | update_num_c = min(max_num_c,num_c); 13 | end 14 | 15 | end --------------------------------------------------------------------------------