├── 0.25_196_ExtractFrame.sh ├── GenerateTrainData1ChanVarMix.m ├── LICENSE ├── README.md ├── caffe └── model │ └── cr5 │ ├── CNNTrainingResultNewOld │ ├── recon │ ├── GenerateTestData.m │ ├── measurement.mat │ ├── measurement100.mat │ ├── measurement25.mat │ ├── measurement5.mat │ ├── measurement50.mat │ ├── store2hdf5.m │ ├── videoNetCNN_cr100_deploy_10172016.prototxt │ ├── videoNetCNN_cr25_deploy_10172016.prototxt │ ├── videoNetCNN_cr50_deploy_10172016.prototxt │ └── videoNetCNN_cr5_deploy_10172016.prototxt │ ├── trainCNN_cr5.sh │ ├── trainCNN_cr5.txt │ ├── valCNN_cr5.txt │ ├── videoNetCNN_cr5_deploy_10172016.prototxt │ ├── videoNetCNN_cr5_deploy_10172016.prototxt.lua │ ├── videoNetCNN_cr5_deploy_feature_10172016.prototxt │ ├── videoNetCNN_cr5_feature_10172016.prototxt.lua │ ├── videoNetCNN_cr5_model_10172016.prototxt │ └── videoNetCNN_cr5_solver.prototxt ├── extractFeatures_5_25.m ├── genPhi.m ├── generateTrainCNN.m ├── generateValCNN.m ├── model └── cr_5_25 │ ├── model │ └── VideoNet.lua │ ├── recon │ ├── Figure1.m │ ├── GenerateTestData.m │ ├── dispResult.m │ ├── measurement.mat │ ├── reconResult.mat │ └── store2hdf5Mix.m │ ├── testVideoNet.lua │ ├── trainVideoNet.lua │ └── util │ ├── DataHandler.lua │ ├── DataHandler2.lua │ └── utils.lua ├── phi └── phi3_cr5.mat ├── store2hdf5.m ├── store2hdf5Mix.m └── util ├── DataHandler.lua ├── DataHandler10.lua ├── DataHandler2.lua ├── DataHandler3.lua ├── DataHandlerVar.lua ├── DataHandlerVar10.lua ├── DataHandlerVar_2.lua ├── DataHandler_1_25.lua ├── DataLoader.lua ├── utils.lua └── weight-init.lua /0.25_196_ExtractFrame.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | mkdir -p "0.25_196/" 3 | for f in *.avi 4 | do 5 | 6 | ffmpeg -ss 1 -i "$f" -r 0.25 -s 196x196 "0.25_196/${f%.avi}"_%02d.jpg 7 | done 8 | -------------------------------------------------------------------------------- /GenerateTrainData1ChanVarMix.m: -------------------------------------------------------------------------------- 1 | %data1-feature1, data2 2-original data2-10 2 | 3 | function GenerateTrainData1ChanVarMix(task, cr1, cr2) 4 | 5 | addpath('/home/user/kaixu/myGitHub/caffe/matlab/'); 6 | 7 | folder = ['~/myGitHub/datasets/UCF101/',task, 'Data/5_196/']; 8 | 9 | paths = dir(folder); 10 | paths(1:2) = []; 11 | totalct = 0; 12 | created_flag = false; 13 | 14 | for i = 1:length(paths) 15 | totalct = GenerateGroupTrainData1ChanVarMix(cr1, cr2, num2str(i), task, totalct, created_flag); 16 | created_flag = true; 17 | end 18 | end 19 | %% 20 | function totalct = GenerateGroupTrainData1ChanVarMix(cr1, cr2, group, task, totalct_last, created_flag) 21 | if cr1 == 5 22 | load ./phi/phi3_cr5.mat 23 | phi1 = phi3; 24 | elseif cr1 == 25 25 | load ./phi/phi3_cr25.mat 26 | phi1 = phi3; 27 | elseif cr1 == 50 28 | load ./phi/phi3_cr50.mat 29 | phi1 = phi3; 30 | elseif cr1 == 100 31 | load ./phi/phi3_cr100.mat 32 | phi1 = phi3; 33 | end 34 | 35 | if cr2 == 5 36 | load ./phi/phi3_cr5.mat 37 | phi2 = phi3; 38 | elseif cr2 == 16 39 | load ./phi/phi3_cr16.mat 40 | phi2 = phi3; 41 | elseif cr2 == 25 42 | load ./phi/phi3_cr25.mat 43 | phi2 = phi3; 44 | elseif cr2 == 50 45 | load ./phi/phi3_cr50.mat 46 | phi2 = phi3; 47 | elseif cr2 == 100 48 | load ./phi/phi3_cr100.mat 49 | phi2 = phi3; 50 | end 51 | 52 | phase = 'test'; 53 | 54 | %% 55 | if cr1 == 5 56 | net_model = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr5/'... ... 57 | 'videoNetCNN_cr5_deploy_feature_10172016.prototxt']; 58 | net_weights = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr5/'... 59 | 'Snapshots/cr_5_CNN_10172016/videoNetCNN_5_iter_175000.caffemodel']; 60 | elseif cr1 == 25 61 | net_model = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr25/'... ... 62 | 'videoNetCNN_cr25_deploy_feature_10172016.prototxt']; 63 | net_weights = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr25/'... 64 | 'Snapshots/cr_25_CNN_10172016/videoNetCNN_25_iter_170000.caffemodel']; 65 | end 66 | id = 2; 67 | caffe.set_mode_gpu(); 68 | caffe.set_device(id); 69 | net = caffe.Net(net_model, net_weights, phase); 70 | 71 | %% 72 | folder = ['~/myGitHub/datasets/UCF101/', task, 'Data/5_196/group', num2str(group), '/']; 73 | savepath = ['./data/',task,'Data_',num2str(cr1),'_',num2str(cr2),'_mix_10172016.h5']; 74 | 75 | if strcmp(task, 'Train') == 1 76 | step = 1; 77 | elseif strcmp(task, 'Val') == 1 78 | step = 2; 79 | else 80 | step = 1; 81 | end 82 | 83 | imgHeight = 196; 84 | imgWidth = 196; 85 | numChannels = 1; 86 | cnnHidden = 16; 87 | 88 | size_input = 32; 89 | size_label = 32; 90 | crop = 18; 91 | stride = size_input; 92 | seq_length = 10; % length of the LSTM 93 | 94 | % data = zeros(seq_length, numChannels, size(phi3,1), 1, 1, 'single'); 95 | % label = zeros(seq_length, numChannels, size_label, size_label, 1, 'uint8'); 96 | data1 = zeros(size_label, size_label, cnnHidden, 1, 1, 'single'); 97 | data2 = zeros(size(phi2,1), 1, numChannels, seq_length-1, 1, 'single'); 98 | label = zeros(size_label, size_label, numChannels, seq_length, 1, 'single'); 99 | 100 | % padding = abs(size_input - size_label)/2; 101 | count = 1; 102 | 103 | im_label = zeros(imgHeight-2*crop, imgWidth-2*crop, numChannels, seq_length); 104 | subim_input = zeros(size_input, size_input, numChannels, seq_length); 105 | subim_label = zeros(size_input, size_input, numChannels, seq_length); 106 | subim_input_dr1 = zeros(size(phi1,1), 1, numChannels, 1); 107 | subim_input_dr2 = zeros(size(phi2,1), 1, numChannels, seq_length-1); 108 | 109 | paths = dir(folder); 110 | paths(1:2) = []; 111 | 112 | newPaths = []; 113 | 114 | for i = 1:step:length(paths) 115 | newPaths = [newPaths,paths(i)]; 116 | end 117 | 118 | % h = waitbar(0,'Writing train data to hdf5 file, please wait...'); 119 | 120 | for i = 1:length(newPaths) 121 | filename = dir([folder, newPaths(i).name]); 122 | filename(1:2) = []; % filenames in each subfolder 123 | 124 | len = length(filename); 125 | tmp = floor(len / seq_length); 126 | if tmp >= 3 127 | tmp = 3; 128 | end 129 | numImage = tmp * seq_length; 130 | 131 | for j = 1:seq_length:numImage 132 | for k = 1:seq_length 133 | rawImg = imread([folder, newPaths(i).name, '/', filename(j+k-1).name]); 134 | rawImg = rgb2ycbcr(rawImg); 135 | rawImg = im2double(rawImg(:,:,1)); 136 | rawImg = rawImg(crop+1:end-crop,crop+1:end-crop); 137 | im_label(:,:,:,k) = rawImg; 138 | [hei,wid, ~, ~] = size(im_label); 139 | end 140 | 141 | for x = 1 : stride : hei-size_input+1 142 | for y = 1 : stride : wid-size_input+1 143 | for z = 1 : seq_length 144 | subim_input(:, :, :, z) = im_label(x : x+size_input-1, y : y+size_input-1, :, z); 145 | subim_input_rs = reshape(subim_input, [],numChannels, seq_length); 146 | if z == 1 147 | for xx = 1:numChannels 148 | if size(phi1, 1) == size_input^2 149 | subim_input_dr1(:,:,xx,1) = subim_input_rs(:,xx,z); 150 | else 151 | subim_input_dr1 = phi1(:,:,xx) * subim_input_rs(:,xx,z); 152 | feature = net.forward({subim_input_dr1}); 153 | feature = feature{1}; 154 | end 155 | end 156 | else 157 | for xx = 1:numChannels 158 | subim_input_dr2(:,:,xx,z-1) = phi2(:,:,xx) * subim_input_rs(:,xx,z); 159 | end 160 | end 161 | 162 | subim_label(:, :, :, z) = im_label(x : x+size_label-1, ... 163 | y : y+size_label-1, :, z); 164 | end 165 | 166 | data1(:, :, :, :, count) = feature; 167 | data2(:, :, :, :, count) = subim_input_dr2; 168 | label(:, :, :, :, count) = subim_label; 169 | 170 | count=count+1; 171 | end 172 | end 173 | end 174 | disp(['group', num2str(group), ', ', num2str(i/length(newPaths)*100),'%']); 175 | 176 | end 177 | 178 | if strcmp(task, 'Train') ~= 0 179 | count = count - 1; 180 | order = randperm(count); 181 | data1 = data1(:, :, :, :, order); 182 | data2 = data2(:, :, :, :, order); 183 | label = label(:, :, :, :, order); 184 | end 185 | 186 | % writing to HDF5 187 | chunksz = 20; 188 | % created_flag = false; 189 | totalct = totalct_last; 190 | 191 | for batchno = 1:floor(count/chunksz) 192 | last_read=(batchno-1) * chunksz; 193 | batchdata1 = data1(:,:,:,:,last_read+1:last_read+chunksz); 194 | batchdata2 = data2(:,:,:,:,last_read+1:last_read+chunksz); 195 | batchlabs = label(:,:,:,:,last_read+1:last_read+chunksz); 196 | 197 | startloc = struct('dat1',[1,1,1,1,totalct+1], 'dat2', [1,1,1,1,totalct+1], 'lab', [1,1,1,1,totalct+1]); 198 | curr_dat_sz = store2hdf5Mix(savepath, batchdata1, batchdata2, batchlabs, ~created_flag, startloc, chunksz); 199 | created_flag = true; 200 | totalct = curr_dat_sz(end); 201 | end 202 | h5disp(savepath); 203 | 204 | end 205 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Public License 2 | 3 | By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. 4 | 5 | Section 1 – Definitions. 6 | 7 | a. Adapted Material means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. 8 | 9 | b. Adapter's License means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. 10 | 11 | c. BY-NC-SA Compatible License means a license listed at creativecommons.org/compatiblelicenses, approved by Creative Commons as essentially the equivalent of this Public License. 12 | 13 | d. Copyright and Similar Rights means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. 14 | 15 | e. Effective Technological Measures means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. 16 | 17 | f. Exceptions and Limitations means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. 18 | 19 | g. License Elements means the license attributes listed in the name of a Creative Commons Public License. The License Elements of this Public License are Attribution, NonCommercial, and ShareAlike. 20 | 21 | h. Licensed Material means the artistic or literary work, database, or other material to which the Licensor applied this Public License. 22 | 23 | i. Licensed Rights means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. 24 | 25 | j. Licensor means the individual(s) or entity(ies) granting rights under this Public License. 26 | 27 | k. NonCommercial means not primarily intended for or directed towards commercial advantage or monetary compensation. For purposes of this Public License, the exchange of the Licensed Material for other material subject to Copyright and Similar Rights by digital file-sharing or similar means is NonCommercial provided there is no payment of monetary compensation in connection with the exchange. 28 | 29 | l. Share means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. 30 | 31 | m. Sui Generis Database Rights means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. 32 | 33 | n. You means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning. 34 | 35 | Section 2 – Scope. 36 | 37 | a. License grant. 38 | 39 | 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: 40 | A. reproduce and Share the Licensed Material, in whole or in part, for NonCommercial purposes only; and 41 | B. produce, reproduce, and Share Adapted Material for NonCommercial purposes only. 42 | 43 | 2. Exceptions and Limitations. For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. 44 | 45 | 3. Term. The term of this Public License is specified in Section 6(a). 46 | 47 | 4. Media and formats; technical modifications allowed. The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. 48 | 49 | 5. Downstream recipients. 50 | A. Offer from the Licensor – Licensed Material. Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. 51 | B. Additional offer from the Licensor – Adapted Material. Every recipient of Adapted Material from You automatically receives an offer from the Licensor to exercise the Licensed Rights in the Adapted Material under the conditions of the Adapter’s License You apply. 52 | C. No downstream restrictions. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. 53 | 54 | 6. No endorsement. Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). 55 | 56 | b. Other rights. 57 | 58 | 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. 59 | 60 | 2. Patent and trademark rights are not licensed under this Public License. 61 | 62 | 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties, including when the Licensed Material is used other than for NonCommercial purposes. 63 | 64 | Section 3 – License Conditions. 65 | 66 | Your exercise of the Licensed Rights is expressly made subject to the following conditions. 67 | 68 | a. Attribution. 69 | 70 | 1. If You Share the Licensed Material (including in modified form), You must: 71 | A. retain the following if it is supplied by the Licensor with the Licensed Material: 72 | i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); 73 | ii. a copyright notice; 74 | iii. a notice that refers to this Public License; 75 | iv. a notice that refers to the disclaimer of warranties; 76 | v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; 77 | B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and 78 | C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. 79 | 80 | 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. 81 | 82 | 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. 83 | 84 | b. ShareAlike. 85 | 86 | In addition to the conditions in Section 3(a), if You Share Adapted Material You produce, the following conditions also apply. 87 | 88 | 1. The Adapter’s License You apply must be a Creative Commons license with the same License Elements, this version or later, or a BY-NC-SA Compatible License. 89 | 90 | 2. You must include the text of, or the URI or hyperlink to, the Adapter's License You apply. You may satisfy this condition in any reasonable manner based on the medium, means, and context in which You Share Adapted Material. 91 | 92 | 3. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, Adapted Material that restrict exercise of the rights granted under the Adapter's License You apply. 93 | 94 | Section 4 – Sui Generis Database Rights. 95 | 96 | Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: 97 | 98 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database for NonCommercial purposes only; 99 | 100 | b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material, including for purposes of Section 3(b); and 101 | 102 | c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. 103 | 104 | For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. 105 | 106 | Section 5 – Disclaimer of Warranties and Limitation of Liability. 107 | 108 | a. Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You. 109 | 110 | b. To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You. 111 | 112 | c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. 113 | 114 | Section 6 – Term and Termination. 115 | 116 | a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. 117 | 118 | b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: 119 | 120 | 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or 121 | 122 | 2. upon express reinstatement by the Licensor. 123 | 124 | For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. 125 | 126 | c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. 127 | 128 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. 129 | 130 | Section 7 – Other Terms and Conditions. 131 | 132 | a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. 133 | 134 | b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. 135 | 136 | Section 8 – Interpretation. 137 | 138 | a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. 139 | 140 | b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. 141 | 142 | c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. 143 | 144 | d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. 145 | 146 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CSVideoNet 2 | This is the implementation of the paper "CSVideoNet: A Recurrent Convolutional Neural Network for Compressive Sensing Video Reconstruction"(https://arxiv.org/abs/1612.05203). 3 | 4 | 1. 0.25_196_ExtractFrame.sh is used to extract frames from videos; 5 | 2. genPhi is used to generate sensing matrix; 6 | 3. generateTrainCNN.m is used to generate image blocks for pre-training key CNN; 7 | 4. "caffe/model/crX" directory contains all the necessary file for training key CNN. To start the training, run "sh trainCNN_crX.sh"; 8 | 5. GenerateTrainData1ChanVarMix.m is used to generate image blocks for training the whole network; 9 | 6. extractFeatures_5_25.m is used to extract the intermediate features extracted by the key CNN, and the extracted features are used for training the whole framework; 10 | 7. "model/crX" directory contains all the files for training the whole framework. The pre-trained key CNN is loaded, further trained with the whole framework using the intermediate feature as input; 11 | To start the training the whole framework, run "th trainVideoNet.lua". 12 | -------------------------------------------------------------------------------- /caffe/model/cr5/recon/GenerateTestData.m: -------------------------------------------------------------------------------- 1 | %data1-feature1, data2 1-original label 1-10 2 | function GenerateTestData(cr) 3 | task = 'Test'; 4 | addpath('/home/user/kaixu/myGitHub/caffe/matlab/'); 5 | 6 | if cr == 5 7 | load /home/user/kaixu/myGitHub/VideoReconNet/phi/phi3_cr5.mat 8 | phi1 = phi3; 9 | elseif cr == 25 10 | load /home/user/kaixu/myGitHub/VideoReconNet/phi/phi3_cr25.mat 11 | phi1 = phi3; 12 | elseif cr == 50 13 | load /home/user/kaixu/myGitHub/VideoReconNet/phi/phi3_cr50.mat 14 | phi1 = phi3; 15 | elseif cr == 100 16 | load /home/user/kaixu/myGitHub/VideoReconNet/phi/phi3_cr100.mat 17 | phi1 = phi3; 18 | end 19 | 20 | phase = 'test'; 21 | 22 | %% 23 | if cr == 5 24 | net_model = 'videoNetCNN_cr5_deploy_10172016.prototxt'; 25 | net_weights = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr5/'... 26 | 'Snapshots/cr_5_CNN_10172016/videoNetCNN_5_iter_175000.caffemodel']; 27 | elseif cr == 25 28 | net_model = ['videoNetCNN_cr25_deploy_10172016.prototxt']; 29 | net_weights = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr25/'... 30 | 'Snapshots/cr_25_CNN_10172016/videoNetCNN_25_iter_186549.caffemodel']; 31 | elseif cr == 50 32 | net_model = 'videoNetCNN_cr50_deploy_10172016.prototxt'; 33 | net_weights = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr50/'... 34 | 'Snapshots/cr_50_CNN_10172016/videoNetCNN_50_iter_338535.caffemodel']; 35 | elseif cr == 100 36 | net_model = 'videoNetCNN_cr100_deploy_10172016.prototxt'; 37 | net_weights = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr100/'... 38 | 'Snapshots/cr_100_CNN_10172016/videoNetCNN_100_iter_320361.caffemodel']; 39 | end 40 | 41 | %% 42 | folder = ['~/myGitHub/datasets/UCF101/', task, 'Data/5_196/test1/group1/']; 43 | savepath = ['data/',task,'Data_',num2str(cr),'_10172016.h5']; 44 | 45 | if strcmp(task, 'Train') == 1 46 | step = 2; 47 | elseif strcmp(task, 'Val') == 1 48 | step = 2; 49 | else 50 | step = 1; 51 | end 52 | 53 | imgHeight = 196; 54 | imgWidth = 196; 55 | numChannels = 1; 56 | 57 | size_input = 32; 58 | size_label = 32; 59 | crop = 18; 60 | stride = size_input; 61 | seq_length = 10; % length of the LSTM 62 | 63 | data = zeros(size(phi1,1), 1, numChannels, 1, 'single'); 64 | label = zeros(size_label, size_label, numChannels, 1, 'single'); 65 | 66 | % padding = abs(size_input - size_label)/2; 67 | count = 1; 68 | 69 | im_label = zeros(imgHeight-2*crop, imgWidth-2*crop, numChannels); 70 | subim_input = zeros(size_input, size_input, numChannels); 71 | subim_label = zeros(size_input, size_input, numChannels); 72 | subim_input_dr = zeros(size(phi1,1), 1, numChannels); 73 | modelOut = zeros(size_input, size_input, numChannels, 1); 74 | 75 | paths = dir(folder); 76 | paths(1:2) = []; 77 | 78 | newPaths = []; 79 | 80 | for i = 1:step:length(paths) 81 | newPaths = [newPaths,paths(i)]; 82 | end 83 | 84 | % h = waitbar(0,'Writing train data to hdf5 file, please wait...'); 85 | 86 | for i = 1:length(newPaths) 87 | filename = dir([folder, newPaths(i).name]); 88 | filename(1:2) = []; % filenames in each subfolder 89 | 90 | len = length(filename); 91 | tmp = floor(len / seq_length); 92 | if tmp >= 3 93 | tmp = 3; 94 | end 95 | numImage = tmp * seq_length; 96 | 97 | for j = 1:1:numImage 98 | rawImg = imread([folder, newPaths(i).name, '/', filename(j).name]); 99 | rawImg = rgb2ycbcr(rawImg); 100 | rawImg = im2double(rawImg(:,:,1)); 101 | rawImg = rawImg(crop+1:end-crop,crop+1:end-crop); 102 | [hei,wid, ~, ~] = size(rawImg); 103 | 104 | for x = 1 : stride : hei-size_input+1 105 | for y = 1 : stride : wid-size_input+1 106 | subim_input = rawImg(x : x+size_input-1, y : y+size_input-1, :); 107 | subim_input_rs = reshape(subim_input, [],numChannels); 108 | for xx = 1:numChannels 109 | subim_input_dr = phi1(:,:,xx) * subim_input_rs(:,xx); 110 | end 111 | subim_label = rawImg(x : x+size_label-1, y : y+size_label-1, :); 112 | 113 | data(:, :, :, count) = subim_input_dr; 114 | label(:, :, :, count) = subim_label; 115 | 116 | count=count+1; 117 | end 118 | end 119 | end 120 | disp([num2str(i/length(newPaths)*100),'%']); 121 | end 122 | 123 | id = 0; 124 | caffe.set_mode_gpu(); 125 | caffe.set_device(id); 126 | net = caffe.Net(net_model, net_weights, phase); 127 | 128 | for i = 1:size(data, 4) 129 | feature = net.forward({data(:,:,:,i)}); 130 | feature = feature{1}; 131 | modelOut(:,:,:,i) = feature; 132 | end 133 | 134 | numImages = size(modelOut,4)/25; 135 | numOrigImages = floor(size(modelOut, 4)/25)*25; 136 | frame = zeros(160, 160, 1, numImages); 137 | frameLabel = zeros(160, 160, 1, numImages); 138 | 139 | dimImage = 32; 140 | for i = 1:numImages 141 | for p = 1:5 142 | for q = 1:5 143 | frame((p-1)*dimImage+1:p*dimImage,(q-1)*dimImage+1:q*dimImage,1,i) ... 144 | = modelOut(:,:,:,(i-1)*25+(p-1)*5+q); 145 | frameLabel((p-1)*dimImage+1:p*dimImage,(q-1)*dimImage+1:q*dimImage,1,i) ... 146 | = label(:,:,:,(i-1)*25+(p-1)*5+q); 147 | end 148 | end 149 | end 150 | 151 | imwrite(frame(:,:,1,190), ['shotPut_recon_', num2str(cr), '.pgm']); 152 | imwrite(frameLabel(:,:,1,190), ['shotPut_base', num2str(cr), '.pgm']); 153 | 154 | imwrite(frame(:,:,1,210), ['skateboard_recon_', num2str(cr), '.pgm']); 155 | imwrite(frameLabel(:,:,1,210), ['skateboard_base_', num2str(cr), '.pgm']); 156 | 157 | imwrite(frame(:,:,1,70), ['guitar_recon_', num2str(cr), '.pgm']); 158 | imwrite(frameLabel(:,:,1,70), ['guitar_base_', num2str(cr), '.pgm']); 159 | 160 | psnrAcc = 0; 161 | ssimAcc = 0; 162 | elAcc = 0; 163 | 164 | index = 1; 165 | videoLength = [30,30,30,30,30,30,20,30]; 166 | psnrAvg = zeros(length(videoLength), 1); 167 | ssimAvg = zeros(length(videoLength), 1); 168 | elAvg = zeros(length(videoLength), 1); 169 | 170 | for i = 1:length(videoLength) 171 | partFrame = frame(:,:,:,index:index+videoLength(i)-1); 172 | partLabel = frameLabel(:,:,:,index:index+videoLength(i)-1); 173 | index = index + videoLength(i); 174 | for j = 1:videoLength(i) 175 | img = partFrame(:,:,:,j); 176 | img = im2uint8(img); 177 | lab = partLabel(:,:,:,j); 178 | lab = im2uint8(lab); 179 | psnrAcc = psnrAcc + psnr(img, lab); 180 | ssimAcc = ssimAcc + ssim(img, lab); 181 | elAcc = elAcc + sum(abs(img(:)-lab(:))); 182 | end 183 | psnrAvg(i) = psnrAcc / videoLength(i); 184 | ssimAvg(i) = ssimAcc / videoLength(i); 185 | elAvg(i) = elAcc / (videoLength(i)*size(frame,3)*size(frame,2)*size(frame,1)); 186 | 187 | psnrAcc = 0; 188 | ssimAcc = 0; 189 | elAcc = 0; 190 | end 191 | save(['measurement', num2str(cr), '.mat'], 'psnrAvg', 'ssimAvg', 'elAvg'); 192 | 193 | end 194 | -------------------------------------------------------------------------------- /caffe/model/cr5/recon/measurement.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PSCLab-ASU/CSVideoNet/99ee2e6f95097223e231c460df51b2cd4fb46b24/caffe/model/cr5/recon/measurement.mat -------------------------------------------------------------------------------- /caffe/model/cr5/recon/measurement100.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PSCLab-ASU/CSVideoNet/99ee2e6f95097223e231c460df51b2cd4fb46b24/caffe/model/cr5/recon/measurement100.mat -------------------------------------------------------------------------------- /caffe/model/cr5/recon/measurement25.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PSCLab-ASU/CSVideoNet/99ee2e6f95097223e231c460df51b2cd4fb46b24/caffe/model/cr5/recon/measurement25.mat -------------------------------------------------------------------------------- /caffe/model/cr5/recon/measurement5.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PSCLab-ASU/CSVideoNet/99ee2e6f95097223e231c460df51b2cd4fb46b24/caffe/model/cr5/recon/measurement5.mat -------------------------------------------------------------------------------- /caffe/model/cr5/recon/measurement50.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PSCLab-ASU/CSVideoNet/99ee2e6f95097223e231c460df51b2cd4fb46b24/caffe/model/cr5/recon/measurement50.mat -------------------------------------------------------------------------------- /caffe/model/cr5/recon/store2hdf5.m: -------------------------------------------------------------------------------- 1 | function [curr_dat_sz, curr_lab_sz] = store2hdf5(filename, data, labels, create, startloc, chunksz) 2 | % *data* is W*H*C*N matrix of images should be normalized (e.g. to lie between 0 and 1) beforehand 3 | % *label* is D*N matrix of labels (D labels per sample) 4 | % *create* [0/1] specifies whether to create file newly or to append to previously created file, useful to store information in batches when a dataset is too big to be held in memory (default: 1) 5 | % *startloc* (point at which to start writing data). By default, 6 | % if create=1 (create mode), startloc.data=[1 1 1 1], and startloc.lab=[1 1]; 7 | % if create=0 (append mode), startloc.data=[1 1 1 K+1], and startloc.lab = [1 K+1]; where K is the current number of samples stored in the HDF 8 | % chunksz (used only in create mode), specifies number of samples to be stored per chunk (see HDF5 documentation on chunking) for creating HDF5 files with unbounded maximum size - TLDR; higher chunk sizes allow faster read-write operations 9 | 10 | % verify that format is right 11 | dat_dims=size(data); 12 | lab_dims=size(labels); 13 | num_samples=dat_dims(end); 14 | 15 | assert(lab_dims(end)==num_samples, 'Number of samples should be matched between data and labels'); 16 | 17 | if ~exist('create','var') 18 | create=true; 19 | end 20 | 21 | 22 | if create 23 | %fprintf('Creating dataset with %d samples\n', num_samples); 24 | if ~exist('chunksz', 'var') 25 | chunksz=1000; 26 | end 27 | if exist(filename, 'file') 28 | fprintf('Warning: replacing existing file %s \n', filename); 29 | delete(filename); 30 | end 31 | % h5create(filename, '/data', [dat_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [dat_dims(1:end-1) chunksz]); % width, height, channels, number 32 | % h5create(filename, '/label', [lab_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [lab_dims(1:end-1) chunksz]); % width, height, channels, number 33 | h5create(filename, '/data', [dat_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [dat_dims(1:end-1) chunksz]); % width, height, channels, number 34 | h5create(filename, '/label', [lab_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [lab_dims(1:end-1) chunksz]); % width, height, channels, number 35 | if ~exist('startloc','var') 36 | startloc.dat=[ones(1,length(dat_dims)-1), 1]; 37 | startloc.lab=[ones(1,length(lab_dims)-1), 1]; 38 | end 39 | else % append mode 40 | if ~exist('startloc','var') 41 | info=h5info(filename); 42 | prev_dat_sz=info.Datasets(1).Dataspace.Size; 43 | prev_lab_sz=info.Datasets(2).Dataspace.Size; 44 | assert(prev_dat_sz(1:end-1)==dat_dims(1:end-1), 'Data dimensions must match existing dimensions in dataset'); 45 | assert(prev_lab_sz(1:end-1)==lab_dims(1:end-1), 'Label dimensions must match existing dimensions in dataset'); 46 | startloc.dat=[ones(1,length(dat_dims)-1), prev_dat_sz(end)+1]; 47 | startloc.lab=[ones(1,length(lab_dims)-1), prev_lab_sz(end)+1]; 48 | end 49 | end 50 | 51 | if ~isempty(data) 52 | % h5write(filename, '/data', single(data), startloc.dat, size(data)); 53 | % h5write(filename, '/label', single(labels), startloc.lab, size(labels)); 54 | h5write(filename, '/data', data, startloc.dat, size(data)); 55 | h5write(filename, '/label', labels, startloc.lab, size(labels)); 56 | end 57 | 58 | if nargout 59 | info=h5info(filename); 60 | curr_dat_sz=info.Datasets(1).Dataspace.Size; 61 | curr_lab_sz=info.Datasets(2).Dataspace.Size; 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /caffe/model/cr5/recon/videoNetCNN_cr100_deploy_10172016.prototxt: -------------------------------------------------------------------------------- 1 | layer { 2 | name: "data" 3 | type: "Input" 4 | top: "data" 5 | input_param { shape: { dim: 1 dim: 1 dim: 1 dim: 10 } } 6 | } 7 | 8 | layer { 9 | name: "fc1" 10 | type: "InnerProduct" 11 | bottom: "data" 12 | top: "fc1" 13 | param { 14 | lr_mult: 1 15 | decay_mult: 1 16 | } 17 | param { 18 | lr_mult: 2 19 | decay_mult: 0 20 | } 21 | inner_product_param { 22 | num_output: 1024 23 | weight_filler { 24 | type: "gaussian" 25 | std: 0.01 #For different measurement rates, different values work better. For MR = 0.25, MR = 0.04 and MR = 0.01, one can use 0.05, 0.01 and 0.03 respectively. 26 | } 27 | bias_filler { 28 | type: "constant" 29 | value: 0 30 | } 31 | } 32 | } 33 | layer { 34 | name: "reshape" 35 | type: "Reshape" 36 | bottom: "fc1" 37 | top: "reshape" 38 | reshape_param { 39 | shape { 40 | dim: 0 41 | dim: 1 42 | dim: 32 43 | dim: 32 44 | } 45 | } 46 | } 47 | layer { 48 | name: "conv1" 49 | type: "Convolution" 50 | bottom: "reshape" 51 | top: "conv1" 52 | param { 53 | lr_mult: 1 54 | } 55 | param { 56 | lr_mult: 0.1 57 | } 58 | convolution_param { 59 | num_output: 128 60 | pad: 0 61 | kernel_size: 1 62 | stride: 1 63 | weight_filler { 64 | type: "gaussian" 65 | std: 0.1 66 | } 67 | bias_filler { 68 | type: "constant" 69 | value: 0 70 | } 71 | } 72 | } 73 | layer { 74 | name: "relu1" 75 | type: "ReLU" 76 | bottom: "conv1" 77 | top: "relu1" 78 | } 79 | layer { 80 | name: "conv2" 81 | type: "Convolution" 82 | bottom: "relu1" 83 | top: "conv2" 84 | param { 85 | lr_mult: 1 86 | } 87 | param { 88 | lr_mult: 0.1 89 | } 90 | convolution_param { 91 | num_output: 64 92 | pad: 0 93 | kernel_size: 1 94 | stride: 1 95 | weight_filler { 96 | type: "gaussian" 97 | std: 0.1 98 | } 99 | bias_filler { 100 | type: "constant" 101 | value: 0 102 | } 103 | } 104 | } 105 | layer { 106 | name: "relu2" 107 | type: "ReLU" 108 | bottom: "conv2" 109 | top: "relu2" 110 | } 111 | layer { 112 | name: "conv3" 113 | type: "Convolution" 114 | bottom: "relu2" 115 | top: "conv3" 116 | param { 117 | lr_mult: 0.1 118 | } 119 | param { 120 | lr_mult: 0.1 121 | } 122 | convolution_param { 123 | num_output: 32 124 | pad: 1 125 | kernel_size: 3 126 | stride: 1 127 | weight_filler { 128 | type: "gaussian" 129 | std: 0.1 130 | } 131 | bias_filler { 132 | type: "constant" 133 | value: 0 134 | } 135 | } 136 | } 137 | layer { 138 | name: "relu3" 139 | type: "ReLU" 140 | bottom: "conv3" 141 | top: "relu3" 142 | } 143 | layer { 144 | name: "conv4" 145 | type: "Convolution" 146 | bottom: "relu3" 147 | top: "conv4" 148 | param { 149 | lr_mult: 1 150 | } 151 | param { 152 | lr_mult: 0.1 153 | } 154 | convolution_param { 155 | num_output: 32 156 | pad: 1 157 | kernel_size: 3 158 | stride: 1 159 | weight_filler { 160 | type: "gaussian" 161 | std: 0.1 162 | } 163 | bias_filler { 164 | type: "constant" 165 | value: 0 166 | } 167 | } 168 | } 169 | layer { 170 | name: "relu4" 171 | type: "ReLU" 172 | bottom: "conv4" 173 | top: "relu4" 174 | } 175 | layer { 176 | name: "conv5" 177 | type: "Convolution" 178 | bottom: "relu4" 179 | top: "conv5" 180 | param { 181 | lr_mult: 1 182 | } 183 | param { 184 | lr_mult: 0.1 185 | } 186 | convolution_param { 187 | num_output: 16 188 | pad: 1 189 | kernel_size: 3 190 | stride: 1 191 | weight_filler { 192 | type: "gaussian" 193 | std: 0.1 194 | } 195 | bias_filler { 196 | type: "constant" 197 | value: 0 198 | } 199 | } 200 | } 201 | layer { 202 | name: "relu5" 203 | type: "ReLU" 204 | bottom: "conv5" 205 | top: "relu5" 206 | } 207 | 208 | layer { 209 | name: "conv6" 210 | type: "Convolution" 211 | bottom: "relu5" 212 | top: "conv6" 213 | param { 214 | lr_mult: 0.1 215 | } 216 | param { 217 | lr_mult: 0.1 218 | } 219 | convolution_param { 220 | num_output: 16 221 | pad: 1 222 | kernel_size: 3 223 | stride: 1 224 | weight_filler { 225 | type: "gaussian" 226 | std: 0.1 227 | } 228 | bias_filler { 229 | type: "constant" 230 | value: 0 231 | } 232 | } 233 | } 234 | layer { 235 | name: "relu6" 236 | type: "ReLU" 237 | bottom: "conv6" 238 | top: "relu6" 239 | } 240 | 241 | layer { 242 | name: "conv7" 243 | type: "Convolution" 244 | bottom: "relu6" 245 | top: "conv7" 246 | param { 247 | lr_mult: 0.1 248 | } 249 | param { 250 | lr_mult: 0.1 251 | } 252 | convolution_param { 253 | num_output: 1 254 | pad: 1 255 | kernel_size: 3 256 | stride: 1 257 | weight_filler { 258 | type: "gaussian" 259 | std: 0.1 260 | } 261 | bias_filler { 262 | type: "constant" 263 | value: 0 264 | } 265 | } 266 | } 267 | -------------------------------------------------------------------------------- /caffe/model/cr5/recon/videoNetCNN_cr25_deploy_10172016.prototxt: -------------------------------------------------------------------------------- 1 | layer { 2 | name: "data" 3 | type: "Input" 4 | top: "data" 5 | input_param { shape: { dim: 1 dim: 1 dim: 1 dim: 40 } } 6 | } 7 | 8 | layer { 9 | name: "fc1" 10 | type: "InnerProduct" 11 | bottom: "data" 12 | top: "fc1" 13 | param { 14 | lr_mult: 1 15 | decay_mult: 1 16 | } 17 | param { 18 | lr_mult: 2 19 | decay_mult: 0 20 | } 21 | inner_product_param { 22 | num_output: 1024 23 | weight_filler { 24 | type: "gaussian" 25 | std: 0.01 #For different measurement rates, different values work better. For MR = 0.25, MR = 0.04 and MR = 0.01, one can use 0.05, 0.01 and 0.03 respectively. 26 | } 27 | bias_filler { 28 | type: "constant" 29 | value: 0 30 | } 31 | } 32 | } 33 | layer { 34 | name: "reshape" 35 | type: "Reshape" 36 | bottom: "fc1" 37 | top: "reshape" 38 | reshape_param { 39 | shape { 40 | dim: 0 41 | dim: 1 42 | dim: 32 43 | dim: 32 44 | } 45 | } 46 | } 47 | layer { 48 | name: "conv1" 49 | type: "Convolution" 50 | bottom: "reshape" 51 | top: "conv1" 52 | param { 53 | lr_mult: 1 54 | } 55 | param { 56 | lr_mult: 0.1 57 | } 58 | convolution_param { 59 | num_output: 128 60 | pad: 0 61 | kernel_size: 1 62 | stride: 1 63 | weight_filler { 64 | type: "gaussian" 65 | std: 0.1 66 | } 67 | bias_filler { 68 | type: "constant" 69 | value: 0 70 | } 71 | } 72 | } 73 | layer { 74 | name: "relu1" 75 | type: "ReLU" 76 | bottom: "conv1" 77 | top: "relu1" 78 | } 79 | layer { 80 | name: "conv2" 81 | type: "Convolution" 82 | bottom: "relu1" 83 | top: "conv2" 84 | param { 85 | lr_mult: 1 86 | } 87 | param { 88 | lr_mult: 0.1 89 | } 90 | convolution_param { 91 | num_output: 64 92 | pad: 0 93 | kernel_size: 1 94 | stride: 1 95 | weight_filler { 96 | type: "gaussian" 97 | std: 0.1 98 | } 99 | bias_filler { 100 | type: "constant" 101 | value: 0 102 | } 103 | } 104 | } 105 | layer { 106 | name: "relu2" 107 | type: "ReLU" 108 | bottom: "conv2" 109 | top: "relu2" 110 | } 111 | layer { 112 | name: "conv3" 113 | type: "Convolution" 114 | bottom: "relu2" 115 | top: "conv3" 116 | param { 117 | lr_mult: 0.1 118 | } 119 | param { 120 | lr_mult: 0.1 121 | } 122 | convolution_param { 123 | num_output: 32 124 | pad: 1 125 | kernel_size: 3 126 | stride: 1 127 | weight_filler { 128 | type: "gaussian" 129 | std: 0.1 130 | } 131 | bias_filler { 132 | type: "constant" 133 | value: 0 134 | } 135 | } 136 | } 137 | layer { 138 | name: "relu3" 139 | type: "ReLU" 140 | bottom: "conv3" 141 | top: "relu3" 142 | } 143 | layer { 144 | name: "conv4" 145 | type: "Convolution" 146 | bottom: "relu3" 147 | top: "conv4" 148 | param { 149 | lr_mult: 1 150 | } 151 | param { 152 | lr_mult: 0.1 153 | } 154 | convolution_param { 155 | num_output: 32 156 | pad: 1 157 | kernel_size: 3 158 | stride: 1 159 | weight_filler { 160 | type: "gaussian" 161 | std: 0.1 162 | } 163 | bias_filler { 164 | type: "constant" 165 | value: 0 166 | } 167 | } 168 | } 169 | layer { 170 | name: "relu4" 171 | type: "ReLU" 172 | bottom: "conv4" 173 | top: "relu4" 174 | } 175 | layer { 176 | name: "conv5" 177 | type: "Convolution" 178 | bottom: "relu4" 179 | top: "conv5" 180 | param { 181 | lr_mult: 1 182 | } 183 | param { 184 | lr_mult: 0.1 185 | } 186 | convolution_param { 187 | num_output: 16 188 | pad: 1 189 | kernel_size: 3 190 | stride: 1 191 | weight_filler { 192 | type: "gaussian" 193 | std: 0.1 194 | } 195 | bias_filler { 196 | type: "constant" 197 | value: 0 198 | } 199 | } 200 | } 201 | layer { 202 | name: "relu5" 203 | type: "ReLU" 204 | bottom: "conv5" 205 | top: "relu5" 206 | } 207 | 208 | layer { 209 | name: "conv6" 210 | type: "Convolution" 211 | bottom: "relu5" 212 | top: "conv6" 213 | param { 214 | lr_mult: 0.1 215 | } 216 | param { 217 | lr_mult: 0.1 218 | } 219 | convolution_param { 220 | num_output: 16 221 | pad: 1 222 | kernel_size: 3 223 | stride: 1 224 | weight_filler { 225 | type: "gaussian" 226 | std: 0.1 227 | } 228 | bias_filler { 229 | type: "constant" 230 | value: 0 231 | } 232 | } 233 | } 234 | layer { 235 | name: "relu6" 236 | type: "ReLU" 237 | bottom: "conv6" 238 | top: "relu6" 239 | } 240 | 241 | layer { 242 | name: "conv7" 243 | type: "Convolution" 244 | bottom: "relu6" 245 | top: "conv7" 246 | param { 247 | lr_mult: 0.1 248 | } 249 | param { 250 | lr_mult: 0.1 251 | } 252 | convolution_param { 253 | num_output: 1 254 | pad: 1 255 | kernel_size: 3 256 | stride: 1 257 | weight_filler { 258 | type: "gaussian" 259 | std: 0.1 260 | } 261 | bias_filler { 262 | type: "constant" 263 | value: 0 264 | } 265 | } 266 | } 267 | 268 | -------------------------------------------------------------------------------- /caffe/model/cr5/recon/videoNetCNN_cr50_deploy_10172016.prototxt: -------------------------------------------------------------------------------- 1 | layer { 2 | name: "data" 3 | type: "Input" 4 | top: "data" 5 | input_param { shape: { dim: 1 dim: 1 dim: 1 dim: 20 } } 6 | } 7 | 8 | layer { 9 | name: "fc1" 10 | type: "InnerProduct" 11 | bottom: "data" 12 | top: "fc1" 13 | param { 14 | lr_mult: 1 15 | decay_mult: 1 16 | } 17 | param { 18 | lr_mult: 2 19 | decay_mult: 0 20 | } 21 | inner_product_param { 22 | num_output: 1024 23 | weight_filler { 24 | type: "gaussian" 25 | std: 0.01 #For different measurement rates, different values work better. For MR = 0.25, MR = 0.04 and MR = 0.01, one can use 0.05, 0.01 and 0.03 respectively. 26 | } 27 | bias_filler { 28 | type: "constant" 29 | value: 0 30 | } 31 | } 32 | } 33 | layer { 34 | name: "reshape" 35 | type: "Reshape" 36 | bottom: "fc1" 37 | top: "reshape" 38 | reshape_param { 39 | shape { 40 | dim: 0 41 | dim: 1 42 | dim: 32 43 | dim: 32 44 | } 45 | } 46 | } 47 | layer { 48 | name: "conv1" 49 | type: "Convolution" 50 | bottom: "reshape" 51 | top: "conv1" 52 | param { 53 | lr_mult: 1 54 | } 55 | param { 56 | lr_mult: 0.1 57 | } 58 | convolution_param { 59 | num_output: 128 60 | pad: 0 61 | kernel_size: 1 62 | stride: 1 63 | weight_filler { 64 | type: "gaussian" 65 | std: 0.1 66 | } 67 | bias_filler { 68 | type: "constant" 69 | value: 0 70 | } 71 | } 72 | } 73 | layer { 74 | name: "relu1" 75 | type: "ReLU" 76 | bottom: "conv1" 77 | top: "relu1" 78 | } 79 | layer { 80 | name: "conv2" 81 | type: "Convolution" 82 | bottom: "relu1" 83 | top: "conv2" 84 | param { 85 | lr_mult: 1 86 | } 87 | param { 88 | lr_mult: 0.1 89 | } 90 | convolution_param { 91 | num_output: 64 92 | pad: 0 93 | kernel_size: 1 94 | stride: 1 95 | weight_filler { 96 | type: "gaussian" 97 | std: 0.1 98 | } 99 | bias_filler { 100 | type: "constant" 101 | value: 0 102 | } 103 | } 104 | } 105 | layer { 106 | name: "relu2" 107 | type: "ReLU" 108 | bottom: "conv2" 109 | top: "relu2" 110 | } 111 | layer { 112 | name: "conv3" 113 | type: "Convolution" 114 | bottom: "relu2" 115 | top: "conv3" 116 | param { 117 | lr_mult: 0.1 118 | } 119 | param { 120 | lr_mult: 0.1 121 | } 122 | convolution_param { 123 | num_output: 32 124 | pad: 1 125 | kernel_size: 3 126 | stride: 1 127 | weight_filler { 128 | type: "gaussian" 129 | std: 0.1 130 | } 131 | bias_filler { 132 | type: "constant" 133 | value: 0 134 | } 135 | } 136 | } 137 | layer { 138 | name: "relu3" 139 | type: "ReLU" 140 | bottom: "conv3" 141 | top: "relu3" 142 | } 143 | layer { 144 | name: "conv4" 145 | type: "Convolution" 146 | bottom: "relu3" 147 | top: "conv4" 148 | param { 149 | lr_mult: 1 150 | } 151 | param { 152 | lr_mult: 0.1 153 | } 154 | convolution_param { 155 | num_output: 32 156 | pad: 1 157 | kernel_size: 3 158 | stride: 1 159 | weight_filler { 160 | type: "gaussian" 161 | std: 0.1 162 | } 163 | bias_filler { 164 | type: "constant" 165 | value: 0 166 | } 167 | } 168 | } 169 | layer { 170 | name: "relu4" 171 | type: "ReLU" 172 | bottom: "conv4" 173 | top: "relu4" 174 | } 175 | layer { 176 | name: "conv5" 177 | type: "Convolution" 178 | bottom: "relu4" 179 | top: "conv5" 180 | param { 181 | lr_mult: 1 182 | } 183 | param { 184 | lr_mult: 0.1 185 | } 186 | convolution_param { 187 | num_output: 16 188 | pad: 1 189 | kernel_size: 3 190 | stride: 1 191 | weight_filler { 192 | type: "gaussian" 193 | std: 0.1 194 | } 195 | bias_filler { 196 | type: "constant" 197 | value: 0 198 | } 199 | } 200 | } 201 | layer { 202 | name: "relu5" 203 | type: "ReLU" 204 | bottom: "conv5" 205 | top: "relu5" 206 | } 207 | 208 | layer { 209 | name: "conv6" 210 | type: "Convolution" 211 | bottom: "relu5" 212 | top: "conv6" 213 | param { 214 | lr_mult: 0.1 215 | } 216 | param { 217 | lr_mult: 0.1 218 | } 219 | convolution_param { 220 | num_output: 16 221 | pad: 1 222 | kernel_size: 3 223 | stride: 1 224 | weight_filler { 225 | type: "gaussian" 226 | std: 0.1 227 | } 228 | bias_filler { 229 | type: "constant" 230 | value: 0 231 | } 232 | } 233 | } 234 | layer { 235 | name: "relu6" 236 | type: "ReLU" 237 | bottom: "conv6" 238 | top: "relu6" 239 | } 240 | 241 | layer { 242 | name: "conv7" 243 | type: "Convolution" 244 | bottom: "relu6" 245 | top: "conv7" 246 | param { 247 | lr_mult: 0.1 248 | } 249 | param { 250 | lr_mult: 0.1 251 | } 252 | convolution_param { 253 | num_output: 1 254 | pad: 1 255 | kernel_size: 3 256 | stride: 1 257 | weight_filler { 258 | type: "gaussian" 259 | std: 0.1 260 | } 261 | bias_filler { 262 | type: "constant" 263 | value: 0 264 | } 265 | } 266 | } 267 | -------------------------------------------------------------------------------- /caffe/model/cr5/recon/videoNetCNN_cr5_deploy_10172016.prototxt: -------------------------------------------------------------------------------- 1 | layer { 2 | name: "data" 3 | type: "Input" 4 | top: "data" 5 | input_param { shape: { dim: 1 dim: 1 dim: 1 dim: 204 } } 6 | } 7 | 8 | layer { 9 | name: "fc1" 10 | type: "InnerProduct" 11 | bottom: "data" 12 | top: "fc1" 13 | param { 14 | lr_mult: 1 15 | decay_mult: 1 16 | } 17 | param { 18 | lr_mult: 2 19 | decay_mult: 0 20 | } 21 | inner_product_param { 22 | num_output: 1024 23 | weight_filler { 24 | type: "gaussian" 25 | std: 0.01 #For different measurement rates, different values work better. For MR = 0.25, MR = 0.04 and MR = 0.01, one can use 0.05, 0.01 and 0.03 respectively. 26 | } 27 | bias_filler { 28 | type: "constant" 29 | value: 0 30 | } 31 | } 32 | } 33 | layer { 34 | name: "reshape" 35 | type: "Reshape" 36 | bottom: "fc1" 37 | top: "reshape" 38 | reshape_param { 39 | shape { 40 | dim: 0 41 | dim: 1 42 | dim: 32 43 | dim: 32 44 | } 45 | } 46 | } 47 | layer { 48 | name: "conv1" 49 | type: "Convolution" 50 | bottom: "reshape" 51 | top: "conv1" 52 | param { 53 | lr_mult: 1 54 | } 55 | param { 56 | lr_mult: 0.1 57 | } 58 | convolution_param { 59 | num_output: 128 60 | pad: 0 61 | kernel_size: 1 62 | stride: 1 63 | weight_filler { 64 | type: "gaussian" 65 | std: 0.1 66 | } 67 | bias_filler { 68 | type: "constant" 69 | value: 0 70 | } 71 | } 72 | } 73 | layer { 74 | name: "relu1" 75 | type: "ReLU" 76 | bottom: "conv1" 77 | top: "relu1" 78 | } 79 | layer { 80 | name: "conv2" 81 | type: "Convolution" 82 | bottom: "relu1" 83 | top: "conv2" 84 | param { 85 | lr_mult: 1 86 | } 87 | param { 88 | lr_mult: 0.1 89 | } 90 | convolution_param { 91 | num_output: 64 92 | pad: 0 93 | kernel_size: 1 94 | stride: 1 95 | weight_filler { 96 | type: "gaussian" 97 | std: 0.1 98 | } 99 | bias_filler { 100 | type: "constant" 101 | value: 0 102 | } 103 | } 104 | } 105 | layer { 106 | name: "relu2" 107 | type: "ReLU" 108 | bottom: "conv2" 109 | top: "relu2" 110 | } 111 | layer { 112 | name: "conv3" 113 | type: "Convolution" 114 | bottom: "relu2" 115 | top: "conv3" 116 | param { 117 | lr_mult: 0.1 118 | } 119 | param { 120 | lr_mult: 0.1 121 | } 122 | convolution_param { 123 | num_output: 32 124 | pad: 1 125 | kernel_size: 3 126 | stride: 1 127 | weight_filler { 128 | type: "gaussian" 129 | std: 0.1 130 | } 131 | bias_filler { 132 | type: "constant" 133 | value: 0 134 | } 135 | } 136 | } 137 | layer { 138 | name: "relu3" 139 | type: "ReLU" 140 | bottom: "conv3" 141 | top: "relu3" 142 | } 143 | layer { 144 | name: "conv4" 145 | type: "Convolution" 146 | bottom: "relu3" 147 | top: "conv4" 148 | param { 149 | lr_mult: 1 150 | } 151 | param { 152 | lr_mult: 0.1 153 | } 154 | convolution_param { 155 | num_output: 32 156 | pad: 1 157 | kernel_size: 3 158 | stride: 1 159 | weight_filler { 160 | type: "gaussian" 161 | std: 0.1 162 | } 163 | bias_filler { 164 | type: "constant" 165 | value: 0 166 | } 167 | } 168 | } 169 | layer { 170 | name: "relu4" 171 | type: "ReLU" 172 | bottom: "conv4" 173 | top: "relu4" 174 | } 175 | layer { 176 | name: "conv5" 177 | type: "Convolution" 178 | bottom: "relu4" 179 | top: "conv5" 180 | param { 181 | lr_mult: 1 182 | } 183 | param { 184 | lr_mult: 0.1 185 | } 186 | convolution_param { 187 | num_output: 16 188 | pad: 1 189 | kernel_size: 3 190 | stride: 1 191 | weight_filler { 192 | type: "gaussian" 193 | std: 0.1 194 | } 195 | bias_filler { 196 | type: "constant" 197 | value: 0 198 | } 199 | } 200 | } 201 | layer { 202 | name: "relu5" 203 | type: "ReLU" 204 | bottom: "conv5" 205 | top: "relu5" 206 | } 207 | 208 | layer { 209 | name: "conv6" 210 | type: "Convolution" 211 | bottom: "relu5" 212 | top: "conv6" 213 | param { 214 | lr_mult: 0.1 215 | } 216 | param { 217 | lr_mult: 0.1 218 | } 219 | convolution_param { 220 | num_output: 16 221 | pad: 1 222 | kernel_size: 3 223 | stride: 1 224 | weight_filler { 225 | type: "gaussian" 226 | std: 0.1 227 | } 228 | bias_filler { 229 | type: "constant" 230 | value: 0 231 | } 232 | } 233 | } 234 | layer { 235 | name: "relu6" 236 | type: "ReLU" 237 | bottom: "conv6" 238 | top: "relu6" 239 | } 240 | 241 | layer { 242 | name: "conv7" 243 | type: "Convolution" 244 | bottom: "relu6" 245 | top: "conv7" 246 | param { 247 | lr_mult: 0.1 248 | } 249 | param { 250 | lr_mult: 0.1 251 | } 252 | convolution_param { 253 | num_output: 1 254 | pad: 1 255 | kernel_size: 3 256 | stride: 1 257 | weight_filler { 258 | type: "gaussian" 259 | std: 0.1 260 | } 261 | bias_filler { 262 | type: "constant" 263 | value: 0 264 | } 265 | } 266 | } 267 | 268 | -------------------------------------------------------------------------------- /caffe/model/cr5/trainCNN_cr5.sh: -------------------------------------------------------------------------------- 1 | mkdir Snapshots/cr_5_CNN_10172016 #Create a folder to save the caffe models (measurement rate is 0.04) 2 | ../../../../../build/tools/caffe train -solver videoNetCNN_cr5_solver.prototxt -gpu 2 3 | 4 | -------------------------------------------------------------------------------- /caffe/model/cr5/trainCNN_cr5.txt: -------------------------------------------------------------------------------- 1 | /home/user/kaixu/myGitHub/caffe/examples/videoNet/training/data/TrainDataCNN_5_10172016.h5 2 | -------------------------------------------------------------------------------- /caffe/model/cr5/valCNN_cr5.txt: -------------------------------------------------------------------------------- 1 | /home/user/kaixu/myGitHub/caffe/examples/videoNet/training/data/ValDataCNN_5_10172016.h5 2 | -------------------------------------------------------------------------------- /caffe/model/cr5/videoNetCNN_cr5_deploy_10172016.prototxt: -------------------------------------------------------------------------------- 1 | layer { 2 | name: "data" 3 | type: "Input" 4 | top: "data" 5 | input_param { shape: { dim: 20 dim: 1 dim: 1 dim: 204 } } 6 | } 7 | 8 | layer { 9 | name: "fc1" 10 | type: "InnerProduct" 11 | bottom: "data" 12 | top: "fc1" 13 | param { 14 | lr_mult: 1 15 | decay_mult: 1 16 | } 17 | param { 18 | lr_mult: 2 19 | decay_mult: 0 20 | } 21 | inner_product_param { 22 | num_output: 1024 23 | weight_filler { 24 | type: "gaussian" 25 | std: 0.01 #For different measurement rates, different values work better. For MR = 0.25, MR = 0.04 and MR = 0.01, one can use 0.05, 0.01 and 0.03 respectively. 26 | } 27 | bias_filler { 28 | type: "constant" 29 | value: 0 30 | } 31 | } 32 | } 33 | layer { 34 | name: "reshape" 35 | type: "Reshape" 36 | bottom: "fc1" 37 | top: "reshape" 38 | reshape_param { 39 | shape { 40 | dim: 0 41 | dim: 1 42 | dim: 32 43 | dim: 32 44 | } 45 | } 46 | } 47 | layer { 48 | name: "conv1" 49 | type: "Convolution" 50 | bottom: "reshape" 51 | top: "conv1" 52 | param { 53 | lr_mult: 1 54 | } 55 | param { 56 | lr_mult: 0.1 57 | } 58 | convolution_param { 59 | num_output: 128 60 | pad: 0 61 | kernel_size: 1 62 | stride: 1 63 | weight_filler { 64 | type: "gaussian" 65 | std: 0.1 66 | } 67 | bias_filler { 68 | type: "constant" 69 | value: 0 70 | } 71 | } 72 | } 73 | layer { 74 | name: "relu1" 75 | type: "ReLU" 76 | bottom: "conv1" 77 | top: "relu1" 78 | } 79 | layer { 80 | name: "conv2" 81 | type: "Convolution" 82 | bottom: "relu1" 83 | top: "conv2" 84 | param { 85 | lr_mult: 1 86 | } 87 | param { 88 | lr_mult: 0.1 89 | } 90 | convolution_param { 91 | num_output: 64 92 | pad: 0 93 | kernel_size: 1 94 | stride: 1 95 | weight_filler { 96 | type: "gaussian" 97 | std: 0.1 98 | } 99 | bias_filler { 100 | type: "constant" 101 | value: 0 102 | } 103 | } 104 | } 105 | layer { 106 | name: "relu2" 107 | type: "ReLU" 108 | bottom: "conv2" 109 | top: "relu2" 110 | } 111 | layer { 112 | name: "conv3" 113 | type: "Convolution" 114 | bottom: "relu2" 115 | top: "conv3" 116 | param { 117 | lr_mult: 0.1 118 | } 119 | param { 120 | lr_mult: 0.1 121 | } 122 | convolution_param { 123 | num_output: 32 124 | pad: 1 125 | kernel_size: 3 126 | stride: 1 127 | weight_filler { 128 | type: "gaussian" 129 | std: 0.1 130 | } 131 | bias_filler { 132 | type: "constant" 133 | value: 0 134 | } 135 | } 136 | } 137 | layer { 138 | name: "relu3" 139 | type: "ReLU" 140 | bottom: "conv3" 141 | top: "relu3" 142 | } 143 | layer { 144 | name: "conv4" 145 | type: "Convolution" 146 | bottom: "relu3" 147 | top: "conv4" 148 | param { 149 | lr_mult: 1 150 | } 151 | param { 152 | lr_mult: 0.1 153 | } 154 | convolution_param { 155 | num_output: 32 156 | pad: 1 157 | kernel_size: 3 158 | stride: 1 159 | weight_filler { 160 | type: "gaussian" 161 | std: 0.1 162 | } 163 | bias_filler { 164 | type: "constant" 165 | value: 0 166 | } 167 | } 168 | } 169 | layer { 170 | name: "relu4" 171 | type: "ReLU" 172 | bottom: "conv4" 173 | top: "relu4" 174 | } 175 | layer { 176 | name: "conv5" 177 | type: "Convolution" 178 | bottom: "relu4" 179 | top: "conv5" 180 | param { 181 | lr_mult: 1 182 | } 183 | param { 184 | lr_mult: 0.1 185 | } 186 | convolution_param { 187 | num_output: 16 188 | pad: 1 189 | kernel_size: 3 190 | stride: 1 191 | weight_filler { 192 | type: "gaussian" 193 | std: 0.1 194 | } 195 | bias_filler { 196 | type: "constant" 197 | value: 0 198 | } 199 | } 200 | } 201 | layer { 202 | name: "relu5" 203 | type: "ReLU" 204 | bottom: "conv5" 205 | top: "relu5" 206 | } 207 | 208 | layer { 209 | name: "conv6" 210 | type: "Convolution" 211 | bottom: "relu5" 212 | top: "conv6" 213 | param { 214 | lr_mult: 0.1 215 | } 216 | param { 217 | lr_mult: 0.1 218 | } 219 | convolution_param { 220 | num_output: 16 221 | pad: 1 222 | kernel_size: 3 223 | stride: 1 224 | weight_filler { 225 | type: "gaussian" 226 | std: 0.1 227 | } 228 | bias_filler { 229 | type: "constant" 230 | value: 0 231 | } 232 | } 233 | } 234 | layer { 235 | name: "relu6" 236 | type: "ReLU" 237 | bottom: "conv6" 238 | top: "relu6" 239 | } 240 | 241 | layer { 242 | name: "conv7" 243 | type: "Convolution" 244 | bottom: "relu6" 245 | top: "conv7" 246 | param { 247 | lr_mult: 0.1 248 | } 249 | param { 250 | lr_mult: 0.1 251 | } 252 | convolution_param { 253 | num_output: 1 254 | pad: 1 255 | kernel_size: 3 256 | stride: 1 257 | weight_filler { 258 | type: "gaussian" 259 | std: 0.1 260 | } 261 | bias_filler { 262 | type: "constant" 263 | value: 0 264 | } 265 | } 266 | } 267 | 268 | -------------------------------------------------------------------------------- /caffe/model/cr5/videoNetCNN_cr5_deploy_10172016.prototxt.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | local model = {} 3 | -- warning: module 'data [type HDF5Data]' not found 4 | table.insert(model, {'torch_view', nn.View(-1):setNumInputDims(3)}) 5 | table.insert(model, {'fc1', nn.Linear(204, 1024)}) 6 | -- warning: module 'reshape [type Reshape]' not found 7 | table.insert(model, {'conv1', nn.SpatialConvolution(1, 128, 1, 1, 1, 1, 0, 0)}) 8 | table.insert(model, {'relu1', nn.ReLU(true)}) 9 | table.insert(model, {'conv2', nn.SpatialConvolution(128, 64, 1, 1, 1, 1, 0, 0)}) 10 | table.insert(model, {'relu2', nn.ReLU(true)}) 11 | table.insert(model, {'conv3', nn.SpatialConvolution(64, 32, 3, 3, 1, 1, 1, 1)}) 12 | table.insert(model, {'relu3', nn.ReLU(true)}) 13 | table.insert(model, {'conv4', nn.SpatialConvolution(32, 32, 3, 3, 1, 1, 1, 1)}) 14 | table.insert(model, {'relu4', nn.ReLU(true)}) 15 | table.insert(model, {'conv5', nn.SpatialConvolution(32, 16, 3, 3, 1, 1, 1, 1)}) 16 | table.insert(model, {'relu5', nn.ReLU(true)}) 17 | table.insert(model, {'conv6', nn.SpatialConvolution(16, 16, 3, 3, 1, 1, 1, 1)}) 18 | table.insert(model, {'relu6', nn.ReLU(true)}) 19 | table.insert(model, {'conv7', nn.SpatialConvolution(16, 1, 3, 3, 1, 1, 1, 1)}) 20 | -- warning: module 'trainLoss [type EuclideanLoss]' not found 21 | return model -------------------------------------------------------------------------------- /caffe/model/cr5/videoNetCNN_cr5_deploy_feature_10172016.prototxt: -------------------------------------------------------------------------------- 1 | layer { 2 | name: "data" 3 | type: "Input" 4 | top: "data" 5 | input_param { shape: { dim: 1 dim: 1 dim: 1 dim: 204 } } 6 | } 7 | 8 | layer { 9 | name: "fc1" 10 | type: "InnerProduct" 11 | bottom: "data" 12 | top: "fc1" 13 | param { 14 | lr_mult: 1 15 | decay_mult: 1 16 | } 17 | param { 18 | lr_mult: 2 19 | decay_mult: 0 20 | } 21 | inner_product_param { 22 | num_output: 1024 23 | weight_filler { 24 | type: "gaussian" 25 | std: 0.01 #For different measurement rates, different values work better. For MR = 0.25, MR = 0.04 and MR = 0.01, one can use 0.05, 0.01 and 0.03 respectively. 26 | } 27 | bias_filler { 28 | type: "constant" 29 | value: 0 30 | } 31 | } 32 | } 33 | layer { 34 | name: "reshape" 35 | type: "Reshape" 36 | bottom: "fc1" 37 | top: "reshape" 38 | reshape_param { 39 | shape { 40 | dim: 0 41 | dim: 1 42 | dim: 32 43 | dim: 32 44 | } 45 | } 46 | } 47 | layer { 48 | name: "conv1" 49 | type: "Convolution" 50 | bottom: "reshape" 51 | top: "conv1" 52 | param { 53 | lr_mult: 1 54 | } 55 | param { 56 | lr_mult: 0.1 57 | } 58 | convolution_param { 59 | num_output: 128 60 | pad: 0 61 | kernel_size: 1 62 | stride: 1 63 | weight_filler { 64 | type: "gaussian" 65 | std: 0.1 66 | } 67 | bias_filler { 68 | type: "constant" 69 | value: 0 70 | } 71 | } 72 | } 73 | layer { 74 | name: "relu1" 75 | type: "ReLU" 76 | bottom: "conv1" 77 | top: "relu1" 78 | } 79 | layer { 80 | name: "conv2" 81 | type: "Convolution" 82 | bottom: "relu1" 83 | top: "conv2" 84 | param { 85 | lr_mult: 1 86 | } 87 | param { 88 | lr_mult: 0.1 89 | } 90 | convolution_param { 91 | num_output: 64 92 | pad: 0 93 | kernel_size: 1 94 | stride: 1 95 | weight_filler { 96 | type: "gaussian" 97 | std: 0.1 98 | } 99 | bias_filler { 100 | type: "constant" 101 | value: 0 102 | } 103 | } 104 | } 105 | layer { 106 | name: "relu2" 107 | type: "ReLU" 108 | bottom: "conv2" 109 | top: "relu2" 110 | } 111 | layer { 112 | name: "conv3" 113 | type: "Convolution" 114 | bottom: "relu2" 115 | top: "conv3" 116 | param { 117 | lr_mult: 0.1 118 | } 119 | param { 120 | lr_mult: 0.1 121 | } 122 | convolution_param { 123 | num_output: 32 124 | pad: 1 125 | kernel_size: 3 126 | stride: 1 127 | weight_filler { 128 | type: "gaussian" 129 | std: 0.1 130 | } 131 | bias_filler { 132 | type: "constant" 133 | value: 0 134 | } 135 | } 136 | } 137 | layer { 138 | name: "relu3" 139 | type: "ReLU" 140 | bottom: "conv3" 141 | top: "relu3" 142 | } 143 | layer { 144 | name: "conv4" 145 | type: "Convolution" 146 | bottom: "relu3" 147 | top: "conv4" 148 | param { 149 | lr_mult: 1 150 | } 151 | param { 152 | lr_mult: 0.1 153 | } 154 | convolution_param { 155 | num_output: 32 156 | pad: 1 157 | kernel_size: 3 158 | stride: 1 159 | weight_filler { 160 | type: "gaussian" 161 | std: 0.1 162 | } 163 | bias_filler { 164 | type: "constant" 165 | value: 0 166 | } 167 | } 168 | } 169 | layer { 170 | name: "relu4" 171 | type: "ReLU" 172 | bottom: "conv4" 173 | top: "relu4" 174 | } 175 | layer { 176 | name: "conv5" 177 | type: "Convolution" 178 | bottom: "relu4" 179 | top: "conv5" 180 | param { 181 | lr_mult: 1 182 | } 183 | param { 184 | lr_mult: 0.1 185 | } 186 | convolution_param { 187 | num_output: 16 188 | pad: 1 189 | kernel_size: 3 190 | stride: 1 191 | weight_filler { 192 | type: "gaussian" 193 | std: 0.1 194 | } 195 | bias_filler { 196 | type: "constant" 197 | value: 0 198 | } 199 | } 200 | } 201 | layer { 202 | name: "relu5" 203 | type: "ReLU" 204 | bottom: "conv5" 205 | top: "relu5" 206 | } 207 | 208 | #layer { 209 | #name: "conv6" 210 | #type: "Convolution" 211 | #bottom: "relu5" 212 | #top: "conv6" 213 | #param { 214 | #lr_mult: 0.1 215 | #} 216 | #param { 217 | #lr_mult: 0.1 218 | #} 219 | #convolution_param { 220 | #num_output: 16 221 | #pad: 1 222 | #kernel_size: 3 223 | #stride: 1 224 | #weight_filler { 225 | #type: "gaussian" 226 | #std: 0.1 227 | #} 228 | #bias_filler { 229 | #type: "constant" 230 | #value: 0 231 | #} 232 | #} 233 | #} 234 | #layer { 235 | #name: "relu6" 236 | #type: "ReLU" 237 | #bottom: "conv6" 238 | #top: "relu6" 239 | #} 240 | # 241 | #layer { 242 | #name: "conv7" 243 | #type: "Convolution" 244 | #bottom: "relu6" 245 | #top: "conv7" 246 | #param { 247 | #lr_mult: 0.1 248 | #} 249 | #param { 250 | #lr_mult: 0.1 251 | #} 252 | #convolution_param { 253 | #num_output: 1 254 | #pad: 1 255 | #kernel_size: 3 256 | #stride: 1 257 | #weight_filler { 258 | #type: "gaussian" 259 | #std: 0.1 260 | #} 261 | #bias_filler { 262 | #type: "constant" 263 | #value: 0 264 | #} 265 | #} 266 | #} 267 | -------------------------------------------------------------------------------- /caffe/model/cr5/videoNetCNN_cr5_feature_10172016.prototxt.lua: -------------------------------------------------------------------------------- 1 | require 'cudnn' 2 | local model = {} 3 | -- warning: module 'data [type HDF5Data]' not found 4 | table.insert(model, {'torch_view', nn.View(-1):setNumInputDims(3)}) 5 | table.insert(model, {'fc1', nn.Linear(204, 10240)}) 6 | -- warning: module 'reshape [type Reshape]' not found 7 | table.insert(model, {'conv1', cudnn.SpatialConvolution(10, 128, 1, 1, 1, 1, 0, 0, 1)}) 8 | table.insert(model, {'relu1', cudnn.ReLU(true)}) 9 | table.insert(model, {'conv2', cudnn.SpatialConvolution(128, 64, 1, 1, 1, 1, 0, 0, 1)}) 10 | table.insert(model, {'relu2', cudnn.ReLU(true)}) 11 | table.insert(model, {'conv3', cudnn.SpatialConvolution(64, 32, 3, 3, 1, 1, 1, 1, 1)}) 12 | table.insert(model, {'relu3', cudnn.ReLU(true)}) 13 | table.insert(model, {'conv4', cudnn.SpatialConvolution(32, 32, 3, 3, 1, 1, 1, 1, 1)}) 14 | table.insert(model, {'relu4', cudnn.ReLU(true)}) 15 | table.insert(model, {'conv5', cudnn.SpatialConvolution(32, 16, 3, 3, 1, 1, 1, 1, 1)}) 16 | table.insert(model, {'relu5', cudnn.ReLU(true)}) 17 | table.insert(model, {'conv6', cudnn.SpatialConvolution(16, 16, 3, 3, 1, 1, 1, 1, 1)}) 18 | table.insert(model, {'relu6', cudnn.ReLU(true)}) 19 | table.insert(model, {'conv7', cudnn.SpatialConvolution(16, 1, 3, 3, 1, 1, 1, 1, 1)}) 20 | -- warning: module 'trainLoss [type EuclideanLoss]' not found 21 | return model -------------------------------------------------------------------------------- /caffe/model/cr5/videoNetCNN_cr5_model_10172016.prototxt: -------------------------------------------------------------------------------- 1 | name: "videoNetCNN_cr5_model_10172016.prototxt" 2 | layer { 3 | name: "data" 4 | type: "HDF5Data" 5 | top: "data" 6 | top: "label" 7 | hdf5_data_param { 8 | source: "trainCNN_cr5.txt" 9 | batch_size: 100 10 | } 11 | include: { phase: TRAIN } 12 | } 13 | layer { 14 | name: "data" 15 | type: "HDF5Data" 16 | top: "data" 17 | top: "label" 18 | hdf5_data_param { 19 | source: "valCNN_cr5.txt" 20 | batch_size: 100 21 | } 22 | include: { phase: TEST } 23 | } 24 | layer { 25 | name: "fc1" 26 | type: "InnerProduct" 27 | bottom: "data" 28 | top: "fc1" 29 | param { 30 | lr_mult: 1 31 | decay_mult: 1 32 | } 33 | param { 34 | lr_mult: 2 35 | decay_mult: 0 36 | } 37 | inner_product_param { 38 | num_output: 1024 39 | weight_filler { 40 | type: "gaussian" 41 | std: 0.01 #For different measurement rates, different values work better. For MR = 0.25, MR = 0.04 and MR = 0.01, one can use 0.05, 0.01 and 0.03 respectively. 42 | } 43 | bias_filler { 44 | type: "constant" 45 | value: 0 46 | } 47 | } 48 | } 49 | layer { 50 | name: "reshape" 51 | type: "Reshape" 52 | bottom: "fc1" 53 | top: "reshape" 54 | reshape_param { 55 | shape { 56 | dim: 0 57 | dim: 1 58 | dim: 32 59 | dim: 32 60 | } 61 | } 62 | } 63 | layer { 64 | name: "conv1" 65 | type: "Convolution" 66 | bottom: "reshape" 67 | top: "conv1" 68 | param { 69 | lr_mult: 1 70 | } 71 | param { 72 | lr_mult: 0.1 73 | } 74 | convolution_param { 75 | num_output: 128 76 | pad: 0 77 | kernel_size: 1 78 | stride: 1 79 | weight_filler { 80 | type: "gaussian" 81 | std: 0.1 82 | } 83 | bias_filler { 84 | type: "constant" 85 | value: 0 86 | } 87 | } 88 | } 89 | layer { 90 | name: "relu1" 91 | type: "ReLU" 92 | bottom: "conv1" 93 | top: "relu1" 94 | } 95 | layer { 96 | name: "conv2" 97 | type: "Convolution" 98 | bottom: "relu1" 99 | top: "conv2" 100 | param { 101 | lr_mult: 1 102 | } 103 | param { 104 | lr_mult: 0.1 105 | } 106 | convolution_param { 107 | num_output: 64 108 | pad: 0 109 | kernel_size: 1 110 | stride: 1 111 | weight_filler { 112 | type: "gaussian" 113 | std: 0.1 114 | } 115 | bias_filler { 116 | type: "constant" 117 | value: 0 118 | } 119 | } 120 | } 121 | layer { 122 | name: "relu2" 123 | type: "ReLU" 124 | bottom: "conv2" 125 | top: "relu2" 126 | } 127 | layer { 128 | name: "conv3" 129 | type: "Convolution" 130 | bottom: "relu2" 131 | top: "conv3" 132 | param { 133 | lr_mult: 0.1 134 | } 135 | param { 136 | lr_mult: 0.1 137 | } 138 | convolution_param { 139 | num_output: 32 140 | pad: 1 141 | kernel_size: 3 142 | stride: 1 143 | weight_filler { 144 | type: "gaussian" 145 | std: 0.1 146 | } 147 | bias_filler { 148 | type: "constant" 149 | value: 0 150 | } 151 | } 152 | } 153 | layer { 154 | name: "relu3" 155 | type: "ReLU" 156 | bottom: "conv3" 157 | top: "relu3" 158 | } 159 | layer { 160 | name: "conv4" 161 | type: "Convolution" 162 | bottom: "relu3" 163 | top: "conv4" 164 | param { 165 | lr_mult: 1 166 | } 167 | param { 168 | lr_mult: 0.1 169 | } 170 | convolution_param { 171 | num_output: 32 172 | pad: 1 173 | kernel_size: 3 174 | stride: 1 175 | weight_filler { 176 | type: "gaussian" 177 | std: 0.1 178 | } 179 | bias_filler { 180 | type: "constant" 181 | value: 0 182 | } 183 | } 184 | } 185 | layer { 186 | name: "relu4" 187 | type: "ReLU" 188 | bottom: "conv4" 189 | top: "relu4" 190 | } 191 | layer { 192 | name: "conv5" 193 | type: "Convolution" 194 | bottom: "relu4" 195 | top: "conv5" 196 | param { 197 | lr_mult: 1 198 | } 199 | param { 200 | lr_mult: 0.1 201 | } 202 | convolution_param { 203 | num_output: 16 204 | pad: 1 205 | kernel_size: 3 206 | stride: 1 207 | weight_filler { 208 | type: "gaussian" 209 | std: 0.1 210 | } 211 | bias_filler { 212 | type: "constant" 213 | value: 0 214 | } 215 | } 216 | } 217 | layer { 218 | name: "relu5" 219 | type: "ReLU" 220 | bottom: "conv5" 221 | top: "relu5" 222 | } 223 | 224 | layer { 225 | name: "conv6" 226 | type: "Convolution" 227 | bottom: "relu5" 228 | top: "conv6" 229 | param { 230 | lr_mult: 0.1 231 | } 232 | param { 233 | lr_mult: 0.1 234 | } 235 | convolution_param { 236 | num_output: 16 237 | pad: 1 238 | kernel_size: 3 239 | stride: 1 240 | weight_filler { 241 | type: "gaussian" 242 | std: 0.1 243 | } 244 | bias_filler { 245 | type: "constant" 246 | value: 0 247 | } 248 | } 249 | } 250 | layer { 251 | name: "relu6" 252 | type: "ReLU" 253 | bottom: "conv6" 254 | top: "relu6" 255 | } 256 | 257 | layer { 258 | name: "conv7" 259 | type: "Convolution" 260 | bottom: "relu6" 261 | top: "conv7" 262 | param { 263 | lr_mult: 0.1 264 | } 265 | param { 266 | lr_mult: 0.1 267 | } 268 | convolution_param { 269 | num_output: 1 270 | pad: 1 271 | kernel_size: 3 272 | stride: 1 273 | weight_filler { 274 | type: "gaussian" 275 | std: 0.1 276 | } 277 | bias_filler { 278 | type: "constant" 279 | value: 0 280 | } 281 | } 282 | } 283 | 284 | layer { 285 | name: "trainLoss" 286 | type: "EuclideanLoss" 287 | bottom: "conv7" 288 | bottom: "label" 289 | top: "trainLoss" 290 | include {phase: TRAIN} 291 | } 292 | 293 | layer { 294 | name: "testLoss" 295 | type: "EuclideanLoss" 296 | bottom: "conv7" 297 | bottom: "label" 298 | top: "testLoss" 299 | include {phase: TEST} 300 | } 301 | 302 | -------------------------------------------------------------------------------- /caffe/model/cr5/videoNetCNN_cr5_solver.prototxt: -------------------------------------------------------------------------------- 1 | net: "videoNetCNN_cr5_model_10172016.prototxt" 2 | test_iter: 100 3 | test_interval: 5000 4 | #test_compute_loss: true 5 | base_lr: 1e-4 6 | lr_policy: "multistep" 7 | #lr_policy:"fixed" 8 | gamma: 0.4 9 | stepvalue: 1000000 10 | #stepvalue: 3500000 11 | #stepvalue: 10000000 12 | display: 100 13 | max_iter: 5000000 14 | momentum: 0.9 15 | weight_decay: 5e-4 16 | snapshot: 5000 17 | snapshot_prefix: "Snapshots/cr_5_CNN_10172016/videoNetCNN_5" 18 | solver_mode: GPU 19 | #random_seed: 1701 20 | #clip_gradients: 10 21 | #average_loss: 50000 22 | solver_type: NESTEROV 23 | -------------------------------------------------------------------------------- /extractFeatures_5_25.m: -------------------------------------------------------------------------------- 1 | function extractFeatures_5_25(task) 2 | 3 | % extract raw CNN features per frame and save to feat_cache 4 | %% 5 | addpath('/home/user/kaixu/myGitHub/caffe/matlab/'); 6 | 7 | net_model_5 = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/' ... 8 | 'videoNetCNN_cr5_model_deploy_feature.prototxt']; 9 | net_weights_5 = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/'... 10 | 'Snapshots/cr_5_CNN_10072016/videoNetCNN_5_iter_390000.caffemodel']; 11 | net_model_25 = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/' ... 12 | 'videoNetCNN_cr25_model_deploy_feature.prototxt']; 13 | net_weights_25 = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/'... 14 | 'Snapshots/cr_25_CNN_10072016/videoNetCNN_25_iter_145000.caffemodel']; 15 | phase = 'test'; 16 | 17 | % task = 'Train'; 18 | % task = 'Val'; 19 | % testFile = '/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/ValData_5_25.h5'; 20 | % savepath = 'feature_conv5_5_25_test_80_test.h5'; 21 | if strcmp(task, 'Train') == 1 22 | testFile = './data/TrainData_5_25_10072016.h5'; 23 | savepath = './data/Feature_conv5_5_25_train.h5'; 24 | elseif strcmp(task, 'Val') == 1 25 | testFile = './data/ValData_5_25_10072016.h5'; 26 | savepath = './data/Feature_conv5_5_25_val.h5'; 27 | end 28 | 29 | 30 | seqLength = 10; 31 | numChannels = 16; 32 | imHeight = 32; 33 | imWidth = 32; 34 | use_gpu = 1; 35 | 36 | %% 37 | if exist('use_gpu', 'var') 38 | % try 39 | % id = obtain_gpu_lock_id; 40 | % catch e 41 | % disp(e) 42 | % end 43 | id = 1; 44 | caffe.set_mode_gpu(); 45 | caffe.set_device(id); 46 | net_5 = caffe.Net(net_model_5, net_weights_5, phase); 47 | net_25 = caffe.Net(net_model_25, net_weights_25, phase); 48 | 49 | % weight = net_5.blobs('conv5').get_data(); 50 | % weights_5 = net_5.copy_from(net_weights_5); 51 | else 52 | caffe.set_mode_cpu(); 53 | end 54 | 55 | %% 56 | testData = h5read(testFile, '/data'); 57 | testLabel = h5read(testFile, '/label'); 58 | 59 | numData = size(testData, 5)/10; 60 | if strcmp(task, 'Train') == 1 61 | numData = 256*80; %300 - 40G 100 - 17G 62 | end 63 | 64 | 65 | chunksz = 128; 66 | totalct = 0; 67 | created_flag = false; 68 | 69 | outputFeature = []; 70 | 71 | for i = 1:numData 72 | data = testData(:,:,:,:,i); 73 | data_5 = data(:,:,:,1); 74 | data_25 = data(:,:,:,2:end); 75 | data_25 = data_25(1:floor(imHeight*imWidth/25),:,:,:); 76 | 77 | feature = net_5.forward({data_5}); 78 | feature = feature{1}; 79 | feature = permute(feature, [4 3 2 1]); 80 | outputFeature_5 = feature; 81 | 82 | feature = net_25.forward({data_25}); 83 | feature = feature{1}; 84 | feature = permute(feature, [4 3 2 1]); 85 | outputFeature_25 = feature; 86 | 87 | outputFeature_5=reshape(outputFeature_5,[1,numChannels,imHeight,imWidth]); 88 | outputFeature_25=reshape(outputFeature_25,[9,numChannels,imHeight,imWidth]); 89 | tmp = [outputFeature_5;outputFeature_25]; 90 | tmp = reshape(tmp, [1, size(tmp)]); 91 | outputFeature = [outputFeature;tmp]; 92 | disp(['Completed',num2str(i/numData),'\n']) 93 | 94 | if mod(i, chunksz) == 0 95 | batchno = i / chunksz; 96 | last_read=(batchno-1) * chunksz; 97 | 98 | outputFeature = permute(outputFeature, [5 4 3 2 1]); 99 | batchdata = outputFeature; 100 | batchlabs = testLabel(:,:,:,:,last_read+1:last_read+chunksz); 101 | 102 | startloc = struct('dat',[1,1,1,1,totalct+1], 'lab', [1,1,1,1,totalct+1]); 103 | curr_dat_sz = store2hdf5(savepath, batchdata, batchlabs, ~created_flag, startloc, chunksz); 104 | created_flag = true; 105 | totalct = curr_dat_sz(end); 106 | outputFeature = []; 107 | end 108 | end 109 | 110 | 111 | %% writing to HDF5 112 | 113 | 114 | h5disp(savepath); 115 | 116 | %% 117 | caffe.reset_all(); 118 | % save('./data/outputFeature_5_25_conv5.mat', 'outputFeature'); 119 | 120 | function [feature] = caffe_forward(net, imseq) 121 | feature = net.forward({imseq}); 122 | feature = feature{1}; 123 | feature = permute(feature, [4 3 2 1]); 124 | end 125 | 126 | end 127 | -------------------------------------------------------------------------------- /genPhi.m: -------------------------------------------------------------------------------- 1 | clear 2 | clc 3 | 4 | cr = 100; 5 | n = 1024; 6 | phi=randn(floor(n/cr), n); 7 | phi=orth(phi')'; 8 | phi3 = zeros(size(phi,1), size(phi,2), 3); 9 | phi3(:,:,1) = phi; 10 | phi3(:,:,2) = phi; 11 | phi3(:,:,3) = phi; 12 | 13 | fileName = ['./phi/phi3_cr', num2str(cr), '_', num2str(n)]; 14 | save(fileName, 'phi3'); -------------------------------------------------------------------------------- /generateTrainCNN.m: -------------------------------------------------------------------------------- 1 | clear all; 2 | cr = 1; 3 | if cr == 1 4 | phi3 = zeros(1024, 1); 5 | else 6 | dataFile = ['./phi/phi3_cr', num2str(cr), '.mat']; 7 | phi3 = load(dataFile); 8 | phi3 = phi3.phi3; 9 | end 10 | folder = '~/myGitHub/datasets/UCF101/TrainData/0.25_196/'; 11 | savepath = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/data/TrainDataCNN_', num2str(cr), '_10172016.h5']; 12 | size_input = 32; 13 | size_label = 32; 14 | crop = 18; 15 | stride = size_input; 16 | 17 | data = zeros(size(phi3,1), 1, 1, 1); 18 | label = zeros(size_label, size_label, 1, 1); 19 | count = 0; 20 | 21 | filepaths = dir(fullfile(folder,'*.jpg')); 22 | 23 | for i = floor(linspace(1, length(filepaths), length(filepaths))) 24 | image = imread(fullfile(folder,filepaths(i).name)); 25 | image = rgb2ycbcr(image); 26 | image = im2double(image(:, :, 1)); 27 | rawImg = image(crop+1:end-crop,crop+1:end-crop); 28 | im_label = rawImg; 29 | [hei,wid, ~, ~] = size(im_label); 30 | 31 | for x = 1 : stride : hei-size_input+1 32 | for y = 1 : stride : wid-size_input+1 33 | subim_input = im_label(x : x+size_input-1, y : y+size_input-1); 34 | if cr == 1 35 | subim_input = subim_input(:); 36 | else 37 | subim_input = phi3(:,:,1)*subim_input(:); 38 | end 39 | 40 | subim_label = im_label(x : x+size_label-1, y : y+size_label-1); 41 | 42 | count=count+1; 43 | data(:, :, 1, count) = subim_input; 44 | label(:, :, 1, count) = subim_label; 45 | end 46 | end 47 | disp([num2str(i/length(filepaths)*100),'%']); 48 | end 49 | 50 | order = randperm(count); 51 | data = data(:, :, 1, order); 52 | label = label(:, :, 1, order); 53 | 54 | %% writing to HDF5 55 | chunksz = 128; 56 | created_flag = false; 57 | totalct = 0; 58 | 59 | for batchno = 1:floor(count/chunksz) 60 | last_read=(batchno-1)*chunksz; 61 | batchdata = data(:,:,1,last_read+1:last_read+chunksz); 62 | batchlabs = label(:,:,1,last_read+1:last_read+chunksz); 63 | 64 | startloc = struct('dat',[1,1,1,totalct+1], 'lab', [1,1,1,totalct+1]); 65 | curr_dat_sz = store2hdf5(savepath, batchdata, batchlabs, ~created_flag, startloc, chunksz); 66 | created_flag = true; 67 | totalct = curr_dat_sz(end); 68 | end 69 | h5disp(savepath); 70 | -------------------------------------------------------------------------------- /generateValCNN.m: -------------------------------------------------------------------------------- 1 | clear all; 2 | cr = 1; 3 | if cr == 1 4 | phi3 = zeros(1024, 1); 5 | else 6 | dataFile = ['./phi/phi3_cr', num2str(cr), '.mat']; 7 | phi3 = load(dataFile); 8 | phi3 = phi3.phi3; 9 | end 10 | 11 | folder = '~/myGitHub/datasets/UCF101/ValData/0.25_196/'; 12 | savepath = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/data/ValDataCNN_', num2str(cr), '_10172016.h5']; 13 | 14 | size_input = 32; 15 | size_label = 32; 16 | crop = 18; 17 | stride = size_input; 18 | 19 | data = zeros(size(phi3,1), 1, 1, 1); 20 | label = zeros(size_label, size_label, 1, 1); 21 | count = 0; 22 | 23 | filepaths = dir(fullfile(folder,'*.jpg')); 24 | 25 | for i = 1 : length(filepaths) 26 | image = imread(fullfile(folder,filepaths(i).name)); 27 | image = rgb2ycbcr(image); 28 | image = im2double(image(:, :, 1)); 29 | rawImg = image(crop+1:end-crop,crop+1:end-crop); 30 | im_label = rawImg; 31 | [hei,wid, ~, ~] = size(im_label); 32 | 33 | for x = 1 : stride : hei-size_input+1 34 | for y = 1 : stride : wid-size_input+1 35 | subim_input = im_label(x : x+size_input-1, y : y+size_input-1); 36 | if cr == 1 37 | subim_input = subim_input(:); 38 | else 39 | subim_input = phi3(:,:,1)*subim_input(:); 40 | end 41 | 42 | subim_label = im_label(x : x+size_label-1, y : y+size_label-1); 43 | 44 | count=count+1; 45 | data(:, :, 1, count) = subim_input; 46 | label(:, :, 1, count) = subim_label; 47 | end 48 | end 49 | disp([num2str(i/length(filepaths)*100),'%']); 50 | end 51 | 52 | order = randperm(count); 53 | data = data(:, :, 1, order); 54 | label = label(:, :, 1, order); 55 | 56 | %% writing to HDF5 57 | chunksz = 128; 58 | created_flag = false; 59 | totalct = 0; 60 | 61 | for batchno = 1:floor(count/chunksz) 62 | last_read=(batchno-1)*chunksz; 63 | batchdata = data(:,:,1,last_read+1:last_read+chunksz); 64 | batchlabs = label(:,:,1,last_read+1:last_read+chunksz); 65 | 66 | startloc = struct('dat',[1,1,1,totalct+1], 'lab', [1,1,1,totalct+1]); 67 | curr_dat_sz = store2hdf5(savepath, batchdata, batchlabs, ~created_flag, startloc, chunksz); 68 | created_flag = true; 69 | totalct = curr_dat_sz(end); 70 | end 71 | h5disp(savepath); 72 | -------------------------------------------------------------------------------- /model/cr_5_25/model/VideoNet.lua: -------------------------------------------------------------------------------- 1 | require 'torch' 2 | --require 'nn' 3 | require 'rnn' 4 | require 'loadcaffe' 5 | 6 | local utils = require 'util.utils' 7 | 8 | local VideoNet, parent = torch.class('nn.VideoNet', 'nn.Module') 9 | 10 | function convRelu(model, inputLayers, hiddenLayers, cnnKernel, cnnStride, cnnPad) 11 | model:add(nn.SpatialConvolution(inputLayers, hiddenLayers, cnnKernel, cnnKernel, cnnStride, cnnStride, cnnPad, cnnPad)) 12 | model:add(nn.SpatialBatchNormalization(hiddenLayers)) 13 | model:add(nn.ReLU()) 14 | end 15 | 16 | function convLayer(model, inputLayers, hiddenLayers, cnnKernel, cnnStride, cnnPad) 17 | model:add(nn.SpatialConvolution(inputLayers, hiddenLayers, cnnKernel, cnnKernel, cnnStride, cnnStride, cnnPad, cnnPad)) 18 | end 19 | 20 | prototxt1 = '/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr5/videoNetCNN_cr5_deploy_10172016.prototxt' 21 | binary1 = '/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr5/Snapshots/cr_5_CNN_10172016/videoNetCNN_5_iter_175000.caffemodel' 22 | 23 | prototxt2 = '/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr25/videoNetCNN_cr25_model_deploy.prototxt' 24 | binary2 = '/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr25/Snapshots/cr_25_CNN_10172016/videoNetCNN_25_iter_170000.caffemodel' 25 | 26 | prototxt3 = '/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr50/videoNetCNN_cr50_model_deploy.prototxt' 27 | binary3 = '/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr50/Snapshots/cr_50_CNN_10172016/videoNetCNN_50_iter_335000.caffemodel' 28 | 29 | prototxt4 = '/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr100/videoNetCNN_cr100_model_deploy.prototxt' 30 | binary4 = '/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr100/Snapshots/cr_100_CNN_10172016/videoNetCNN_100_iter_315000.caffemodel' 31 | 32 | prototxt5 = '/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr1/videoNetCNN_cr1_deploy_10172016.prototxt' 33 | binary5 = '/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr1/Snapshots/cr_1_CNN_10172016/videoNetCNN_5_iter_350000.caffemodel' 34 | 35 | function VideoNet:__init(args) 36 | assert(args ~= nil) 37 | 38 | local batchNorm = utils.getArgs(args, 'batchNorm') 39 | local batchSize = utils.getArgs(args, 'batchSize') 40 | local dropout = utils.getArgs(args, 'dropout') 41 | local Height = utils.getArgs(args, 'Height') 42 | local Width = utils.getArgs(args, 'Width') 43 | local seqLength = utils.getArgs(args, 'seqLength') 44 | local numChannels = utils.getArgs(args, 'numChannels') 45 | local measurements1 = utils.getArgs(args, 'measurements1') 46 | local measurements2 = utils.getArgs(args, 'measurements2') 47 | local backend = utils.getArgs(args, 'backend') 48 | local cr = utils.getArgs(args, 'cr') 49 | local rnnModel = utils.getArgs(args, 'rnnModel') 50 | 51 | local cnn = {} 52 | cnn.numHidden = {} 53 | cnn.numHidden[1] = 64 54 | cnn.numHidden[2] = 32 55 | cnn.numHidden[3] = 1 56 | cnn.kernel = 3 57 | cnn.pad = 1 58 | cnn.stride = 1 59 | 60 | local lstm = {} 61 | lstm.input = cnn.numHidden[3] * Height * Width 62 | lstm.numHidden = {} 63 | lstm.numHidden[1] = 6 * Height * Width 64 | lstm.numHidden[2] = 6 * Height * Width 65 | lstm.numHidden[3] = numChannels * Height * Width 66 | lstm.numHidden[4] = 5 * Height * Width 67 | lstm.numHidden[5] = numChannels * Height * Width 68 | 69 | -- load CNN_5 and CNN_25 70 | if cr == 5 then 71 | modelCNN = loadcaffe.load(prototxt1, binary1, backend) 72 | elseif cr == 25 then 73 | modelCNN = loadcaffe.load(prototxt2, binary2, backend) 74 | elseif cr == 50 then 75 | modelCNN = loadcaffe.load(prototxt3, binary3, backend) 76 | elseif cr == 100 then 77 | modelCNN = loadcaffe.load(prototxt4, binary4, backend) 78 | elseif cr == 1 then 79 | modelCNN = loadcaffe.load(prototxt5, binary5, backend) 80 | end 81 | 82 | local start = 0 83 | modelCNN1 = nn.Sequential() 84 | for i = 1, #modelCNN do 85 | local layer = modelCNN:get(i) 86 | local layer_name = layer.name 87 | if layer_name == 'conv7' then 88 | start = 1 89 | end 90 | if start == 1 then 91 | modelCNN1:add(layer) 92 | end 93 | end 94 | 95 | measurements = measurements1 + measurements2 96 | 97 | parallel_model = nn.ParallelTable() -- model that concatenates net1 and net2 98 | parallel_model:add(modelCNN1) 99 | 100 | modelCNN2 = nn.Sequential() 101 | modelCNN2:add(nn.View(-1, measurements)) 102 | modelCNN2:add(nn.Linear(measurements, Height*Width)) 103 | modelCNN2:add(nn.View(-1, Height, Width)) 104 | modelCNN2:add(nn.View(-1, 1, Height, Width)) 105 | convRelu(modelCNN2, numChannels, cnn.numHidden[1], cnn.kernel, cnn.stride, cnn.pad) 106 | convRelu(modelCNN2, cnn.numHidden[1], cnn.numHidden[2], cnn.kernel, cnn.stride, cnn.pad) 107 | convLayer(modelCNN2, cnn.numHidden[2], cnn.numHidden[3], cnn.kernel, cnn.stride, cnn.pad) 108 | modelCNN2:add(nn.View(-1, 1, cnn.numHidden[3], Height, Width)) 109 | parallel_model:add(modelCNN2) 110 | 111 | modelCNN3 = nn.Sequential() 112 | modelCNN3:add(nn.View(-1, measurements)) 113 | modelCNN3:add(nn.Linear(measurements, Height*Width)) 114 | modelCNN3:add(nn.View(-1, Height, Width)) 115 | modelCNN3:add(nn.View(-1, 1, Height, Width)) 116 | convRelu(modelCNN3, numChannels, cnn.numHidden[1], cnn.kernel, cnn.stride,cnn.pad) 117 | convRelu(modelCNN3, cnn.numHidden[1], cnn.numHidden[2], cnn.kernel,cnn.stride, cnn.pad) 118 | convLayer(modelCNN3, cnn.numHidden[2], cnn.numHidden[3], cnn.kernel,cnn.stride, cnn.pad) 119 | modelCNN3:add(nn.View(-1, 1, cnn.numHidden[3], Height, Width)) 120 | parallel_model:add(modelCNN3) 121 | 122 | modelCNN4 = nn.Sequential() 123 | modelCNN4:add(nn.View(-1, measurements)) 124 | modelCNN4:add(nn.Linear(measurements, Height*Width)) 125 | modelCNN4:add(nn.View(-1, Height, Width)) 126 | modelCNN4:add(nn.View(-1, 1, Height, Width)) 127 | convRelu(modelCNN4, numChannels, cnn.numHidden[1], cnn.kernel, cnn.stride,cnn.pad) 128 | convRelu(modelCNN4, cnn.numHidden[1], cnn.numHidden[2], cnn.kernel,cnn.stride, cnn.pad) 129 | convLayer(modelCNN4, cnn.numHidden[2], cnn.numHidden[3], cnn.kernel,cnn.stride, cnn.pad) 130 | modelCNN4:add(nn.View(-1, 1, cnn.numHidden[3], Height, Width)) 131 | parallel_model:add(modelCNN4) 132 | 133 | modelCNN5 = nn.Sequential() 134 | modelCNN5:add(nn.View(-1, measurements)) 135 | modelCNN5:add(nn.Linear(measurements, Height*Width)) 136 | modelCNN5:add(nn.View(-1, Height, Width)) 137 | modelCNN5:add(nn.View(-1, 1, Height, Width)) 138 | convRelu(modelCNN5, numChannels, cnn.numHidden[1], cnn.kernel, cnn.stride,cnn.pad) 139 | convRelu(modelCNN5, cnn.numHidden[1], cnn.numHidden[2], cnn.kernel,cnn.stride, cnn.pad) 140 | convLayer(modelCNN5, cnn.numHidden[2], cnn.numHidden[3], cnn.kernel,cnn.stride, cnn.pad) 141 | modelCNN5:add(nn.View(-1, 1, cnn.numHidden[3], Height, Width)) 142 | parallel_model:add(modelCNN5) 143 | 144 | modelCNN6 = nn.Sequential() 145 | modelCNN6:add(nn.View(-1, measurements)) 146 | modelCNN6:add(nn.Linear(measurements, Height*Width)) 147 | modelCNN6:add(nn.View(-1, Height, Width)) 148 | modelCNN6:add(nn.View(-1, 1, Height, Width)) 149 | convRelu(modelCNN6, numChannels, cnn.numHidden[1], cnn.kernel, cnn.stride,cnn.pad) 150 | convRelu(modelCNN6, cnn.numHidden[1], cnn.numHidden[2], cnn.kernel,cnn.stride, cnn.pad) 151 | convLayer(modelCNN6, cnn.numHidden[2], cnn.numHidden[3], cnn.kernel,cnn.stride, cnn.pad) 152 | modelCNN6:add(nn.View(-1, 1, cnn.numHidden[3], Height, Width)) 153 | parallel_model:add(modelCNN6) 154 | 155 | modelCNN7 = nn.Sequential() 156 | modelCNN7:add(nn.View(-1, measurements)) 157 | modelCNN7:add(nn.Linear(measurements, Height*Width)) 158 | modelCNN7:add(nn.View(-1, Height, Width)) 159 | modelCNN7:add(nn.View(-1, 1, Height, Width)) 160 | convRelu(modelCNN7, numChannels, cnn.numHidden[1], cnn.kernel, cnn.stride,cnn.pad) 161 | convRelu(modelCNN7, cnn.numHidden[1], cnn.numHidden[2], cnn.kernel,cnn.stride, cnn.pad) 162 | convLayer(modelCNN7, cnn.numHidden[2], cnn.numHidden[3], cnn.kernel,cnn.stride, cnn.pad) 163 | modelCNN7:add(nn.View(-1, 1, cnn.numHidden[3], Height, Width)) 164 | parallel_model:add(modelCNN7) 165 | 166 | modelCNN8 = nn.Sequential() 167 | modelCNN8:add(nn.View(-1, measurements)) 168 | modelCNN8:add(nn.Linear(measurements, Height*Width)) 169 | modelCNN8:add(nn.View(-1, Height, Width)) 170 | modelCNN8:add(nn.View(-1, 1, Height, Width)) 171 | convRelu(modelCNN8, numChannels, cnn.numHidden[1], cnn.kernel, cnn.stride,cnn.pad) 172 | convRelu(modelCNN8, cnn.numHidden[1], cnn.numHidden[2], cnn.kernel,cnn.stride, cnn.pad) 173 | convLayer(modelCNN8, cnn.numHidden[2], cnn.numHidden[3], cnn.kernel,cnn.stride, cnn.pad) 174 | modelCNN8:add(nn.View(-1, 1, cnn.numHidden[3], Height, Width)) 175 | parallel_model:add(modelCNN8) 176 | 177 | modelCNN9 = nn.Sequential() 178 | modelCNN9:add(nn.View(-1, measurements)) 179 | modelCNN9:add(nn.Linear(measurements, Height*Width)) 180 | modelCNN9:add(nn.View(-1, Height, Width)) 181 | modelCNN9:add(nn.View(-1, 1, Height, Width)) 182 | convRelu(modelCNN9, numChannels, cnn.numHidden[1], cnn.kernel, cnn.stride,cnn.pad) 183 | convRelu(modelCNN9, cnn.numHidden[1], cnn.numHidden[2], cnn.kernel,cnn.stride, cnn.pad) 184 | convLayer(modelCNN9, cnn.numHidden[2], cnn.numHidden[3], cnn.kernel,cnn.stride, cnn.pad) 185 | modelCNN9:add(nn.View(-1, 1, cnn.numHidden[3], Height, Width)) 186 | parallel_model:add(modelCNN9) 187 | 188 | modelCNN10 = nn.Sequential() 189 | modelCNN10:add(nn.View(-1, measurements)) 190 | modelCNN10:add(nn.Linear(measurements, Height*Width)) 191 | modelCNN10:add(nn.View(-1, Height, Width)) 192 | modelCNN10:add(nn.View(-1, 1, Height, Width)) 193 | convRelu(modelCNN10, numChannels, cnn.numHidden[1], cnn.kernel, cnn.stride,cnn.pad) 194 | convRelu(modelCNN10, cnn.numHidden[1], cnn.numHidden[2], cnn.kernel,cnn.stride, cnn.pad) 195 | convLayer(modelCNN10, cnn.numHidden[2], cnn.numHidden[3], cnn.kernel,cnn.stride, cnn.pad) 196 | modelCNN10:add(nn.View(-1, 1, cnn.numHidden[3], Height, Width)) 197 | parallel_model:add(modelCNN10) 198 | 199 | self.model = nn.Sequential() 200 | self.model:add(parallel_model) 201 | self.model:add(nn.JoinTable(2)) 202 | 203 | self.model:add(nn.View(batchSize, seqLength, cnn.numHidden[3]*Height*Width))-- [20, 16x32x32] 204 | self.model:add(nn.Transpose({1,2})) 205 | 206 | if rnnModel == 'LSTM' then 207 | rnn1 = nn.SeqLSTM(lstm.input, lstm.numHidden[1]) 208 | rnn2 = nn.SeqLSTM(lstm.numHidden[1], lstm.numHidden[2]) 209 | rnn3 = nn.SeqLSTM(lstm.numHidden[2], lstm.numHidden[3]) 210 | rnn4 = nn.SeqLSTM(lstm.numHidden[3], lstm.numHidden[4]) 211 | rnn5 = nn.SeqLSTM(lstm.numHidden[4], lstm.numHidden[5]) 212 | elseif rnnModel == 'GRU' then 213 | rnn1 = nn.seqGRU(lstm.input, lstm.numHidden[1]) 214 | rnn2 = nn.seqGRU(lstm.numHidden[1], lstm.numHidden[2]) 215 | rnn3 = nn.seqGRU(lstm.numHidden[2], lstm.numHidden[3]) 216 | rnn4 = nn.seqGRU(lstm.numHidden[3], lstm.numHidden[4]) 217 | rnn5 = nn.seqGRU(lstm.numHidden[4], lstm.numHidden[5]) 218 | end 219 | 220 | self.model:add(rnn1) 221 | if batchNorm == 1 then 222 | self.model:add(nn.Sequencer(nn.NormStabilizer())) 223 | end 224 | 225 | self.model:add(rnn2) 226 | if batchNorm == 1 then 227 | self.model:add(nn.Sequencer(nn.NormStabilizer())) 228 | end 229 | 230 | self.model:add(rnn3)--[[ 231 | if batchNorm == 1 then 232 | self.model:add(nn.Sequencer(nn.NormStabilizer())) 233 | end 234 | 235 | self.model:add(rnn4) 236 | if batchNorm == 1 then 237 | self.model:add(nn.Sequencer(nn.NormStabilizer())) 238 | end 239 | 240 | self.model:add(rnn5) 241 | --]] 242 | print(self.model) 243 | 244 | return self.model 245 | end 246 | 247 | function VideoNet:updateOutput(input) 248 | return self.model:forward(input) 249 | 250 | end 251 | 252 | function VideoNet:backward(input, gradOutput) 253 | return self.model:backward(input, gradOutput) 254 | end 255 | 256 | 257 | function VideoNet:parameters() 258 | return self.model:parameters() 259 | end 260 | -------------------------------------------------------------------------------- /model/cr_5_25/recon/Figure1.m: -------------------------------------------------------------------------------- 1 | function Figure1() 2 | load 'reconResult.mat' 3 | 4 | modelOut = reshape(modelOut, [sqrt(size(modelOut,1)), sqrt(size(modelOut,1)), size(modelOut,2), size(modelOut,3), size(modelOut,4) ]); 5 | labelOut = reshape(labelOut, [sqrt(size(labelOut,1)), sqrt(size(labelOut,1)), size(labelOut,2), size(labelOut,3), size(labelOut,4) ]); 6 | disp(num2str(size(modelOut))); 7 | disp(num2str(size(labelOut))); 8 | 9 | hei = 160; 10 | wid = 160; 11 | dimImage = 32; 12 | numChannels = 1; 13 | seqLength = 10; 14 | numImageBlk = size(modelOut, 5); 15 | blkDim = hei/dimImage; 16 | blkImage = blkDim^2; 17 | numImage = floor(numImageBlk / blkImage); 18 | 19 | seqFrame = zeros(numImage, hei, wid, numChannels, seqLength); 20 | frame = zeros(hei, wid, seqLength, numImage); 21 | for i = 1:numImage 22 | seqImage = modelOut(:,:,:,:,(i-1)*blkImage+1:i*blkImage); 23 | seqLabel = labelOut(:,:,:,:,(i-1)*blkImage+1:i*blkImage); 24 | for j = 1:seqLength 25 | image = seqImage(:,:,:,j,:); 26 | image = squeeze(image); 27 | label = seqLabel(:,:,:,j,:); 28 | label = squeeze(label); 29 | 30 | for p = 1:blkDim 31 | for q = 1:blkDim 32 | frame((p-1)*dimImage+1:p*dimImage,(q-1)*dimImage+1:q*dimImage,j,i) ... 33 | = image(:,:,(p-1)*blkDim+q); 34 | frameLabel((p-1)*dimImage+1:p*dimImage,(q-1)*dimImage+1:q*dimImage,j,i) ... 35 | = label(:,:,(p-1)*blkDim+q); 36 | end 37 | end 38 | end 39 | end 40 | 41 | imwrite(frame(:,:,1,19), 'shotPut_recon_25_1.pgm'); 42 | imwrite(frameLabel(:,:,1,19), 'shotPut_base_25_1.pgm'); 43 | imwrite(frame(:,:,5,19), 'shotPut_recon_25_5.pgm'); 44 | imwrite(frameLabel(:,:,5,19), 'shotPut_base_25_5.pgm'); 45 | 46 | imwrite(frame(:,:,1,22), 'skateboard_recon_25_1.pgm'); 47 | imwrite(frameLabel(:,:,1,22), 'skateboard_base_25_1.pgm'); 48 | imwrite(frame(:,:,5,22), 'skateboard_recon_25_5.pgm'); 49 | imwrite(frameLabel(:,:,5,22), 'skateboard_base_25_5.pgm'); 50 | 51 | imwrite(frame(:,:,1,7), 'guitar_recon_25_1.pgm'); 52 | imwrite(frameLabel(:,:,1,7), 'guitar_base_25_1.pgm'); 53 | imwrite(frame(:,:,5,7), 'guitar_recon_25_5.pgm'); 54 | imwrite(frameLabel(:,:,5,7), 'guitar_base_25_5.pgm'); 55 | 56 | img = frame(:,:,1,19); 57 | img = im2uint8(img); 58 | lab = frameLabel(:,:,1,19); 59 | lab = im2uint8(lab); 60 | psnr1 = psnr(img, lab); 61 | ssim1 = ssim(img, lab); 62 | el1 = sum(abs(img(:)-lab(:))) / (size(img,1)*size(img,2)); 63 | 64 | img = frame(:,:,5,19); 65 | img = im2uint8(img); 66 | lab = frameLabel(:,:,5,19); 67 | lab = im2uint8(lab); 68 | psnr5 = psnr(img, lab); 69 | ssim5 = ssim(img, lab); 70 | el5 = sum(abs(img(:)-lab(:))) / (size(img,1)*size(img,2)); 71 | 72 | index = 1; 73 | videoLength = [3,3,3,3,3,3,2,2]; 74 | psnrAvg = zeros(length(videoLength), 1); 75 | ssimAvg = zeros(length(videoLength), 1); 76 | elAvg = zeros(length(videoLength), 1); 77 | for i = 1:length(videoLength) 78 | frameSeq=frame(:,:,:,index:index+videoLength(i)-1); 79 | frameLabelSeq = frameLabel(:,:,:,index:index+videoLength(i)-1); 80 | [psnrAvg(i), ssimAvg(i), elAvg(i)] = measure(frameSeq, frameLabelSeq, 1); 81 | index = index + videoLength(i); 82 | end 83 | save('measurement.mat', 'psnrAvg', 'ssimAvg', 'elAvg'); 84 | end 85 | 86 | function [psnrAvg, ssimAvg, elAvg] = measure(frame, frameLabel, disp) 87 | psnrAcc = 0; 88 | ssimAcc = 0; 89 | elAcc = 0; 90 | psnrDebug1 = 0; 91 | psnrDebug2 = 0; 92 | ssimDebug1 = 0; 93 | ssimDebug2 = 0; 94 | elDebug1 = 0; 95 | elDebug2 = 0; 96 | 97 | if disp == 1 98 | v = VideoWriter('video_1_25.avi'); 99 | v.FrameRate = 5; 100 | open(v) 101 | fig = figure(1); 102 | end 103 | 104 | for i = 1:size(frame, 4) 105 | for j = 1:size(frame,3) 106 | img = frame(:,:,j,i); 107 | img = im2uint8(img); 108 | 109 | lab = frameLabel(:,:,j,i); 110 | lab = im2uint8(lab); 111 | 112 | psnrAcc = psnrAcc + psnr(img, lab); 113 | ssimAcc = ssimAcc + ssim(img, lab); 114 | elAcc = elAcc + sum(abs(img(:)-lab(:))); 115 | 116 | if disp == 1 117 | subplot(121) 118 | imshow(img); 119 | title(['current psnr: ' num2str(psnr(img, lab)), 'frame:', num2str(j)]) 120 | subplot(122) 121 | imshow(lab); 122 | 123 | title(['current ssim:', num2str(ssim(img, lab))]); 124 | cFrame = getframe(fig); 125 | writeVideo(v, cFrame); 126 | end 127 | 128 | if j==1 129 | psnrDebug1 = psnrDebug1 + psnr(img, lab); 130 | ssimDebug1 = ssimDebug1 + ssim(img, lab); 131 | else 132 | psnrDebug2 = psnrDebug2 + psnr(img, lab); 133 | ssimDebug2 = ssimDebug2 + ssim(img, lab); 134 | end 135 | end 136 | end 137 | if disp == 1 138 | close(v) 139 | end 140 | 141 | psnrAvg = psnrAcc / (size(frame,4)*size(frame,3)); 142 | psnrDebug1 = psnrDebug1 / (0.1*(size(frame,4)*size(frame,3))); 143 | psnrDebug2 = psnrDebug2 / (0.9*(size(frame,4)*size(frame,3))); 144 | 145 | ssimAvg = ssimAcc / (size(frame,4)*size(frame,3)); 146 | ssimDebug1 = ssimDebug1 / (0.1*(size(frame,4)*size(frame,3))); 147 | ssimDebug2 = ssimDebug2 / (0.9*(size(frame,4)*size(frame,3))); 148 | 149 | elAvg = elAcc / (size(frame,4)*size(frame,3)*size(frame,2)*size(frame,1)) ; 150 | 151 | end 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /model/cr_5_25/recon/GenerateTestData.m: -------------------------------------------------------------------------------- 1 | %data1-feature1, data2 1-original label 1-10 2 | function GenerateTestData(cr1, cr2) 3 | 4 | task = 'Test'; 5 | addpath('/home/user/kaixu/myGitHub/caffe/matlab/'); 6 | 7 | folder = ['~/myGitHub/datasets/UCF101/',task, 'Data/5_196/test1/']; 8 | 9 | paths = dir(folder); 10 | paths(1:2) = []; 11 | totalct = 0; 12 | created_flag = false; 13 | 14 | for i = 1:length(paths) 15 | totalct = GenerateGroupTrainData1ChanVarMix(cr1, cr2, num2str(i), task, totalct, created_flag); 16 | created_flag = true; 17 | end 18 | 19 | end 20 | %% 21 | function totalct = GenerateGroupTrainData1ChanVarMix(cr1, cr2, group, task, totalct_last, created_flag) 22 | if cr1 == 5 23 | load ../../../phi/phi3_cr5.mat 24 | phi1 = phi3; 25 | elseif cr1 == 25 26 | load ../../../phi/phi3_cr25.mat 27 | phi1 = phi3; 28 | elseif cr1 == 50 29 | load ../../../phi/phi3_cr50.mat 30 | phi1 = phi3; 31 | elseif cr1 == 100 32 | load ../../../phi/phi3_cr100.mat 33 | phi1 = phi3; 34 | end 35 | 36 | if cr2 == 5 37 | load ../../../phi/phi3_cr5.mat 38 | phi2 = phi3; 39 | elseif cr2 == 25 40 | load ../../../phi/phi3_cr25.mat 41 | phi2 = phi3; 42 | elseif cr2 == 50 43 | load ../../../phi/phi3_cr50.mat 44 | phi2 = phi3; 45 | elseif cr2 == 100 46 | load ../../../phi/phi3_cr100.mat 47 | phi2 = phi3; 48 | end 49 | 50 | phase = 'test'; 51 | 52 | %% 53 | if cr1 == 5 54 | net_model = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr5/'... ... 55 | 'videoNetCNN_cr5_deploy_feature_10172016.prototxt']; 56 | net_weights = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr5/'... 57 | 'Snapshots/cr_5_CNN_10172016/videoNetCNN_5_iter_175000.caffemodel']; 58 | elseif cr1 == 25 59 | net_model = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr25/'... ... 60 | 'videoNetCNN_cr25_deploy_feature_10172016.prototxt']; 61 | net_weights = ['/home/user/kaixu/myGitHub/caffe/examples/videoNet/training/models/cr25/'... 62 | 'Snapshots/cr_25_CNN_10172016/videoNetCNN_25_iter_170000.caffemodel']; 63 | end 64 | id = 3; 65 | caffe.set_mode_gpu(); 66 | caffe.set_device(id); 67 | net = caffe.Net(net_model, net_weights, phase); 68 | 69 | %% 70 | folder = ['~/myGitHub/datasets/UCF101/', task, 'Data/5_196/test1/group', num2str(group), '/']; 71 | savepath = ['data/',task,'Data_',num2str(cr1),'_',num2str(cr2),'_mix_feed_10172016.h5']; 72 | 73 | if strcmp(task, 'Train') == 1 74 | step = 2; 75 | elseif strcmp(task, 'Val') == 1 76 | step = 2; 77 | else 78 | step = 1; 79 | end 80 | 81 | imgHeight = 196; 82 | imgWidth = 196; 83 | numChannels = 1; 84 | cnnHidden = 16; 85 | 86 | size_input = 32; 87 | size_label = 32; 88 | crop = 18; 89 | stride = size_input; 90 | seq_length = 10; % length of the LSTM 91 | 92 | % data = zeros(seq_length, numChannels, size(phi3,1), 1, 1, 'single'); 93 | % label = zeros(seq_length, numChannels, size_label, size_label, 1, 'uint8'); 94 | data1 = zeros(size_label, size_label, cnnHidden, 1, 1, 'single'); 95 | data2 = zeros(size(phi1,1), 1, numChannels, seq_length, 1, 'single'); 96 | label = zeros(size_label, size_label, numChannels, seq_length, 1, 'single'); 97 | 98 | % padding = abs(size_input - size_label)/2; 99 | count = 1; 100 | 101 | im_label = zeros(imgHeight-2*crop, imgWidth-2*crop, numChannels, seq_length); 102 | subim_input = zeros(size_input, size_input, numChannels, seq_length); 103 | subim_label = zeros(size_input, size_input, numChannels, seq_length); 104 | subim_input_dr1 = zeros(size(phi1,1), 1, numChannels, 1); 105 | subim_input_dr2 = zeros(size(phi1,1), 1, numChannels, seq_length-1); 106 | 107 | paths = dir(folder); 108 | paths(1:2) = []; 109 | 110 | newPaths = []; 111 | 112 | for i = 1:step:length(paths) 113 | newPaths = [newPaths,paths(i)]; 114 | end 115 | 116 | % h = waitbar(0,'Writing train data to hdf5 file, please wait...'); 117 | 118 | for i = 1:length(newPaths) 119 | filename = dir([folder, newPaths(i).name]); 120 | filename(1:2) = []; % filenames in each subfolder 121 | 122 | len = length(filename); 123 | tmp = floor(len / seq_length); 124 | if tmp >= 3 125 | tmp = 3; 126 | end 127 | numImage = tmp * seq_length; 128 | 129 | for j = 1:seq_length:numImage 130 | for k = 1:seq_length 131 | rawImg = imread([folder, newPaths(i).name, '/', filename(j+k-1).name]); 132 | rawImg = rgb2ycbcr(rawImg); 133 | rawImg = im2double(rawImg(:,:,1)); 134 | rawImg = rawImg(crop+1:end-crop,crop+1:end-crop); 135 | im_label(:,:,:,k) = rawImg; 136 | [hei,wid, ~, ~] = size(im_label); 137 | end 138 | 139 | for x = 1 : stride : hei-size_input+1 140 | for y = 1 : stride : wid-size_input+1 141 | for z = 1 : seq_length 142 | subim_input(:, :, :, z) = im_label(x : x+size_input-1, y : y+size_input-1, :, z); 143 | subim_input_rs = reshape(subim_input, [],numChannels, seq_length); 144 | if z == 1 145 | for xx = 1:numChannels 146 | if size(phi1, 1) == size_input^2 147 | subim_input_dr1(:,:,xx,1) = subim_input_rs(:,xx,z); 148 | else 149 | subim_input_dr1 = phi1(:,:,xx) * subim_input_rs(:,xx,z); 150 | feature = net.forward({subim_input_dr1}); 151 | feature = feature{1}; 152 | end 153 | end 154 | else 155 | for xx = 1:numChannels 156 | tmp = phi2(:,:,xx) * subim_input_rs(:,xx,z); 157 | tmp(size(tmp,1)+1:size(phi1,1)) = 0; 158 | subim_input_dr2(:,:,xx,z-1) = tmp; 159 | end 160 | end 161 | 162 | subim_label(:, :, :, z) = im_label(x : x+size_label-1, ... 163 | y : y+size_label-1, :, z); 164 | end 165 | 166 | tmp = cat(4, subim_input_dr1, subim_input_dr2); 167 | data1(:, :, :, :, count) = feature; 168 | data2(:, :, :, :, count) = tmp; 169 | label(:, :, :, :, count) = subim_label; 170 | 171 | count=count+1; 172 | end 173 | end 174 | end 175 | disp(['group', num2str(group), ', ', num2str(i/length(newPaths)*100),'%']); 176 | 177 | end 178 | 179 | if strcmp(task, 'Train') ~= 0 180 | count = count - 1; 181 | order = randperm(count); 182 | data1 = data1(:, :, :, :, order); 183 | data2 = data2(:, :, :, :, order); 184 | label = label(:, :, :, :, order); 185 | end 186 | 187 | % writing to HDF5 188 | chunksz = 20; 189 | % created_flag = false; 190 | totalct = totalct_last; 191 | 192 | for batchno = 1:floor(count/chunksz) 193 | last_read=(batchno-1) * chunksz; 194 | batchdata1 = data1(:,:,:,:,last_read+1:last_read+chunksz); 195 | batchdata2 = data2(:,:,:,:,last_read+1:last_read+chunksz); 196 | batchlabs = label(:,:,:,:,last_read+1:last_read+chunksz); 197 | 198 | startloc = struct('dat1',[1,1,1,1,totalct+1], 'dat2', [1,1,1,1,totalct+1], 'lab', [1,1,1,1,totalct+1]); 199 | curr_dat_sz = store2hdf5Mix(savepath, batchdata1, batchdata2, batchlabs, ~created_flag, startloc, chunksz); 200 | created_flag = true; 201 | totalct = curr_dat_sz(end); 202 | end 203 | h5disp(savepath); 204 | 205 | end 206 | 207 | 208 | -------------------------------------------------------------------------------- /model/cr_5_25/recon/dispResult.m: -------------------------------------------------------------------------------- 1 | clear 2 | 3 | load 'reconResult.mat' 4 | 5 | disp(num2str(size(modelOut))); 6 | disp(num2str(size(labelOut))); 7 | 8 | hei = 160; 9 | wid = 160; 10 | dimImage = 32; 11 | numChannels = 1; 12 | seqLength = 10; 13 | numImageBlk = size(modelOut, 5); 14 | blkDim = hei/dimImage; 15 | blkImage = blkDim^2; 16 | numImage = floor(numImageBlk / blkImage); 17 | 18 | seqFrame = zeros(numImage, hei, wid, numChannels, seqLength); 19 | frame = zeros(hei, wid, seqLength, numImage); 20 | for i = 1:numImage 21 | seqImage = modelOut(:,:,:,:,(i-1)*blkImage+1:i*blkImage); 22 | seqLabel = labelOut(:,:,:,:,(i-1)*blkImage+1:i*blkImage); 23 | for j = 1:seqLength 24 | image = seqImage(:,:,:,j,:); 25 | image = squeeze(image); 26 | label = seqLabel(:,:,:,j,:); 27 | label = squeeze(label); 28 | 29 | for p = 1:blkDim 30 | for q = 1:blkDim 31 | frame((p-1)*dimImage+1:p*dimImage,(q-1)*dimImage+1:q*dimImage,j,i) ... 32 | = image(:,:,(p-1)*blkDim+q); 33 | frameLabel((p-1)*dimImage+1:p*dimImage,(q-1)*dimImage+1:q*dimImage,j,i) ... 34 | = label(:,:,(p-1)*blkDim+q); 35 | end 36 | end 37 | end 38 | end 39 | 40 | psnrAcc = 0; 41 | ssimAcc = 0; 42 | psnrDebug1 = 0; 43 | psnrDebug2 = 0; 44 | ssimDebug1 = 0; 45 | ssimDebug2 = 0; 46 | 47 | v = VideoWriter('video_1_25.avi'); 48 | v.FrameRate = 5; 49 | open(v) 50 | 51 | fig = figure(1); 52 | for i = 1:size(frame, 4) 53 | for j = 1:size(frame,3) 54 | img = frame(:,:,j,i); 55 | img = im2uint8(img); 56 | 57 | lab = frameLabel(:,:,j,i); 58 | lab = im2uint8(lab); 59 | 60 | subplot(121) 61 | imshow(img); 62 | title(['current psnr: ' num2str(psnr(img, lab)), 'frame:', num2str(j)]) 63 | 64 | subplot(122) 65 | imshow(lab); 66 | psnrAcc = psnrAcc + psnr(img, lab); 67 | ssimAcc = ssimAcc + ssim(img, lab); 68 | title(['current ssim:', num2str(ssim(img, lab))]); 69 | cFrame = getframe(fig); 70 | writeVideo(v, cFrame); 71 | 72 | if j==1 73 | psnrDebug1 = psnrDebug1 + psnr(img, lab); 74 | ssimDebug1 = ssimDebug1 + ssim(img, lab); 75 | else 76 | psnrDebug2 = psnrDebug2 + psnr(img, lab); 77 | ssimDebug2 = ssimDebug2 + ssim(img, lab); 78 | end 79 | end 80 | end 81 | close(v) 82 | 83 | psnrAvg = psnrAcc / (size(frame,4)*size(frame,3)); 84 | psnrDebug1 = psnrDebug1 / (0.1*(size(frame,4)*size(frame,3))); 85 | psnrDebug2 = psnrDebug2 / (0.9*(size(frame,4)*size(frame,3))); 86 | 87 | ssimAvg = ssimAcc / (size(frame,4)*size(frame,3)); 88 | ssimDebug1 = ssimDebug1 / (0.1*(size(frame,4)*size(frame,3))); 89 | ssimDebug2 = ssimDebug2 / (0.9*(size(frame,4)*size(frame,3))); 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /model/cr_5_25/recon/measurement.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PSCLab-ASU/CSVideoNet/99ee2e6f95097223e231c460df51b2cd4fb46b24/model/cr_5_25/recon/measurement.mat -------------------------------------------------------------------------------- /model/cr_5_25/recon/reconResult.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PSCLab-ASU/CSVideoNet/99ee2e6f95097223e231c460df51b2cd4fb46b24/model/cr_5_25/recon/reconResult.mat -------------------------------------------------------------------------------- /model/cr_5_25/recon/store2hdf5Mix.m: -------------------------------------------------------------------------------- 1 | function [curr_dat1_sz, curr_dat2_sz, curr_lab_sz] = store2hdf5Mix(filename, data1, data2, labels, create, startloc, chunksz) 2 | % *data* is W*H*C*N matrix of images should be normalized (e.g. to lie between 0 and 1) beforehand 3 | % *label* is D*N matrix of labels (D labels per sample) 4 | % *create* [0/1] specifies whether to create file newly or to append to previously created file, useful to store information in batches when a dataset is too big to be held in memory (default: 1) 5 | % *startloc* (point at which to start writing data). By default, 6 | % if create=1 (create mode), startloc.data=[1 1 1 1], and startloc.lab=[1 1]; 7 | % if create=0 (append mode), startloc.data=[1 1 1 K+1], and startloc.lab = [1 K+1]; where K is the current number of samples stored in the HDF 8 | % chunksz (used only in create mode), specifies number of samples to be stored per chunk (see HDF5 documentation on chunking) for creating HDF5 files with unbounded maximum size - TLDR; higher chunk sizes allow faster read-write operations 9 | 10 | % verify that format is right 11 | dat1_dims=size(data1); 12 | dat2_dims=size(data2); 13 | lab_dims=size(labels); 14 | num_samples=dat1_dims(end); 15 | 16 | assert(lab_dims(end)==num_samples, 'Number of samples should be matched between data and labels'); 17 | 18 | if ~exist('create','var') 19 | create=true; 20 | end 21 | 22 | if create 23 | %fprintf('Creating dataset with %d samples\n', num_samples); 24 | if ~exist('chunksz', 'var') 25 | chunksz=1000; 26 | end 27 | if exist(filename, 'file') 28 | fprintf('Warning: replacing existing file %s \n', filename); 29 | delete(filename); 30 | end 31 | % h5create(filename, '/data', [dat_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [dat_dims(1:end-1) chunksz]); % width, height, channels, number 32 | % h5create(filename, '/label', [lab_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [lab_dims(1:end-1) chunksz]); % width, height, channels, number 33 | h5create(filename, '/data1', [dat1_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [dat1_dims(1:end-1) chunksz]); % width, height, channels, number 34 | h5create(filename, '/data2', [dat2_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [dat2_dims(1:end-1) chunksz]); 35 | h5create(filename, '/label', [lab_dims(1:end-1) Inf], 'Datatype', 'single', 'ChunkSize', [lab_dims(1:end-1) chunksz]); % width, height, channels, number 36 | if ~exist('startloc','var') 37 | startloc.dat1=[ones(1,length(dat1_dims)-1), 1]; 38 | startloc.dat2=[ones(1,length(dat2_dims)-1), 1]; 39 | startloc.lab=[ones(1,length(lab_dims)-1), 1]; 40 | end 41 | else % append mode 42 | if ~exist('startloc','var') 43 | info=h5info(filename); 44 | prev_dat1_sz=info.Datasets(1).Dataspace.Size; 45 | prev_dat2_sz=info.Datasets(2).Dataspace.Size; 46 | prev_lab_sz=info.Datasets(3).Dataspace.Size; 47 | assert(prev_dat1_sz(1:end-1)==dat1_dims(1:end-1), 'Data dimensions must match existing dimensions in dataset'); 48 | assert(prev_dat2_sz(1:end-1)==dat1_dims(1:end-1), 'Data dimensions must match existing dimensions in dataset'); 49 | assert(prev_lab_sz(1:end-1)==lab_dims(1:end-1), 'Label dimensions must match existing dimensions in dataset'); 50 | startloc.dat1=[ones(1,length(dat1_dims)-1), prev_dat1_sz(end)+1]; 51 | startloc.dat2=[ones(1,length(dat2_dims)-1), prev_dat2_sz(end)+1]; 52 | startloc.lab=[ones(1,length(lab_dims)-1), prev_lab_sz(end)+1]; 53 | end 54 | end 55 | 56 | if ~isempty(data1) 57 | % h5write(filename, '/data', single(data), startloc.dat, size(data)); 58 | % h5write(filename, '/label', single(labels), startloc.lab, size(labels)); 59 | data1 = single(data1); 60 | data2 = single(data2); 61 | labels = single(labels); 62 | h5write(filename, '/data1', data1, startloc.dat1, size(data1)); 63 | h5write(filename, '/data2', data2, startloc.dat2, size(data2)); 64 | h5write(filename, '/label', labels, startloc.lab, size(labels)); 65 | end 66 | 67 | if nargout 68 | info=h5info(filename); 69 | curr_dat1_sz=info.Datasets(1).Dataspace.Size; 70 | curr_dat2_sz=info.Datasets(2).Dataspace.Size; 71 | curr_lab_sz=info.Datasets(3).Dataspace.Size; 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /model/cr_5_25/testVideoNet.lua: -------------------------------------------------------------------------------- 1 | require 'torch' 2 | require 'nn' 3 | require 'image' 4 | require 'model.VideoNet' 5 | require 'util.DataHandler' 6 | require 'mattorch' 7 | 8 | local utils = require 'util.utils' 9 | 10 | local cmd = torch.CmdLine() 11 | 12 | -- Dataset options 13 | cmd:option('-batchNorm', 1) 14 | cmd:option('-batchSize', 80) 15 | cmd:option('-Width', 32) 16 | cmd:option('-Height', 32) 17 | cmd:option('-numChannels', 1) 18 | cmd:option('-measurements1', 204) 19 | cmd:option('-measurements2', 40) 20 | 21 | -- Model options 22 | cmd:option('-dropout', 0.5) 23 | cmd:option('-seqLength', 10) 24 | 25 | -- Output options 26 | cmd:option('-printEvery', 10) 27 | cmd:option('-checkpointEvery', 5000) 28 | cmd:option('-checkpoint', 'checkpoints/checkpoint_0.0001_ep12_4000.t7') 29 | 30 | cmd:option('-cuda', '1') 31 | cmd:option('-gpu', '3') 32 | cmd:option('-backend', 'cudnn') 33 | 34 | local opt = cmd:parse(arg) 35 | for k, v in pairs(opt) do 36 | if tonumber(v) then 37 | opt[k] = tonumber(v) 38 | end 39 | end 40 | 41 | -- set GPU 42 | opt.dtype = 'torch.FloatTensor' 43 | if opt.gpu >= 0 and opt.cuda == 1 then 44 | require 'cutorch' 45 | require 'cudnn' 46 | require 'cunn' 47 | opt.dtype = 'torch.CudaTensor' 48 | cutorch.setDevice(opt.gpu+1) 49 | print(string.format('Running with CUDA on GPU %d', opt.gpu)) 50 | else 51 | print(string.format('Running in CPU mode')) 52 | end 53 | 54 | -- load dataset 55 | utils.printTime("Initializing DataLoader") 56 | local loader = DataHandler(opt) 57 | loader = loader:loadTestData() 58 | 59 | -- initialize model 60 | utils.printTime("Initializing VideoNet") 61 | model = nn.Sequential() 62 | local checkpoint = torch.load(opt.checkpoint) 63 | local model = checkpoint.model 64 | --model:testing() 65 | 66 | local cri = nn.MSECriterion() 67 | cri.sizeAverage = false 68 | local criterion = nn.SequencerCriterion(cri, true):type(opt.dtype) 69 | criterion:cuda() 70 | 71 | function test(model) 72 | collectgarbage() 73 | utils.printTime(string.format("Starting testing on %s dataset", 'test')) 74 | 75 | local evalData = {} 76 | evalData.predictedLabels = {} 77 | evalData.trueLabels = {} 78 | 79 | local totalLoss = 0 80 | 81 | local loopCount = 1 82 | 83 | local dataInitialIndex = torch.range(1, loader.test.inputNum, opt.batchSize) 84 | local numBatches = dataInitialIndex:size(1) 85 | local dataIndex = torch.range(1, numBatches) 86 | 87 | local matModelOut = torch.DoubleTensor(numBatches*opt.batchSize, opt.seqLength, opt.numChannels, opt.Height*opt.Width) 88 | local matLabelOut = torch.DoubleTensor(numBatches*opt.batchSize, opt.seqLength, opt.numChannels, opt.Height*opt.Width) 89 | for i = 1, numBatches do 90 | local data1, data2, data3, data4, data5, data6, data7, data8, data9, data10, label = loader:getBatch('test', dataInitialIndex[dataIndex[i]]) 91 | if opt.cuda == 1 then 92 | data1 = data1:cuda() 93 | data2 = data2:cuda() 94 | data3 = data3:cuda() 95 | data4 = data4:cuda() 96 | data5 = data5:cuda() 97 | data6 = data6:cuda() 98 | data7 = data7:cuda() 99 | data8 = data8:cuda() 100 | data9 = data9:cuda() 101 | data10 = data10:cuda() 102 | label = label:cuda() 103 | end 104 | 105 | local testModelOut = model:forward({data1, data2, data3, data4, data5, data6, data7, data8, data9, data10}) 106 | 107 | local loss = criterion:forward(testModelOut, label) 108 | totalLoss = totalLoss + loss 109 | 110 | testModelOut = testModelOut:transpose(1,2):float() 111 | label = label:float():transpose(1,2) 112 | matModelOut[{ {(i-1)*opt.batchSize+1, i*opt.batchSize},{},{} }] = testModelOut 113 | matLabelOut[{ {(i-1)*opt.batchSize+1, i*opt.batchSize},{},{} }] = label 114 | 115 | if((opt.printEvery > 0 and loopCount % opt.printEvery == 0) or loopCount <= 10) then 116 | local errEveryPixel = loss / (opt.batchSize * opt.numChannels * opt.Width * opt.Height) 117 | utils.printTime("----------------------------------------") 118 | utils.printTime(string.format("Testing: sample %d, Testing loss: %f, average loss per pixel: %f", loopCount, loss/opt.batchSize, errEveryPixel)) 119 | end 120 | 121 | loopCount = loopCount + 1 122 | end 123 | 124 | local avgLoss = totalLoss / (loopCount-1) 125 | utils.printTime("+++++++++++++++++++++++++++++++++++++++++++++++") 126 | 127 | local errEveryPixel = avgLoss / (opt.batchSize * opt.numChannels * opt.Width * opt.Height) 128 | utils.printTime(string.format("Valdation: sample %d, average testing loss: %f, average loss per pixel: %f", loopCount, avgLoss / opt.batchSize, errEveryPixel)) 129 | 130 | -- Save model output and label 131 | list = {modelOut = matModelOut, labelOut = matLabelOut} 132 | mattorch.save('./recon/reconResult.mat', list) 133 | --matio.save('testResult.mat', matModelOut) 134 | 135 | collectgarbage() 136 | 137 | return loss 138 | end 139 | 140 | 141 | test(model) 142 | -------------------------------------------------------------------------------- /model/cr_5_25/trainVideoNet.lua: -------------------------------------------------------------------------------- 1 | require 'torch' 2 | require 'nn' 3 | require 'image' 4 | require 'optim' 5 | require 'model.VideoNet' 6 | require 'util.DataHandler' 7 | 8 | local utils = require 'util.utils' 9 | 10 | local cmd = torch.CmdLine() 11 | 12 | -- Dataset options 13 | cmd:option('-batchNorm', 1) 14 | cmd:option('-batchSize', 80) 15 | cmd:option('-Width', 32) 16 | cmd:option('-Height', 32) 17 | cmd:option('-numChannels', 1) 18 | cmd:option('-measurements1', 204) 19 | cmd:option('-measurements2', 40) 20 | 21 | -- Model options 22 | cmd:option('-dropout', 0.5) 23 | cmd:option('-seqLength', 10) 24 | 25 | -- Optimization options 26 | cmd:option('-numEpochesTrain', 1000) 27 | cmd:option('-learningRate', 1e-4) 28 | cmd:option('-momentum', 0.9) 29 | cmd:option('-lrDecayFactor', 0.5) 30 | cmd:option('-lrDecayEvery', 3) 31 | cmd:option('-weightDecay', 0); 32 | 33 | -- Output options 34 | cmd:option('-printEvery', 10) 35 | cmd:option('-checkpointEvery', 4000) 36 | cmd:option('-checkpointName', 'checkpoints/checkpoint') 37 | 38 | cmd:option('-cuda', 1) 39 | cmd:option('-gpu', 0) 40 | cmd:option('-backend', 'nn') 41 | cmd:option('-cr', 5) 42 | cmd:option('-rnnModel', 'LSTM') 43 | cmd:option('-init_from', '') 44 | cmd:option('-epochStart', '') 45 | --checkpoints/checkpointsVideoNet_5_25_2/checkpoint_0.0001_ep4_10000.t7 46 | 47 | local opt = cmd:parse(arg) 48 | for k, v in pairs(opt) do 49 | if tonumber(v) then 50 | opt[k] = tonumber(v) 51 | end 52 | end 53 | 54 | -- set GPU 55 | opt.dtype = 'torch.FloatTensor' 56 | if opt.gpu >= 0 and opt.cuda == 1 then 57 | require 'cutorch' 58 | require 'cunn' 59 | require 'cudnn' 60 | opt.dtype = 'torch.CudaTensor' 61 | cutorch.setDevice(opt.gpu+1) 62 | opt.dtype = 'torch.CudaTensor' 63 | print(string.format('Running with CUDA on GPU %d', opt.gpu)) 64 | else 65 | print(string.format('Running in CPU mode')) 66 | end 67 | 68 | -- load dataset 69 | utils.printTime("Initializing DataLoader") 70 | local loader = DataHandler(opt) 71 | loader = loader:loadData() 72 | 73 | -- initialize model 74 | utils.printTime("Initializing videoNet") 75 | 76 | if opt.init_from ~= '' then 77 | print('Initializing from ', opt.init_from) 78 | model = nn.Sequential() 79 | local checkpoint = torch.load(opt.init_from) 80 | model = checkpoint.model 81 | --model = checkpoint.model:type(dtype) 82 | else 83 | model = nn.VideoNet(opt):type(opt.dtype) 84 | end 85 | 86 | if opt.epochStart == '' then 87 | opt.epochStart = 1 88 | end 89 | 90 | local params, gradParams = model:getParameters() 91 | print(model) 92 | model:training() 93 | model = model:cuda() 94 | 95 | local cri = nn.MSECriterion() 96 | cri.sizeAverage = false 97 | local criterion = nn.SequencerCriterion(cri, true):type(opt.dtype) 98 | criterion:cuda() 99 | 100 | function train(model) 101 | utils.printTime(string.format("Starting training for %d epochs", opt.numEpochesTrain)) 102 | 103 | local trainLossHistory = {} 104 | local valLossHistory = {} 105 | local testLossHistory = {} 106 | 107 | local optimState = { 108 | learningRate = opt.learningRate, 109 | --learningRateDecay = opt.lrDecayFactor, 110 | --momentum = opt.momentum, 111 | -- dampening = 0.0, 112 | --weightDecay = opt.weightDecay 113 | } 114 | 115 | -- For each epoch training 116 | for iEpochs = opt.epochStart, opt.numEpochesTrain do 117 | local epochLoss = {} 118 | 119 | if iEpochs % opt.lrDecayEvery == 0 then 120 | local oldLearningRate = optimState.learningRate 121 | optimState.learningRate = oldLearningRate * opt.lrDecayFactor 122 | end 123 | 124 | local loopCount = 1 125 | 126 | dataInitialIndex = torch.range(1, loader.train.inputNum, opt.batchSize) 127 | numBatches = dataInitialIndex:size(1) 128 | dataIndex = torch.randperm(numBatches) 129 | 130 | for i = 1, numBatches do 131 | local data1, data2, data3, data4, data5, data6, data7, data8, data9, data10, label = loader:getBatch('train', dataInitialIndex[dataIndex[i]]) 132 | 133 | if opt.cuda == 1 then 134 | data1 = data1:cuda() 135 | data2 = data2:cuda() 136 | data3 = data3:cuda() 137 | data4 = data4:cuda() 138 | data5 = data5:cuda() 139 | data6 = data6:cuda() 140 | data7 = data7:cuda() 141 | data8 = data8:cuda() 142 | data9 = data9:cuda() 143 | data10 = data10:cuda() 144 | label = label:cuda() 145 | end 146 | 147 | local function feval(x) 148 | if x ~= params then 149 | params:copy(x) 150 | end 151 | gradParams:zero() 152 | 153 | local modelOut = model:forward({data1, data2, data3, data4, data5, data6, data7, data8, data9, data10}) 154 | -- local label_view = label:view(opt.seqLength, opt.numChannels, opt.Height*opt.Width) 155 | local frameLoss = criterion:forward(modelOut, label) 156 | local gradOutput = criterion:backward(modelOut, label) 157 | local gradModel = model:backward({data1, data2, data3, data4, data5, data6, data7, data8, data9, data10}, gradOutput) 158 | 159 | collectgarbage() 160 | return frameLoss, gradParams 161 | -- gradient clamp ??????????????? 162 | end 163 | 164 | local _, loss = optim.adam(feval, params, optimState) 165 | table.insert(epochLoss, loss[1]) 166 | 167 | if((opt.printEvery > 0 and loopCount % opt.printEvery == 0) or loopCount <= 10) then 168 | local errEveryPixel = loss[1] / (opt.batchSize * opt.numChannels * opt.Width * opt.Height) 169 | utils.printTime(string.format("Epoch %d, sample %d, training loss: %f, average loss per pixel: %f, current learning rate: %f", 170 | iEpochs, loopCount, loss[1]/opt.batchSize, errEveryPixel, optimState.learningRate)) 171 | end 172 | 173 | if (opt.checkpointEvery > 0 and loopCount % opt.checkpointEvery == 0) or i == numBatches then 174 | model:evaluate() 175 | local valLoss = test(criterion) 176 | print(optimState) 177 | model:training() 178 | end 179 | 180 | if (opt.checkpointEvery > 0 and loopCount % opt.checkpointEvery == 0) or i == numBatches then 181 | 182 | local checkpoint = { 183 | opt = opt 184 | --trainLossHistory = trainLossHistory, 185 | --valLossHistory = valLossHistory 186 | } 187 | 188 | local filename 189 | if i == opt.numEpochs then 190 | filename = string.format('%s_%s_ep%s_%s.t7', opt.checkpointName, opt.learningRate, iEpochs, 'final') 191 | else 192 | filename = string.format('%s_%s_ep%s_%d.t7', opt.checkpointName, opt.learningRate, iEpochs, loopCount) 193 | end 194 | 195 | -- Make sure the output directory exists before we try to write it 196 | paths.mkdir(paths.dirname(filename)) 197 | 198 | -- Cast model to float so it can be used on CPU 199 | -- model:float() 200 | checkpoint.model = model 201 | torch.save(filename, checkpoint) 202 | 203 | -- Cast model back so that it can continue to be used 204 | -- model:type(opt.dtype) 205 | utils.printTime(string.format("Saved checkpoint model and opt at %s", filename)) 206 | end 207 | 208 | loopCount = loopCount + 1 209 | collectgarbage() 210 | 211 | end 212 | 213 | epochLoss = torch.mean(torch.Tensor(epochLoss)) 214 | 215 | table.insert(trainLossHistory, epochLoss) 216 | 217 | utils. printTime(string.format("Epoch %d training loss: %f", iEpochs, epochLoss)) 218 | 219 | collectgarbage() 220 | end 221 | end 222 | 223 | function test(criterion) 224 | utils.printTime(string.format("Starting testing on %s dataset", 'val')) 225 | 226 | local evalData = {} 227 | evalData.predictedLabels = {} 228 | evalData.trueLabels = {} 229 | 230 | local totalLoss = 0 231 | 232 | local loopCount = 1 233 | 234 | local dataInitialIndex = torch.range(1, loader.val.inputNum, opt.batchSize) 235 | local numBatches = dataInitialIndex:size(1) 236 | local dataIndex = torch.randperm(numBatches) 237 | 238 | for i = 1, numBatches do 239 | local data1, data2, data3, data4, data5, data6, data7, data8, data9, data10, label= loader:getBatch('val', dataInitialIndex[dataIndex[i]]) 240 | 241 | if opt.cuda == 1 then 242 | data1 = data1:cuda() 243 | data2 = data2:cuda() 244 | data3 = data3:cuda() 245 | data4 = data4:cuda() 246 | data5 = data5:cuda() 247 | data6 = data6:cuda() 248 | data7 = data7:cuda() 249 | data8 = data8:cuda() 250 | data9 = data9:cuda() 251 | data10 = data10:cuda() 252 | label = label:cuda() 253 | end 254 | 255 | local valModelOut = model:forward({data1, data2, data3, data4, data5, data6, data7, data8, data9, data10}) 256 | local loss = criterion:forward(valModelOut, label) 257 | totalLoss = totalLoss + loss 258 | 259 | if((opt.printEvery > 0 and loopCount % opt.printEvery == 0) or loopCount <= 10) then 260 | local errEveryPixel = loss / (opt.batchSize * opt.numChannels * opt.Width * opt.Height) 261 | utils.printTime("----------------------------------------") 262 | utils.printTime(string.format("Valdation: sample %d, Valdation loss: %f, average loss per pixel: %f, current learning rate: %f", loopCount, loss/opt.batchSize, errEveryPixel, opt.learningRate)) 263 | end 264 | 265 | loopCount = loopCount + 1 266 | end 267 | 268 | local avgLoss = totalLoss / ( (loopCount-1) * opt.batchSize) 269 | utils.printTime("+++++++++++++++++++++++++++++++++++++++++++++++") 270 | 271 | local errEveryPixel = avgLoss / (opt.batchSize * opt.numChannels * opt.Width * opt.Height) 272 | utils.printTime(string.format("Valdation: sample %d, average validation loss: %f, average loss per pixel: %f", loopCount, avgLoss, errEveryPixel)) 273 | 274 | collectgarbage() 275 | return loss 276 | end 277 | 278 | train(model) 279 | -------------------------------------------------------------------------------- /model/cr_5_25/util/DataHandler.lua: -------------------------------------------------------------------------------- 1 | require 'torch' 2 | require 'hdf5' 3 | local utils = require 'util.utils' 4 | 5 | local DataHandler = torch.class('DataHandler') 6 | 7 | trainDataFile = '/home/user/kaixu/myGitHub/VideoReconNet/data/TrainData_5_25_mix_feed_10172016.h5' 8 | valDataFile = '/home/user/kaixu/myGitHub/VideoReconNet/data/ValData_5_25_mix_feed_10172016.h5' 9 | --testDataFile = '/home/user/kaixu/myGitHub/VideoReconNet/recon/Feature_conv5_5_25_recon.h5' 10 | testDataFile = '/home/user/kaixu/myGitHub/VideoReconNet/model/cr_5_25_feed/recon/data/TestData_5_25_mix_feed_10172016.h5' 11 | 12 | function DataHandler:__init(args) 13 | self.train = {} 14 | self.val = {} 15 | self.test= {} 16 | 17 | self.inputDim = 0 18 | 19 | self.imgParam = {} 20 | self.imgParam.batchSize = utils.getArgs(args, 'batchSize') 21 | self.imgParam.seqLength = utils.getArgs(args, 'seqLength') 22 | self.imgParam.numChannels = utils.getArgs(args, 'numChannels') 23 | self.imgParam.Height = utils.getArgs(args, 'Height') 24 | self.imgParam.Width = utils.getArgs(args, 'Width') 25 | self.imgParam.measurements1 = utils.getArgs(args, 'measurements1') 26 | self.imgParam.measurements2 = utils.getArgs(args, 'measurements2') 27 | end 28 | 29 | function DataHandler:loadData() 30 | local myFile = hdf5.open(trainDataFile, 'r') 31 | self.train.data1 = myFile:read('data1'):all() 32 | 33 | local totalSamples = torch.floor(self.train.data1:size(1) / self.imgParam.batchSize)*self.imgParam.batchSize 34 | self.train.data1 = self.train.data1[{{1,totalSamples},{},{},{},{}}] 35 | 36 | self.train.data2 = myFile:read('data2'):all() 37 | self.train.data2 = self.train.data2[{{1,totalSamples},{},{},{},{}}] 38 | 39 | self.train.label = myFile:read('label'):all() 40 | self.train.label = self.train.label[{{1,totalSamples},{},{},{},{}}] 41 | 42 | self.train.inputNum = totalSamples 43 | 44 | myFile = hdf5.open(valDataFile, 'r') 45 | self.val.data1 = myFile:read('data1'):all() 46 | totalSamples = torch.floor(self.val.data1:size(1) / self.imgParam.batchSize)*self.imgParam.batchSize 47 | self.val.data1 = self.val.data1[{{1,totalSamples},{},{},{},{}}] 48 | 49 | self.val.data2 = myFile:read('data2'):all() 50 | self.val.data2 = self.val.data2[{{1,totalSamples},{},{},{},{}}] 51 | 52 | self.val.label = myFile:read('label'):all() 53 | self.val.label = self.val.label[{{1,totalSamples},{},{},{},{}}] 54 | 55 | self.val.inputNum = totalSamples 56 | 57 | return self 58 | end 59 | 60 | function DataHandler:loadTestData() 61 | local myFile = hdf5.open(testDataFile, 'r') 62 | self.test.data1 = myFile:read('data1'):all() 63 | totalSamples = torch.floor(self.test.data1:size(1) / self.imgParam.batchSize)*self.imgParam.batchSize 64 | self.test.data1 = self.test.data1[{{1,totalSamples},{},{},{},{}}] 65 | self.test.data2 = myFile:read('data2'):all() 66 | self.test.data2 = self.test.data2[{{1,totalSamples},{},{},{},{}}] 67 | self.test.label = myFile:read('label'):all() 68 | self.test.label = self.test.label[{{1,totalSamples},{},{},{},{}}] 69 | self.test.inputNum = totalSamples 70 | 71 | return self 72 | end 73 | 74 | function DataHandler:getBatch(dataset, index) 75 | local i = 1 76 | local data1_1 = torch.FloatTensor() 77 | local data2_1 = torch.FloatTensor() 78 | local data2_2 = torch.FloatTensor() 79 | local data2_3 = torch.FloatTensor() 80 | local data2_4 = torch.FloatTensor() 81 | local data2_5 = torch.FloatTensor() 82 | local data2_6 = torch.FloatTensor() 83 | local data2_7 = torch.FloatTensor() 84 | local data2_8 = torch.FloatTensor() 85 | local data2_9 = torch.FloatTensor() 86 | local data2_10 = torch.FloatTensor() 87 | 88 | local label = torch.FloatTensor() 89 | local rawData1 = torch.FloatTensor() 90 | local rawData2 = torch.FloatTensor() 91 | local rawLabel = torch.FloatTensor() 92 | 93 | for i = 1,self.imgParam.batchSize do 94 | rawData1 = self[dataset].data1[index+i-1] 95 | rawData2 = self[dataset].data2[index+i-1] 96 | rawLabel = self[dataset].label[index+i-1] 97 | 98 | if i==1 then 99 | data1_1 = rawData1[{{1}, {}, {}, {} }] -- [1, numChannels, 1, mea] 100 | data2_1 = rawData2[{{1}, {}, {}, {} }] -- [9, numChannels, 1, mea] 101 | data2_2 = rawData2[{{2}, {}, {}, {1,self.imgParam.measurements2} }] -- [9, numChannels, 1, mea] 102 | data2_3 = rawData2[{{3}, {}, {}, {1,self.imgParam.measurements2} }] -- [9, numChannels, 1, mea] 103 | data2_4 = rawData2[{{4}, {}, {}, {1,self.imgParam.measurements2} }] -- [9, numChannels, 1, mea] 104 | data2_5 = rawData2[{{5}, {}, {}, {1,self.imgParam.measurements2} }] -- [9, numChannels, 1, mea] 105 | data2_6 = rawData2[{{6}, {}, {}, {1,self.imgParam.measurements2} }] -- [9, numChannels, 1, mea] 106 | data2_7 = rawData2[{{7}, {}, {}, {1,self.imgParam.measurements2} }] -- [9, numChannels, 1, mea] 107 | data2_8 = rawData2[{{8}, {}, {}, {1,self.imgParam.measurements2} }] -- [9, numChannels, 1, mea] 108 | data2_9 = rawData2[{{9}, {}, {}, {1,self.imgParam.measurements2} }] -- [9, numChannels, 1, mea] 109 | data2_10 = rawData2[{{10}, {}, {}, {1,self.imgParam.measurements2} }] -- [9, numChannels, 1, mea] 110 | 111 | label = rawLabel:view(self.imgParam.seqLength, -1) 112 | label = label:view(-1, self.imgParam.seqLength, self.imgParam.Height*self.imgParam.Width) 113 | else 114 | data1_1 = data1_1:cat(rawData1[{{1}, {}, {}, {} }], 1) 115 | data2_1 = data2_1:cat(rawData2[{{1}, {}, {}, {} }], 1) 116 | data2_2 = data2_2:cat(rawData2[{{2}, {}, {}, {1,self.imgParam.measurements2} }], 1) 117 | data2_3 = data2_3:cat(rawData2[{{3}, {}, {}, {1,self.imgParam.measurements2} }], 1) 118 | data2_4 = data2_4:cat(rawData2[{{4}, {}, {}, {1,self.imgParam.measurements2} }], 1) 119 | data2_5 = data2_5:cat(rawData2[{{5}, {}, {}, {1,self.imgParam.measurements2} }], 1) 120 | data2_6 = data2_6:cat(rawData2[{{6}, {}, {}, {1,self.imgParam.measurements2} }], 1) 121 | data2_7 = data2_7:cat(rawData2[{{7}, {}, {}, {1,self.imgParam.measurements2} }], 1) 122 | data2_8 = data2_8:cat(rawData2[{{8}, {}, {}, {1,self.imgParam.measurements2} }], 1) 123 | data2_9 = data2_9:cat(rawData2[{{9}, {}, {}, {1,self.imgParam.measurements2} }], 1) 124 | data2_10 = data2_10:cat(rawData2[{{10}, {}, {}, {1,self.imgParam.measurements2} }], 1) 125 | 126 | labelTmp = rawLabel:view(self.imgParam.seqLength, -1) 127 | labelTmp = rawLabel:view(-1, self.imgParam.seqLength, self.imgParam.Height*self.imgParam.Width) 128 | label = label:cat(labelTmp, 1) 129 | end 130 | end 131 | label = label:transpose(1,2) 132 | 133 | --data1 = data1:reshape(-1, 1, self.imgParam.numChannels, self.imgParam.measurements) 134 | --data2 = data2:reshape(-1, (self.imgParam.seqLength-1), 1, self.imgParam.measurements2) 135 | --label1 = label1:reshape(-1, 1, self.imgParam.numChannels, self.imgParam.Height, self.imgParam.Width) 136 | --label2 = label2:reshape(-1, (self.imgParam.seqLength-1), self.imgParam.numChannels, self.imgParam.Height, self.imgParam.Width) 137 | 138 | -- batch.data = {data1, data2} 139 | -- batch.label = {label1, label2} 140 | 141 | local data1 = data1_1 142 | local data2 = data2_1:cat(data2_2, 4) 143 | local data3 = data2_1:cat(data2_3, 4) 144 | local data4 = data2_1:cat(data2_4, 4) 145 | local data5 = data2_1:cat(data2_5, 4) 146 | local data6 = data2_1:cat(data2_6, 4) 147 | local data7 = data2_1:cat(data2_7, 4) 148 | local data8 = data2_1:cat(data2_8, 4) 149 | local data9 = data2_1:cat(data2_9, 4) 150 | local data10 = data2_1:cat(data2_10, 4) 151 | 152 | return data1, data2, data3, data4, data5, data6, data7, data8, data9, data10, label 153 | end 154 | 155 | function DataHandler:getValData() 156 | return self.val 157 | end 158 | -------------------------------------------------------------------------------- /model/cr_5_25/util/DataHandler2.lua: -------------------------------------------------------------------------------- 1 | require 'torch' 2 | require 'hdf5' 3 | local utils = require 'util.utils' 4 | 5 | local DataHandler2 = torch.class('DataHandler2') 6 | 7 | trainDataFile = '/home/user/kaixu/myGitHub/VideoReconNet/data/TrainData_5_25_mix_feed_10172016.h5' 8 | valDataFile = '/home/user/kaixu/myGitHub/VideoReconNet/data/ValData_5_25_mix_feed_10172016.h5' 9 | --testDataFile = '/home/user/kaixu/myGitHub/VideoReconNet/recon/Feature_conv5_5_25_recon.h5' 10 | testDataFile = '/home/user/kaixu/myGitHub/VideoReconNet/data/TestData_1_25_10172016.h5' 11 | 12 | function DataHandler2:__init(args) 13 | self.train = {} 14 | self.val = {} 15 | self.test= {} 16 | 17 | self.inputDim = 0 18 | 19 | self.imgParam = {} 20 | self.imgParam.batchSize = utils.getArgs(args, 'batchSize') 21 | self.imgParam.seqLength = utils.getArgs(args, 'seqLength') 22 | self.imgParam.numChannels = utils.getArgs(args, 'numChannels') 23 | self.imgParam.Height = utils.getArgs(args, 'Height') 24 | self.imgParam.Width = utils.getArgs(args, 'Width') 25 | self.imgParam.measurements = utils.getArgs(args, 'measurements') 26 | end 27 | 28 | function DataHandler2:loadData() 29 | local myFile = hdf5.open(trainDataFile, 'r') 30 | self.train.data1 = myFile:read('data1'):all() 31 | 32 | local totalSamples = torch.floor(self.train.data1:size(1) / self.imgParam.batchSize)*self.imgParam.batchSize 33 | self.train.data1 = self.train.data1[{{1,totalSamples},{},{},{},{}}] 34 | 35 | self.train.data2 = myFile:read('data2'):all() 36 | self.train.data2 = self.train.data2[{{1,totalSamples},{},{},{},{}}] 37 | 38 | self.train.label = myFile:read('label'):all() 39 | self.train.label = self.train.label[{{1,totalSamples},{},{},{},{}}] 40 | 41 | self.train.inputNum = totalSamples 42 | 43 | myFile = hdf5.open(valDataFile, 'r') 44 | self.val.data1 = myFile:read('data1'):all() 45 | totalSamples = torch.floor(self.val.data1:size(1) / self.imgParam.batchSize)*self.imgParam.batchSize 46 | self.val.data1 = self.val.data1[{{1,totalSamples},{},{},{},{}}] 47 | 48 | self.val.data2 = myFile:read('data2'):all() 49 | self.val.data2 = self.val.data2[{{1,totalSamples},{},{},{},{}}] 50 | 51 | self.val.label = myFile:read('label'):all() 52 | self.val.label = self.val.label[{{1,totalSamples},{},{},{},{}}] 53 | 54 | self.val.inputNum = totalSamples 55 | 56 | return self 57 | end 58 | 59 | function DataHandler2:loadTestData() 60 | local myFile = hdf5.open(testDataFile, 'r') 61 | self.test.data = myFile:read('data'):all() 62 | local totalSamples = torch.floor(self.test.data:size(1) / self.imgParam.batchSize)*self.imgParam.batchSize 63 | self.test.data = self.test.data[{{1,totalSamples},{},{},{},{}}] 64 | self.test.label = myFile:read('label'):all() 65 | self.test.label = self.test.label[{{1,totalSamples},{},{},{},{}}] 66 | self.test.inputNum = totalSamples 67 | 68 | return self 69 | end 70 | 71 | function DataHandler2:getBatch(dataset, index) 72 | local i = 1 73 | local data1_1 = torch.FloatTensor() 74 | local data2_1 = torch.FloatTensor() 75 | local data2_2 = torch.FloatTensor() 76 | local data2_3 = torch.FloatTensor() 77 | local data2_4 = torch.FloatTensor() 78 | local data2_5 = torch.FloatTensor() 79 | local data2_6 = torch.FloatTensor() 80 | local data2_7 = torch.FloatTensor() 81 | local data2_8 = torch.FloatTensor() 82 | local data2_9 = torch.FloatTensor() 83 | local data2_10 = torch.FloatTensor() 84 | 85 | local label = torch.FloatTensor() 86 | local rawData1 = torch.FloatTensor() 87 | local rawData2 = torch.FloatTensor() 88 | local rawLabel = torch.FloatTensor() 89 | 90 | for i = 1,self.imgParam.batchSize do 91 | rawData1 = self[dataset].data1[index+i-1] 92 | rawData2 = self[dataset].data2[index+i-1] 93 | rawLabel = self[dataset].label[index+i-1] 94 | 95 | if i==1 then 96 | data1_1 = rawData1[{{1}, {}, {}, {}}] -- [1, numChannels, 1, mea] 97 | data2_1 = rawData2[{{1}, {}, {}, {} }] -- [9, numChannels, 1, mea] 98 | data2_2 = rawData2[{{2}, {}, {}, {} }] -- [9, numChannels, 1, mea] 99 | data2_3 = rawData2[{{3}, {}, {}, {} }] -- [9, numChannels, 1, mea] 100 | data2_4 = rawData2[{{4}, {}, {}, {} }] -- [9, numChannels, 1, mea] 101 | data2_5 = rawData2[{{5}, {}, {}, {} }] -- [9, numChannels, 1, mea] 102 | data2_6 = rawData2[{{6}, {}, {}, {} }] -- [9, numChannels, 1, mea] 103 | data2_7 = rawData2[{{7}, {}, {}, {} }] -- [9, numChannels, 1, mea] 104 | data2_8 = rawData2[{{8}, {}, {}, {} }] -- [9, numChannels, 1, mea] 105 | data2_9 = rawData2[{{9}, {}, {}, {} }] -- [9, numChannels, 1, mea] 106 | data2_10 = rawData2[{{10}, {}, {}, {} }] -- [9, numChannels, 1, mea] 107 | 108 | label = rawLabel:view(self.imgParam.seqLength, -1) 109 | label = label:view(-1, self.imgParam.seqLength, self.imgParam.Height*self.imgParam.Width) 110 | else 111 | data1_1 = data1_1:cat(rawData1[{{1}, {}, {}, {}}], 1) 112 | data2_1 = data2_1:cat(rawData2[{{1}, {}, {}, {} }], 1) 113 | data2_2 = data2_2:cat(rawData2[{{2}, {}, {}, {} }], 1) 114 | data2_3 = data2_3:cat(rawData2[{{3}, {}, {}, {} }], 1) 115 | data2_4 = data2_4:cat(rawData2[{{4}, {}, {}, {} }], 1) 116 | data2_5 = data2_5:cat(rawData2[{{5}, {}, {}, {} }], 1) 117 | data2_6 = data2_6:cat(rawData2[{{6}, {}, {}, {} }], 1) 118 | data2_7 = data2_7:cat(rawData2[{{7}, {}, {}, {} }], 1) 119 | data2_8 = data2_8:cat(rawData2[{{8}, {}, {}, {} }], 1) 120 | data2_9 = data2_9:cat(rawData2[{{9}, {}, {}, {} }], 1) 121 | data2_10 = data2_10:cat(rawData2[{{10}, {}, {}, {} }], 1) 122 | 123 | labelTmp = rawLabel:view(self.imgParam.seqLength, -1) 124 | labelTmp = rawLabel:view(-1, self.imgParam.seqLength, self.imgParam.Height*self.imgParam.Width) 125 | label = label:cat(labelTmp, 1) 126 | end 127 | end 128 | label = label:transpose(1,2) 129 | 130 | --data1 = data1:reshape(-1, 1, self.imgParam.numChannels, self.imgParam.measurements) 131 | --data2 = data2:reshape(-1, (self.imgParam.seqLength-1), 1, self.imgParam.measurements2) 132 | --label1 = label1:reshape(-1, 1, self.imgParam.numChannels, self.imgParam.Height, self.imgParam.Width) 133 | --label2 = label2:reshape(-1, (self.imgParam.seqLength-1), self.imgParam.numChannels, self.imgParam.Height, self.imgParam.Width) 134 | 135 | -- batch.data = {data1, data2} 136 | -- batch.label = {label1, label2} 137 | 138 | local data1 = data1_1 139 | local data2 = data2_1:cat(data2_2, 2) 140 | local data3 = data2_1:cat(data2_3, 2) 141 | local data4 = data2_1:cat(data2_4, 2) 142 | local data5 = data2_1:cat(data2_5, 2) 143 | local data6 = data2_1:cat(data2_6, 2) 144 | local data7 = data2_1:cat(data2_7, 2) 145 | local data8 = data2_1:cat(data2_8, 2) 146 | local data9 = data2_1:cat(data2_9, 2) 147 | local data10 = data2_1:cat(data2_10, 2) 148 | 149 | return data1, data2, data3, data4, data5, data6, data7, data8, data9, data10, label 150 | end 151 | 152 | function DataHandler2:getValData() 153 | return self.val 154 | end 155 | -------------------------------------------------------------------------------- /model/cr_5_25/util/utils.lua: -------------------------------------------------------------------------------- 1 | local utils = {} 2 | 3 | function utils.getArgs(args, name, default) 4 | if args == nil then args = {} end 5 | if args[name] == nil and default == nil then 6 | assert(false, string.format('"%s" expected and not given', name)) 7 | elseif args[name] == nil then 8 | return default 9 | else 10 | return args[name] 11 | end 12 | end 13 | 14 | --[[ 15 | Prints the time and a message in the form of 16 | 17 |