├── README.md ├── bn_layer.cpp ├── bn_layer.cu ├── caffe.proto ├── common_layers.hpp ├── layer_factory.cpp ├── lenet_BN_sgd.log ├── lenet_BN_sgd_solver.prototxt └── lenet_BN_train_valid.prototxt /README.md: -------------------------------------------------------------------------------- 1 | # Batch Normalization Layer for Caffe 2 | 3 | This implementation of [Batch Normalization](http://arxiv.org/pdf/1502.03167v1.pdf) is based on [MVNLayer](https://github.com/BVLC/caffe/blob/master/src/caffe/layers/mvn_layer.cpp) in Caffe. 4 | 5 | To add this layer, you have to modify `common_layers.hpp`, `layer_factory.cpp` and `caffe.proto`. See [Caffe wiki](https://github.com/BVLC/caffe/wiki/Development) 6 | 7 | # NOTE 8 | 9 | This implementation is very basic which just performs batch normalization computation. Two pieces in [the paper](http://arxiv.org/pdf/1502.03167v1.pdf) are still missing here: 10 | * fixed mean \& variance for inference 11 | * per batch shuffling for thorough randomness 12 | 13 | # Update log 14 | 15 | 2016/01/18 An improved version is provided [here](https://github.com/ChenglongChen/caffe-windows) with moving average statistics for inference. 16 | -------------------------------------------------------------------------------- /bn_layer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "caffe/common_layers.hpp" 5 | #include "caffe/filler.hpp" 6 | #include "caffe/layer.hpp" 7 | #include "caffe/util/math_functions.hpp" 8 | 9 | namespace caffe { 10 | 11 | template 12 | void BNLayer::LayerSetUp(const vector*>& bottom, 13 | vector*>* top) { 14 | 15 | // Figure out the dimensions 16 | N_ = bottom[0]->num(); 17 | C_ = bottom[0]->channels(); 18 | H_ = bottom[0]->height(); 19 | W_ = bottom[0]->width(); 20 | var_eps_ = 1e-10; 21 | 22 | // reshape blob 23 | (*top)[0]->Reshape(N_, C_, H_, W_); 24 | x_norm_.Reshape(N_, C_, H_, W_); 25 | 26 | // mean 27 | spatial_mean_.Reshape(N_, C_, 1, 1); 28 | batch_mean_.Reshape(1, C_, 1, 1); 29 | // variance 30 | spatial_variance_.Reshape(N_, C_, 1, 1); 31 | batch_variance_.Reshape(1, C_, 1, 1); 32 | // buffer blod 33 | buffer_blob_.Reshape(N_, C_, H_, W_); 34 | 35 | // fill spatial multiplier 36 | spatial_sum_multiplier_.Reshape(1, 1, H_, W_); 37 | Dtype* spatial_multiplier_data = spatial_sum_multiplier_.mutable_cpu_data(); 38 | caffe_set(spatial_sum_multiplier_.count(), Dtype(1), spatial_multiplier_data); 39 | // fill batch multiplier 40 | batch_sum_multiplier_.Reshape(N_, 1, 1, 1); 41 | Dtype* batch_multiplier_data = batch_sum_multiplier_.mutable_cpu_data(); 42 | caffe_set(batch_sum_multiplier_.count(), Dtype(1), batch_multiplier_data); 43 | 44 | // Check if we need to set up the weights 45 | if (this->blobs_.size() > 0) { 46 | LOG(INFO) << "Skipping parameter initialization"; 47 | } else { 48 | this->blobs_.resize(2); 49 | 50 | // fill scale with scale_filler 51 | this->blobs_[0].reset(new Blob(1, C_, 1, 1)); 52 | shared_ptr > scale_filler(GetFiller( 53 | this->layer_param_.bn_param().scale_filler())); 54 | scale_filler->Fill(this->blobs_[0].get()); 55 | 56 | // fill shift with shift_filler 57 | this->blobs_[1].reset(new Blob(1, C_, 1, 1)); 58 | shared_ptr > shift_filler(GetFiller( 59 | this->layer_param_.bn_param().shift_filler())); 60 | shift_filler->Fill(this->blobs_[1].get()); 61 | 62 | /* 63 | // fill scale and shift with data mean and variance 64 | const Dtype* bottom_data = bottom[0]->cpu_data(); 65 | Dtype* top_data = (*top)[0]->mutable_cpu_data(); 66 | 67 | // put the squares of bottom into buffer_blob_ 68 | caffe_powx(bottom[0]->count(), bottom_data, Dtype(2), 69 | buffer_blob_.mutable_cpu_data()); 70 | 71 | // computes variance using var(X) = E(X^2) - (EX)^2 72 | // EX across spatial 73 | caffe_cpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1. / (H_ * W_)), bottom_data, 74 | spatial_sum_multiplier_.cpu_data(), Dtype(0), spatial_mean_.mutable_cpu_data()); 75 | // EX across batch 76 | caffe_cpu_gemv(CblasTrans, N_, C_, Dtype(1. / N_), spatial_mean_.cpu_data(), 77 | batch_sum_multiplier_.cpu_data(), Dtype(0), batch_mean_.mutable_cpu_data()); 78 | // fill shift 79 | caffe_copy(batch_mean_.count(), batch_mean_.cpu_data(), this->blobs_[1]->mutable_cpu_data()); 80 | 81 | // E(X^2) across spatial 82 | caffe_cpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1. / (H_ * W_)), buffer_blob_.cpu_data(), 83 | spatial_sum_multiplier_.cpu_data(), Dtype(0), spatial_variance_.mutable_cpu_data()); 84 | // E(X^2) across batch 85 | caffe_cpu_gemv(CblasTrans, N_, C_, Dtype(1. / N_), spatial_variance_.cpu_data(), 86 | batch_sum_multiplier_.cpu_data(), Dtype(0), batch_variance_.mutable_cpu_data()); 87 | 88 | caffe_powx(batch_mean_.count(), batch_mean_.cpu_data(), Dtype(2), 89 | buffer_blob_.mutable_cpu_data()); // (EX)^2 90 | caffe_sub(batch_mean_.count(), batch_variance_.cpu_data(), buffer_blob_.cpu_data(), 91 | batch_variance_.mutable_cpu_data()); // variance 92 | // normalize variance 93 | caffe_powx(batch_variance_.count(), batch_variance_.cpu_data(), Dtype(0.5), 94 | batch_variance_.mutable_cpu_data()); 95 | caffe_add_scalar(batch_variance_.count(), var_eps_, batch_variance_.mutable_cpu_data()); 96 | // fill scale 97 | caffe_copy(batch_variance_.count(), batch_variance_.cpu_data(), this->blobs_[0]->mutable_cpu_data()); 98 | */ 99 | 100 | } // parameter initialization 101 | this->param_propagate_down_.resize(this->blobs_.size(), true); 102 | } 103 | 104 | template 105 | void BNLayer::Forward_cpu(const vector*>& bottom, 106 | vector*>* top) { 107 | const Dtype* const_bottom_data = bottom[0]->cpu_data(); 108 | const Dtype* const_top_data = (*top)[0]->cpu_data(); 109 | Dtype* top_data = (*top)[0]->mutable_cpu_data(); 110 | 111 | const Dtype* scale_data = this->blobs_[0]->cpu_data(); 112 | const Dtype* shift_data = this->blobs_[1]->cpu_data(); 113 | 114 | // put the squares of bottom into buffer_blob_ 115 | caffe_powx(bottom[0]->count(), const_bottom_data, Dtype(2), 116 | buffer_blob_.mutable_cpu_data()); 117 | 118 | // computes variance using var(X) = E(X^2) - (EX)^2 119 | // EX across spatial 120 | caffe_cpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1. / (H_ * W_)), const_bottom_data, 121 | spatial_sum_multiplier_.cpu_data(), Dtype(0), spatial_mean_.mutable_cpu_data()); 122 | // EX across batch 123 | caffe_cpu_gemv(CblasTrans, N_, C_, Dtype(1. / N_), spatial_mean_.cpu_data(), 124 | batch_sum_multiplier_.cpu_data(), Dtype(0), batch_mean_.mutable_cpu_data()); 125 | 126 | // E(X^2) across spatial 127 | caffe_cpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1. / (H_ * W_)), buffer_blob_.cpu_data(), 128 | spatial_sum_multiplier_.cpu_data(), Dtype(0), spatial_variance_.mutable_cpu_data()); 129 | // E(X^2) across batch 130 | caffe_cpu_gemv(CblasTrans, N_, C_, Dtype(1. / N_), spatial_variance_.cpu_data(), 131 | batch_sum_multiplier_.cpu_data(), Dtype(0), batch_variance_.mutable_cpu_data()); 132 | 133 | caffe_powx(batch_mean_.count(), batch_mean_.cpu_data(), Dtype(2), 134 | buffer_blob_.mutable_cpu_data()); // (EX)^2 135 | caffe_sub(batch_mean_.count(), batch_variance_.cpu_data(), buffer_blob_.cpu_data(), 136 | batch_variance_.mutable_cpu_data()); // variance 137 | 138 | // do mean and variance normalization 139 | // subtract mean 140 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 141 | batch_sum_multiplier_.cpu_data(), batch_mean_.cpu_data(), Dtype(0), 142 | spatial_mean_.mutable_cpu_data()); 143 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(-1), 144 | spatial_mean_.cpu_data(), spatial_sum_multiplier_.cpu_data(), Dtype(0), 145 | buffer_blob_.mutable_cpu_data()); 146 | 147 | caffe_add(buffer_blob_.count(), const_bottom_data, buffer_blob_.cpu_data(), top_data); 148 | 149 | // normalize variance 150 | caffe_add_scalar(batch_variance_.count(), var_eps_, batch_variance_.mutable_cpu_data()); 151 | caffe_powx(batch_variance_.count(), batch_variance_.cpu_data(), Dtype(0.5), 152 | batch_variance_.mutable_cpu_data()); 153 | 154 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 155 | batch_sum_multiplier_.cpu_data(), batch_variance_.cpu_data(), Dtype(0), 156 | spatial_variance_.mutable_cpu_data()); 157 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 158 | spatial_variance_.cpu_data(), spatial_sum_multiplier_.cpu_data(), Dtype(0), 159 | buffer_blob_.mutable_cpu_data()); 160 | 161 | caffe_div(buffer_blob_.count(), const_top_data, buffer_blob_.cpu_data(), top_data); 162 | 163 | // save x_norm 164 | caffe_copy(buffer_blob_.count(), const_top_data, x_norm_.mutable_cpu_data()); 165 | 166 | // scale 167 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 168 | batch_sum_multiplier_.cpu_data(), scale_data, Dtype(0), 169 | spatial_variance_.mutable_cpu_data()); 170 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 171 | spatial_variance_.cpu_data(), spatial_sum_multiplier_.cpu_data(), Dtype(0), 172 | buffer_blob_.mutable_cpu_data()); 173 | caffe_mul(buffer_blob_.count(), const_top_data, buffer_blob_.cpu_data(), top_data); 174 | 175 | // shift 176 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 177 | batch_sum_multiplier_.cpu_data(), shift_data, Dtype(0), 178 | spatial_mean_.mutable_cpu_data()); 179 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 180 | spatial_mean_.cpu_data(), spatial_sum_multiplier_.cpu_data(), Dtype(0), 181 | buffer_blob_.mutable_cpu_data()); 182 | caffe_add(buffer_blob_.count(), const_top_data, buffer_blob_.cpu_data(), top_data); 183 | 184 | } 185 | 186 | template 187 | void BNLayer::Backward_cpu(const vector*>& top, 188 | const vector& propagate_down, 189 | vector*>* bottom) { 190 | const Dtype* const_bottom_diff = (*bottom)[0]->cpu_diff(); 191 | Dtype* bottom_diff = (*bottom)[0]->mutable_cpu_diff(); 192 | const Dtype* const_top_diff = top[0]->cpu_diff(); 193 | 194 | Dtype* scale_diff = this->blobs_[0]->mutable_cpu_diff(); 195 | Dtype* shift_diff = this->blobs_[1]->mutable_cpu_diff(); 196 | const Dtype* scale_data = this->blobs_[0]->cpu_data(); 197 | 198 | // gradient w.r.t. scale 199 | caffe_mul(buffer_blob_.count(), x_norm_.cpu_data(), const_top_diff, buffer_blob_.mutable_cpu_data()); 200 | // EX across spatial 201 | caffe_cpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1), buffer_blob_.cpu_data(), 202 | spatial_sum_multiplier_.cpu_data(), Dtype(0), spatial_variance_.mutable_cpu_data()); 203 | // EX across batch 204 | caffe_cpu_gemv(CblasTrans, N_, C_, Dtype(1), spatial_variance_.cpu_data(), 205 | batch_sum_multiplier_.cpu_data(), Dtype(0), scale_diff); 206 | 207 | // gradient w.r.t. shift 208 | // EX across spatial 209 | caffe_cpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1), const_top_diff, 210 | spatial_sum_multiplier_.cpu_data(), Dtype(0), spatial_mean_.mutable_cpu_data()); 211 | // EX across batch 212 | caffe_cpu_gemv(CblasTrans, N_, C_, Dtype(1), spatial_mean_.cpu_data(), 213 | batch_sum_multiplier_.cpu_data(), Dtype(0), shift_diff); 214 | 215 | // put scale * top_diff to buffer_blob_ 216 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 217 | batch_sum_multiplier_.cpu_data(), scale_data, Dtype(0), 218 | spatial_variance_.mutable_cpu_data()); 219 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 220 | spatial_variance_.cpu_data(), spatial_sum_multiplier_.cpu_data(), Dtype(0), 221 | buffer_blob_.mutable_cpu_data()); 222 | caffe_mul(buffer_blob_.count(), const_top_diff, buffer_blob_.cpu_data(), buffer_blob_.mutable_cpu_data()); 223 | 224 | // use new top diff for computation 225 | caffe_mul(buffer_blob_.count(), x_norm_.cpu_data(), buffer_blob_.cpu_data(), bottom_diff); 226 | // EX across spatial 227 | caffe_cpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1), const_bottom_diff, 228 | spatial_sum_multiplier_.cpu_data(), Dtype(0), spatial_mean_.mutable_cpu_data()); 229 | // EX across batch 230 | caffe_cpu_gemv(CblasTrans, N_, C_, Dtype(1), spatial_mean_.cpu_data(), 231 | batch_sum_multiplier_.cpu_data(), Dtype(0), batch_mean_.mutable_cpu_data()); 232 | 233 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 234 | batch_sum_multiplier_.cpu_data(), batch_mean_.cpu_data(), Dtype(0), 235 | spatial_mean_.mutable_cpu_data()); 236 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 237 | spatial_mean_.cpu_data(), spatial_sum_multiplier_.cpu_data(), Dtype(0), 238 | bottom_diff); 239 | 240 | caffe_mul(buffer_blob_.count(), x_norm_.cpu_data(), const_bottom_diff, bottom_diff); 241 | 242 | // EX across spatial 243 | caffe_cpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1), buffer_blob_.cpu_data(), 244 | spatial_sum_multiplier_.cpu_data(), Dtype(0), spatial_mean_.mutable_cpu_data()); 245 | // EX across batch 246 | caffe_cpu_gemv(CblasTrans, N_, C_, Dtype(1), spatial_mean_.cpu_data(), 247 | batch_sum_multiplier_.cpu_data(), Dtype(0), batch_mean_.mutable_cpu_data()); 248 | 249 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 250 | batch_sum_multiplier_.cpu_data(), batch_mean_.cpu_data(), Dtype(0), 251 | spatial_mean_.mutable_cpu_data()); 252 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 253 | spatial_mean_.cpu_data(), spatial_sum_multiplier_.cpu_data(), Dtype(1), 254 | bottom_diff); 255 | 256 | caffe_cpu_axpby(buffer_blob_.count(), Dtype(1), buffer_blob_.cpu_data(), Dtype(-1. / (N_ * H_ * W_)), 257 | bottom_diff); 258 | 259 | // variance normalization 260 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 261 | batch_sum_multiplier_.cpu_data(), batch_variance_.cpu_data(), Dtype(0), 262 | spatial_variance_.mutable_cpu_data()); 263 | caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 264 | spatial_variance_.cpu_data(), spatial_sum_multiplier_.cpu_data(), Dtype(0), 265 | buffer_blob_.mutable_cpu_data()); 266 | 267 | caffe_div(buffer_blob_.count(), const_bottom_diff, buffer_blob_.cpu_data(), bottom_diff); 268 | 269 | } 270 | 271 | #ifdef CPU_ONLY 272 | STUB_GPU(BNLayer); 273 | #endif 274 | 275 | INSTANTIATE_CLASS(BNLayer); 276 | 277 | } // namespace caffe 278 | -------------------------------------------------------------------------------- /bn_layer.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "caffe/common_layers.hpp" 5 | #include "caffe/filler.hpp" 6 | #include "caffe/layer.hpp" 7 | #include "caffe/util/math_functions.hpp" 8 | 9 | namespace caffe { 10 | 11 | template 12 | void BNLayer::Forward_gpu(const vector*>& bottom, 13 | vector*>* top) { 14 | const Dtype* const_bottom_data = bottom[0]->gpu_data(); 15 | const Dtype* const_top_data = (*top)[0]->gpu_data(); 16 | Dtype* top_data = (*top)[0]->mutable_gpu_data(); 17 | 18 | const Dtype* scale_data = this->blobs_[0]->gpu_data(); 19 | const Dtype* shift_data = this->blobs_[1]->gpu_data(); 20 | 21 | // put the squares of bottom into buffer_blob_ 22 | caffe_gpu_powx(bottom[0]->count(), const_bottom_data, Dtype(2), 23 | buffer_blob_.mutable_gpu_data()); 24 | 25 | // computes variance using var(X) = E(X^2) - (EX)^2 26 | // EX across spatial 27 | caffe_gpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1. / (H_ * W_)), const_bottom_data, 28 | spatial_sum_multiplier_.gpu_data(), Dtype(0), spatial_mean_.mutable_gpu_data()); 29 | // EX across batch 30 | caffe_gpu_gemv(CblasTrans, N_, C_, Dtype(1. / N_), spatial_mean_.gpu_data(), 31 | batch_sum_multiplier_.gpu_data(), Dtype(0), batch_mean_.mutable_gpu_data()); 32 | 33 | // E(X^2) across spatial 34 | caffe_gpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1. / (H_ * W_)), buffer_blob_.gpu_data(), 35 | spatial_sum_multiplier_.gpu_data(), Dtype(0), spatial_variance_.mutable_gpu_data()); 36 | // E(X^2) across batch 37 | caffe_gpu_gemv(CblasTrans, N_, C_, Dtype(1. / N_), spatial_variance_.gpu_data(), 38 | batch_sum_multiplier_.gpu_data(), Dtype(0), batch_variance_.mutable_gpu_data()); 39 | 40 | caffe_gpu_powx(batch_mean_.count(), batch_mean_.gpu_data(), Dtype(2), 41 | buffer_blob_.mutable_gpu_data()); // (EX)^2 42 | caffe_gpu_sub(batch_mean_.count(), batch_variance_.gpu_data(), buffer_blob_.gpu_data(), 43 | batch_variance_.mutable_gpu_data()); // variance 44 | 45 | // do mean and variance normalization 46 | // subtract mean 47 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 48 | batch_sum_multiplier_.gpu_data(), batch_mean_.gpu_data(), Dtype(0), 49 | spatial_mean_.mutable_gpu_data()); 50 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(-1), 51 | spatial_mean_.gpu_data(), spatial_sum_multiplier_.gpu_data(), Dtype(0), 52 | buffer_blob_.mutable_gpu_data()); 53 | 54 | caffe_gpu_add(buffer_blob_.count(), const_bottom_data, buffer_blob_.gpu_data(), top_data); 55 | 56 | // normalize variance 57 | caffe_gpu_add_scalar(batch_variance_.count(), var_eps_, batch_variance_.mutable_gpu_data()); 58 | caffe_gpu_powx(batch_variance_.count(), batch_variance_.gpu_data(), Dtype(0.5), 59 | batch_variance_.mutable_gpu_data()); 60 | 61 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 62 | batch_sum_multiplier_.gpu_data(), batch_variance_.gpu_data(), Dtype(0), 63 | spatial_variance_.mutable_gpu_data()); 64 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 65 | spatial_variance_.gpu_data(), spatial_sum_multiplier_.gpu_data(), Dtype(0), 66 | buffer_blob_.mutable_gpu_data()); 67 | 68 | caffe_gpu_div(buffer_blob_.count(), const_top_data, buffer_blob_.gpu_data(), top_data); 69 | 70 | // save x_norm 71 | caffe_copy(buffer_blob_.count(), const_top_data, x_norm_.mutable_gpu_data()); 72 | 73 | // scale 74 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 75 | batch_sum_multiplier_.gpu_data(), scale_data, Dtype(0), 76 | spatial_variance_.mutable_gpu_data()); 77 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 78 | spatial_variance_.gpu_data(), spatial_sum_multiplier_.gpu_data(), Dtype(0), 79 | buffer_blob_.mutable_gpu_data()); 80 | caffe_gpu_mul(buffer_blob_.count(), const_top_data, buffer_blob_.gpu_data(), top_data); 81 | 82 | // shift 83 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 84 | batch_sum_multiplier_.gpu_data(), shift_data, Dtype(0), 85 | spatial_mean_.mutable_gpu_data()); 86 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 87 | spatial_mean_.gpu_data(), spatial_sum_multiplier_.gpu_data(), Dtype(0), 88 | buffer_blob_.mutable_gpu_data()); 89 | caffe_gpu_add(buffer_blob_.count(), const_top_data, buffer_blob_.gpu_data(), top_data); 90 | 91 | } 92 | 93 | template 94 | void BNLayer::Backward_gpu(const vector*>& top, 95 | const vector& propagate_down, 96 | vector*>* bottom) { 97 | const Dtype* const_bottom_diff = (*bottom)[0]->gpu_diff(); 98 | Dtype* bottom_diff = (*bottom)[0]->mutable_gpu_diff(); 99 | const Dtype* const_top_diff = top[0]->gpu_diff(); 100 | 101 | Dtype* scale_diff = this->blobs_[0]->mutable_gpu_diff(); 102 | Dtype* shift_diff = this->blobs_[1]->mutable_gpu_diff(); 103 | const Dtype* scale_data = this->blobs_[0]->gpu_data(); 104 | 105 | // gradient w.r.t. scale 106 | caffe_gpu_mul(buffer_blob_.count(), x_norm_.gpu_data(), const_top_diff, buffer_blob_.mutable_gpu_data()); 107 | // EX across spatial 108 | caffe_gpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1), buffer_blob_.gpu_data(), 109 | spatial_sum_multiplier_.gpu_data(), Dtype(0), spatial_variance_.mutable_gpu_data()); 110 | // EX across batch 111 | caffe_gpu_gemv(CblasTrans, N_, C_, Dtype(1), spatial_variance_.gpu_data(), 112 | batch_sum_multiplier_.gpu_data(), Dtype(0), scale_diff); 113 | 114 | // gradient w.r.t. shift 115 | // EX across spatial 116 | caffe_gpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1), const_top_diff, 117 | spatial_sum_multiplier_.gpu_data(), Dtype(0), spatial_mean_.mutable_gpu_data()); 118 | // EX across batch 119 | caffe_gpu_gemv(CblasTrans, N_, C_, Dtype(1), spatial_mean_.gpu_data(), 120 | batch_sum_multiplier_.gpu_data(), Dtype(0), shift_diff); 121 | 122 | // put scale * top_diff to buffer_blob_ 123 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 124 | batch_sum_multiplier_.gpu_data(), scale_data, Dtype(0), 125 | spatial_variance_.mutable_gpu_data()); 126 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 127 | spatial_variance_.gpu_data(), spatial_sum_multiplier_.gpu_data(), Dtype(0), 128 | buffer_blob_.mutable_gpu_data()); 129 | caffe_gpu_mul(buffer_blob_.count(), const_top_diff, buffer_blob_.gpu_data(), buffer_blob_.mutable_gpu_data()); 130 | 131 | // use new top diff for computation 132 | caffe_gpu_mul(buffer_blob_.count(), x_norm_.gpu_data(), buffer_blob_.gpu_data(), bottom_diff); 133 | // EX across spatial 134 | caffe_gpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1), const_bottom_diff, 135 | spatial_sum_multiplier_.gpu_data(), Dtype(0), spatial_mean_.mutable_gpu_data()); 136 | // EX across batch 137 | caffe_gpu_gemv(CblasTrans, N_, C_, Dtype(1), spatial_mean_.gpu_data(), 138 | batch_sum_multiplier_.gpu_data(), Dtype(0), batch_mean_.mutable_gpu_data()); 139 | 140 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 141 | batch_sum_multiplier_.gpu_data(), batch_mean_.gpu_data(), Dtype(0), 142 | spatial_mean_.mutable_gpu_data()); 143 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 144 | spatial_mean_.gpu_data(), spatial_sum_multiplier_.gpu_data(), Dtype(0), 145 | bottom_diff); 146 | 147 | caffe_gpu_mul(buffer_blob_.count(), x_norm_.gpu_data(), const_bottom_diff, bottom_diff); 148 | 149 | // EX across spatial 150 | caffe_gpu_gemv(CblasNoTrans, N_ * C_, H_ * W_, Dtype(1), buffer_blob_.gpu_data(), 151 | spatial_sum_multiplier_.gpu_data(), Dtype(0), spatial_mean_.mutable_gpu_data()); 152 | // EX across batch 153 | caffe_gpu_gemv(CblasTrans, N_, C_, Dtype(1), spatial_mean_.gpu_data(), 154 | batch_sum_multiplier_.gpu_data(), Dtype(0), batch_mean_.mutable_gpu_data()); 155 | 156 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 157 | batch_sum_multiplier_.gpu_data(), batch_mean_.gpu_data(), Dtype(0), 158 | spatial_mean_.mutable_gpu_data()); 159 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 160 | spatial_mean_.gpu_data(), spatial_sum_multiplier_.gpu_data(), Dtype(1), 161 | bottom_diff); 162 | 163 | caffe_gpu_axpby(buffer_blob_.count(), Dtype(1), buffer_blob_.gpu_data(), Dtype(-1. / (N_ * H_ * W_)), 164 | bottom_diff); 165 | 166 | // variance normalization 167 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_, C_, 1, Dtype(1), 168 | batch_sum_multiplier_.gpu_data(), batch_variance_.gpu_data(), Dtype(0), 169 | spatial_variance_.mutable_gpu_data()); 170 | caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, N_ * C_, H_ * W_, 1, Dtype(1), 171 | spatial_variance_.gpu_data(), spatial_sum_multiplier_.gpu_data(), Dtype(0), 172 | buffer_blob_.mutable_gpu_data()); 173 | 174 | caffe_gpu_div(buffer_blob_.count(), const_bottom_diff, buffer_blob_.gpu_data(), bottom_diff); 175 | 176 | } 177 | 178 | INSTANTIATE_CLASS(BNLayer); 179 | 180 | } // namespace caffe 181 | -------------------------------------------------------------------------------- /caffe.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package caffe; 4 | 5 | message BlobProto { 6 | optional int32 num = 1 [default = 0]; 7 | optional int32 channels = 2 [default = 0]; 8 | optional int32 height = 3 [default = 0]; 9 | optional int32 width = 4 [default = 0]; 10 | repeated float data = 5 [packed = true]; 11 | repeated float diff = 6 [packed = true]; 12 | } 13 | 14 | // The BlobProtoVector is simply a way to pass multiple blobproto instances 15 | // around. 16 | message BlobProtoVector { 17 | repeated BlobProto blobs = 1; 18 | } 19 | 20 | message Datum { 21 | optional int32 channels = 1; 22 | optional int32 height = 2; 23 | optional int32 width = 3; 24 | // the actual image data, in bytes 25 | optional bytes data = 4; 26 | optional int32 label = 5; 27 | // Optionally, the datum could also hold float data. 28 | repeated float float_data = 6; 29 | } 30 | 31 | message FillerParameter { 32 | // The filler type. 33 | optional string type = 1 [default = 'constant']; 34 | optional float value = 2 [default = 0]; // the value in constant filler 35 | optional float min = 3 [default = 0]; // the min value in uniform filler 36 | optional float max = 4 [default = 1]; // the max value in uniform filler 37 | optional float mean = 5 [default = 0]; // the mean value in Gaussian filler 38 | optional float std = 6 [default = 1]; // the std value in Gaussian filler 39 | // The expected number of non-zero input weights for a given output in 40 | // Gaussian filler -- the default -1 means don't perform sparsification. 41 | optional int32 sparse = 7 [default = -1]; 42 | // the negative slope of the leaky relu attached to the previous activation 43 | optional float negative_slope = 8 [default = 0]; 44 | // Normalize the filler variance by fan_in, fan_out, or their average. 45 | // Applies to xavier and relu fillers. 46 | enum VarianceNorm { 47 | FAN_IN = 0; 48 | FAN_OUT = 1; 49 | AVERAGE = 2; 50 | } 51 | optional VarianceNorm variance_norm = 9 [default = FAN_IN]; 52 | } 53 | 54 | message NetParameter { 55 | optional string name = 1; // consider giving the network a name 56 | repeated LayerParameter layers = 2; // a bunch of layers. 57 | // The input blobs to the network. 58 | repeated string input = 3; 59 | // The dim of the input blobs. For each input blob there should be four 60 | // values specifying the num, channels, height and width of the input blob. 61 | // Thus, there should be a total of (4 * #input) numbers. 62 | repeated int32 input_dim = 4; 63 | // Whether the network will force every layer to carry out backward operation. 64 | // If set False, then whether to carry out backward is determined 65 | // automatically according to the net structure and learning rates. 66 | optional bool force_backward = 5 [default = false]; 67 | // The current "state" of the network, including the phase, level, and stage. 68 | // Some layers may be included/excluded depending on this state and the states 69 | // specified in the layers' include and exclude fields. 70 | optional NetState state = 6; 71 | } 72 | 73 | // NOTE 74 | // Update the next available ID when you add a new SolverParameter field. 75 | // 76 | // SolverParameter next available ID: 32 (last added: delta) 77 | message SolverParameter { 78 | ////////////////////////////////////////////////////////////////////////////// 79 | // Specifying the train and test networks 80 | // 81 | // Exactly one train net must be specified using one of the following fields: 82 | // train_net_param, train_net, net_param, net 83 | // One or more test nets may be specified using any of the following fields: 84 | // test_net_param, test_net, net_param, net 85 | // If more than one test net field is specified (e.g., both net and 86 | // test_net are specified), they will be evaluated in the field order given 87 | // above: (1) test_net_param, (2) test_net, (3) net_param/net. 88 | // A test_iter must be specified for each test_net. 89 | // A test_level and/or a test_stage may also be specified for each test_net. 90 | ////////////////////////////////////////////////////////////////////////////// 91 | 92 | // Proto filename for the train net, possibly combined with one or more 93 | // test nets. 94 | optional string net = 24; 95 | // Inline train net param, possibly combined with one or more test nets. 96 | optional NetParameter net_param = 25; 97 | 98 | optional string train_net = 1; // Proto filename for the train net. 99 | repeated string test_net = 2; // Proto filenames for the test nets. 100 | optional NetParameter train_net_param = 21; // Inline train net params. 101 | repeated NetParameter test_net_param = 22; // Inline test net params. 102 | 103 | // The states for the train/test nets. Must be unspecified or 104 | // specified once per net. 105 | // 106 | // By default, all states will have solver = true; 107 | // train_state will have phase = TRAIN, 108 | // and all test_state's will have phase = TEST. 109 | // Other defaults are set according to the NetState defaults. 110 | optional NetState train_state = 26; 111 | repeated NetState test_state = 27; 112 | 113 | // The number of iterations for each test net. 114 | repeated int32 test_iter = 3; 115 | 116 | // The number of iterations between two testing phases. 117 | optional int32 test_interval = 4 [default = 0]; 118 | optional bool test_compute_loss = 19 [default = false]; 119 | // If true, run an initial test pass before the first iteration, 120 | // ensuring memory availability and printing the starting value of the loss. 121 | optional bool test_initialization = 32 [default = true]; 122 | optional float base_lr = 5; // The base learning rate 123 | // the number of iterations between displaying info. If display = 0, no info 124 | // will be displayed. 125 | optional int32 display = 6; 126 | // Display the cost averaged over the last average_cost iterations 127 | optional int32 average_loss = 34 [default = 1]; 128 | optional int32 max_iter = 7; // the maximum number of iterations 129 | optional string lr_policy = 8; // The learning rate decay policy. 130 | optional float gamma = 9; // The parameter to compute the learning rate. 131 | optional float power = 10; // The parameter to compute the learning rate. 132 | optional float momentum = 11; // The momentum value. 133 | optional float weight_decay = 12; // The weight decay. 134 | // regularization types supported: L1 and L2 135 | // controled by weight_decay 136 | optional string regularization_type = 29 [default = "L2"]; 137 | optional int32 stepsize = 13; // the stepsize for learning rate policy "step" 138 | optional int32 snapshot = 14 [default = 0]; // The snapshot interval 139 | optional string snapshot_prefix = 15; // The prefix for the snapshot. 140 | // whether to snapshot diff in the results or not. Snapshotting diff will help 141 | // debugging but the final protocol buffer size will be much larger. 142 | optional bool snapshot_diff = 16 [default = false]; 143 | // the mode solver will use: 0 for CPU and 1 for GPU. Use GPU in default. 144 | enum SolverMode { 145 | CPU = 0; 146 | GPU = 1; 147 | } 148 | optional SolverMode solver_mode = 17 [default = GPU]; 149 | // the device_id will that be used in GPU mode. Use device_id = 0 in default. 150 | optional int32 device_id = 18 [default = 0]; 151 | // If non-negative, the seed with which the Solver will initialize the Caffe 152 | // random number generator -- useful for reproducible results. Otherwise, 153 | // (and by default) initialize using a seed derived from the system clock. 154 | optional int64 random_seed = 20 [default = -1]; 155 | // added to allow big batch_size 156 | optional int32 update_interval = 33 [default = 1]; 157 | 158 | // Solver type 159 | enum SolverType { 160 | SGD = 0; 161 | NESTEROV = 1; 162 | ADAGRAD = 2; 163 | ADADELTA = 3; 164 | } 165 | optional SolverType solver_type = 30 [default = SGD]; 166 | // numerical stability for AdaGrad 167 | optional float delta = 31 [default = 1e-8]; 168 | 169 | // If true, print information about the state of the net that may help with 170 | // debugging learning problems. 171 | optional bool debug_info = 23 [default = false]; 172 | 173 | // If false, don't save a snapshot after training finishes. 174 | optional bool snapshot_after_train = 28 [default = true]; 175 | } 176 | 177 | // A message that stores the solver snapshots 178 | message SolverState { 179 | optional int32 iter = 1; // The current iteration 180 | optional string learned_net = 2; // The file that stores the learned net. 181 | repeated BlobProto history = 3; // The history for sgd solvers 182 | } 183 | 184 | enum Phase { 185 | TRAIN = 0; 186 | TEST = 1; 187 | } 188 | 189 | message NetState { 190 | optional Phase phase = 1 [default = TEST]; 191 | optional int32 level = 2 [default = 0]; 192 | repeated string stage = 3; 193 | } 194 | 195 | message NetStateRule { 196 | // Set phase to require the NetState have a particular phase (TRAIN or TEST) 197 | // to meet this rule. 198 | optional Phase phase = 1; 199 | 200 | // Set the minimum and/or maximum levels in which the layer should be used. 201 | // Leave undefined to meet the rule regardless of level. 202 | optional int32 min_level = 2; 203 | optional int32 max_level = 3; 204 | 205 | // Customizable sets of stages to include or exclude. 206 | // The net must have ALL of the specified stages and NONE of the specified 207 | // "not_stage"s to meet the rule. 208 | // (Use multiple NetStateRules to specify conjunctions of stages.) 209 | repeated string stage = 4; 210 | repeated string not_stage = 5; 211 | } 212 | 213 | // NOTE 214 | // Update the next available ID when you add a new LayerParameter field. 215 | // 216 | // LayerParameter next available ID: 40 (last added: softmax_param) 217 | message LayerParameter { 218 | repeated string bottom = 2; // the name of the bottom blobs 219 | repeated string top = 3; // the name of the top blobs 220 | optional string name = 4; // the layer name 221 | 222 | // Rules controlling whether and when a layer is included in the network, 223 | // based on the current NetState. You may specify a non-zero number of rules 224 | // to include OR exclude, but not both. If no include or exclude rules are 225 | // specified, the layer is always included. If the current NetState meets 226 | // ANY (i.e., one or more) of the specified rules, the layer is 227 | // included/excluded. 228 | repeated NetStateRule include = 32; 229 | repeated NetStateRule exclude = 33; 230 | 231 | // NOTE 232 | // Add new LayerTypes to the enum below in lexicographical order (other than 233 | // starting with NONE), starting with the next available ID in the comment 234 | // line above the enum. Update the next available ID when you add a new 235 | // LayerType. 236 | // 237 | // LayerType next available ID: 40 (last added: BN) 238 | enum LayerType { 239 | // "NONE" layer type is 0th enum element so that we don't cause confusion 240 | // by defaulting to an existent LayerType (instead, should usually error if 241 | // the type is unspecified). 242 | NONE = 0; 243 | ABSVAL = 35; 244 | ACCURACY = 1; 245 | ARGMAX = 30; 246 | BN = 39; 247 | BNLL = 2; 248 | CONCAT = 3; 249 | COMPACT_DATA = 37; 250 | CONVOLUTION = 4; 251 | DATA = 5; 252 | DROPOUT = 6; 253 | DUMMY_DATA = 32; 254 | EUCLIDEAN_LOSS = 7; 255 | ELTWISE = 25; 256 | FLATTEN = 8; 257 | HDF5_DATA = 9; 258 | HDF5_OUTPUT = 10; 259 | HINGE_LOSS = 28; 260 | IM2COL = 11; 261 | IMAGE_DATA = 12; 262 | INFOGAIN_LOSS = 13; 263 | INNER_PRODUCT = 14; 264 | LRN = 15; 265 | MEMORY_DATA = 29; 266 | MULTINOMIAL_LOGISTIC_LOSS = 16; 267 | MVN = 34; 268 | POOLING = 17; 269 | POWER = 26; 270 | PRELU = 38; 271 | RELU = 18; 272 | SIGMOID = 19; 273 | SIGMOID_CROSS_ENTROPY_LOSS = 27; 274 | SILENCE = 36; 275 | SOFTMAX = 20; 276 | SOFTMAX_LOSS = 21; 277 | SPLIT = 22; 278 | SLICE = 33; 279 | TANH = 23; 280 | WINDOW_DATA = 24; 281 | THRESHOLD = 31; 282 | } 283 | optional LayerType type = 5; // the layer type from the enum above 284 | 285 | // The blobs containing the numeric parameters of the layer 286 | repeated BlobProto blobs = 6; 287 | // The names of the parameter blobs -- useful for sharing parameters among 288 | // layers (but never required). 289 | repeated string param = 1001; 290 | // Whether to require shared weights to have the same shape, or just the same 291 | // count -- defaults to STRICT if unspecified. 292 | repeated DimCheckMode blob_share_mode = 1002; 293 | enum DimCheckMode { 294 | // STRICT (default) requires that num, channels, height, width each match. 295 | STRICT = 0; 296 | // PERMISSIVE requires only the count (num*channels*height*width) to match. 297 | PERMISSIVE = 1; 298 | } 299 | // The ratio that is multiplied on the global learning rate. If you want to 300 | // set the learning ratio for one blob, you need to set it for all blobs. 301 | repeated float blobs_lr = 7; 302 | // The weight decay that is multiplied on the global weight decay. 303 | repeated float weight_decay = 8; 304 | 305 | // The amount of weight to assign each top blob in the objective. 306 | // Each layer assigns a default value, usually of either 0 or 1, 307 | // to each top blob. 308 | repeated float loss_weight = 35; 309 | 310 | optional AccuracyParameter accuracy_param = 27; 311 | optional ArgMaxParameter argmax_param = 23; 312 | optional BNParameter bn_param = 41; 313 | optional ConcatParameter concat_param = 9; 314 | optional ConvolutionParameter convolution_param = 10; 315 | optional DataParameter data_param = 11; 316 | optional DropoutParameter dropout_param = 12; 317 | optional DummyDataParameter dummy_data_param = 26; 318 | optional EltwiseParameter eltwise_param = 24; 319 | optional HDF5DataParameter hdf5_data_param = 13; 320 | optional HDF5OutputParameter hdf5_output_param = 14; 321 | optional HingeLossParameter hinge_loss_param = 29; 322 | optional ImageDataParameter image_data_param = 15; 323 | optional InfogainLossParameter infogain_loss_param = 16; 324 | optional InnerProductParameter inner_product_param = 17; 325 | optional LRNParameter lrn_param = 18; 326 | optional MemoryDataParameter memory_data_param = 22; 327 | optional MVNParameter mvn_param = 34; 328 | optional PoolingParameter pooling_param = 19; 329 | optional PowerParameter power_param = 21; 330 | optional PReLUParameter prelu_param = 40; 331 | optional ReLUParameter relu_param = 30; 332 | optional SigmoidParameter sigmoid_param = 38; 333 | optional SoftmaxParameter softmax_param = 39; 334 | optional SliceParameter slice_param = 31; 335 | optional TanHParameter tanh_param = 37; 336 | optional ThresholdParameter threshold_param = 25; 337 | optional WindowDataParameter window_data_param = 20; 338 | 339 | // Parameters for data pre-processing. 340 | optional TransformationParameter transform_param = 36; 341 | 342 | // Note: certain layers may have more than one computational engine 343 | // for their implementation. These layers include an Engine type and 344 | // engine parameter for selecting the implementation. 345 | // The default for the engine is set by the ENGINE switch at compile-time. 346 | 347 | // DEPRECATED: The layer parameters specified as a V0LayerParameter. 348 | // This should never be used by any code except to upgrade to the new 349 | // LayerParameter specification. 350 | optional V0LayerParameter layer = 1; 351 | } 352 | 353 | // Message that stores parameters used to apply transformation 354 | // to the data layer's data 355 | message TransformationParameter { 356 | // For data pre-processing, we can do simple scaling and subtracting the 357 | // data mean, if provided. Note that the mean subtraction is always carried 358 | // out before scaling. 359 | optional float scale = 1 [default = 1]; 360 | // Specify if we want to randomly mirror data. 361 | optional bool mirror = 2 [default = false]; 362 | // Specify if we would like to randomly crop an image. 363 | optional uint32 crop_size = 3 [default = 0]; 364 | optional string mean_file = 4; 365 | optional bool multiscale = 5 [default = false]; 366 | // Specify the range of scaling factor for doing resizing 367 | optional float min_scaling_factor = 6 [default = 0.75]; 368 | optional float max_scaling_factor = 7 [default = 1.50]; 369 | // Specify the angle interval for doing rotation 370 | optional float angle_interval = 8 [default = 1]; 371 | // Specify the shearing ratio 372 | optional float min_shear_bxy = 9 [default = -0.125]; 373 | optional float max_shear_bxy = 10 [default = 0.125]; 374 | // Specify the perspective ratio 375 | optional float max_perspective_ratio = 11 [default = 0.125]; 376 | // Specify the constant value we would like to fill with when doing affine transformation 377 | optional float warp_fillval = 12 [default = 255]; 378 | optional bool debug_display = 13 [default = false]; 379 | optional bool contrast_adjustment = 14 [default = false]; 380 | optional bool smooth_filtering = 15 [default = false]; 381 | optional bool jpeg_compression = 16 [default = false]; 382 | } 383 | 384 | // Message that stores parameters used by AccuracyLayer 385 | message AccuracyParameter { 386 | // When computing accuracy, count as correct by comparing the true label to 387 | // the top k scoring classes. By default, only compare to the top scoring 388 | // class (i.e. argmax). 389 | optional uint32 top_k = 1 [default = 1]; 390 | } 391 | 392 | // Message that stores parameters used by ArgMaxLayer 393 | message ArgMaxParameter { 394 | // If true produce pairs (argmax, maxval) 395 | optional bool out_max_val = 1 [default = false]; 396 | optional uint32 top_k = 2 [default = 1]; 397 | } 398 | 399 | // Message that stores parameters used by ConcatLayer 400 | message ConcatParameter { 401 | // Concat Layer needs to specify the dimension along the concat will happen, 402 | // the other dimensions must be the same for all the bottom blobs 403 | // By default it will concatenate blobs along channels dimension 404 | optional uint32 concat_dim = 1 [default = 1]; 405 | } 406 | 407 | // Message that stores parameters used by ConvolutionLayer 408 | message ConvolutionParameter { 409 | optional uint32 num_output = 1; // The number of outputs for the layer 410 | optional bool bias_term = 2 [default = true]; // whether to have bias terms 411 | // Pad, kernel size, and stride are all given as a single value for equal 412 | // dimensions in height and width or as Y, X pairs. 413 | optional uint32 pad = 3 [default = 0]; // The padding size (equal in Y, X) 414 | optional uint32 pad_h = 9 [default = 0]; // The padding height 415 | optional uint32 pad_w = 10 [default = 0]; // The padding width 416 | optional uint32 kernel_size = 4; // The kernel size (square) 417 | optional uint32 kernel_h = 11; // The kernel height 418 | optional uint32 kernel_w = 12; // The kernel width 419 | optional uint32 group = 5 [default = 1]; // The group size for group conv 420 | optional uint32 stride = 6 [default = 1]; // The stride (equal in Y, X) 421 | optional uint32 stride_h = 13; // The stride height 422 | optional uint32 stride_w = 14; // The stride width 423 | optional FillerParameter weight_filler = 7; // The filler for the weight 424 | optional FillerParameter bias_filler = 8; // The filler for the bias 425 | enum Engine { 426 | DEFAULT = 0; 427 | CAFFE = 1; 428 | CUDNN = 2; 429 | } 430 | optional Engine engine = 15 [default = DEFAULT]; 431 | } 432 | 433 | // Message that stores parameters used by DataLayer 434 | message DataParameter { 435 | enum DB { 436 | LEVELDB = 0; 437 | LMDB = 1; 438 | } 439 | // Specify the data source. 440 | optional string source = 1; 441 | // Specify the batch size. 442 | optional uint32 batch_size = 4; 443 | // The rand_skip variable is for the data layer to skip a few data points 444 | // to avoid all asynchronous sgd clients to start at the same point. The skip 445 | // point would be set as rand_skip * rand(0,1). Note that rand_skip should not 446 | // be larger than the number of keys in the leveldb. 447 | optional uint32 rand_skip = 7 [default = 0]; 448 | optional DB backend = 8 [default = LEVELDB]; 449 | // DEPRECATED. See TransformationParameter. For data pre-processing, we can do 450 | // simple scaling and subtracting the data mean, if provided. Note that the 451 | // mean subtraction is always carried out before scaling. 452 | optional float scale = 2 [default = 1]; 453 | optional string mean_file = 3; 454 | // DEPRECATED. See TransformationParameter. Specify if we would like to randomly 455 | // crop an image. 456 | optional uint32 crop_size = 5 [default = 0]; 457 | // DEPRECATED. See TransformationParameter. Specify if we want to randomly mirror 458 | // data. 459 | optional bool mirror = 6 [default = false]; 460 | } 461 | 462 | // Message that stores parameters used by DropoutLayer 463 | message DropoutParameter { 464 | optional float dropout_ratio = 1 [default = 0.5]; // dropout ratio 465 | } 466 | 467 | // Message that stores parameters used by DummyDataLayer. 468 | // DummyDataLayer fills any number of arbitrarily shaped blobs with random 469 | // (or constant) data generated by "Fillers" (see "message FillerParameter"). 470 | message DummyDataParameter { 471 | // This layer produces N >= 1 top blobs. DummyDataParameter must specify 1 or N 472 | // num, N channels, N height, and N width fields, and must specify 0, 1 or N 473 | // data_fillers. 474 | // 475 | // If 0 data_fillers are specified, ConstantFiller with a value of 0 is used. 476 | // If 1 data_filler is specified, it is applied to all top blobs. If N are 477 | // specified, the ith is applied to the ith top blob. 478 | repeated FillerParameter data_filler = 1; 479 | repeated uint32 num = 2; 480 | repeated uint32 channels = 3; 481 | repeated uint32 height = 4; 482 | repeated uint32 width = 5; 483 | } 484 | 485 | // Message that stores parameters used by EltwiseLayer 486 | message EltwiseParameter { 487 | enum EltwiseOp { 488 | PROD = 0; 489 | SUM = 1; 490 | MAX = 2; 491 | } 492 | optional EltwiseOp operation = 1 [default = SUM]; // element-wise operation 493 | repeated float coeff = 2; // blob-wise coefficient for SUM operation 494 | 495 | // Whether to use an asymptotically slower (for >2 inputs) but stabler method 496 | // of computing the gradient for the PROD operation. (No effect for SUM op.) 497 | optional bool stable_prod_grad = 3 [default = true]; 498 | } 499 | 500 | // Message that stores parameters used by ThresholdLayer 501 | message ThresholdParameter { 502 | optional float threshold = 1 [default = 0]; // Strictly Positive values 503 | } 504 | 505 | // Message that stores parameters used by HDF5DataLayer 506 | message HDF5DataParameter { 507 | // Specify the data source. 508 | optional string source = 1; 509 | // Specify the batch size. 510 | optional uint32 batch_size = 2; 511 | } 512 | 513 | // Message that stores parameters used by HDF5OutputLayer 514 | message HDF5OutputParameter { 515 | optional string file_name = 1; 516 | } 517 | 518 | message HingeLossParameter { 519 | enum Norm { 520 | L1 = 1; 521 | L2 = 2; 522 | } 523 | // Specify the Norm to use L1 or L2 524 | optional Norm norm = 1 [default = L1]; 525 | } 526 | 527 | // Message that stores parameters used by ImageDataLayer 528 | message ImageDataParameter { 529 | // Specify the data source. 530 | optional string source = 1; 531 | // Specify the batch size. 532 | optional uint32 batch_size = 4; 533 | // The rand_skip variable is for the data layer to skip a few data points 534 | // to avoid all asynchronous sgd clients to start at the same point. The skip 535 | // point would be set as rand_skip * rand(0,1). Note that rand_skip should not 536 | // be larger than the number of keys in the leveldb. 537 | optional uint32 rand_skip = 7 [default = 0]; 538 | // Whether or not ImageLayer should shuffle the list of files at every epoch. 539 | optional bool shuffle = 8 [default = false]; 540 | // It will also resize images if new_height or new_width are not zero. 541 | optional uint32 new_height = 9 [default = 0]; 542 | optional uint32 new_width = 10 [default = 0]; 543 | // DEPRECATED. See TransformationParameter. For data pre-processing, we can do 544 | // simple scaling and subtracting the data mean, if provided. Note that the 545 | // mean subtraction is always carried out before scaling. 546 | optional float scale = 2 [default = 1]; 547 | optional string mean_file = 3; 548 | // DEPRECATED. See TransformationParameter. Specify if we would like to randomly 549 | // crop an image. 550 | optional uint32 crop_size = 5 [default = 0]; 551 | // DEPRECATED. See TransformationParameter. Specify if we want to randomly mirror 552 | // data. 553 | optional bool mirror = 6 [default = false]; 554 | } 555 | 556 | // Message that stores parameters InfogainLossLayer 557 | message InfogainLossParameter { 558 | // Specify the infogain matrix source. 559 | optional string source = 1; 560 | } 561 | 562 | // Message that stores parameters used by InnerProductLayer 563 | message InnerProductParameter { 564 | optional uint32 num_output = 1; // The number of outputs for the layer 565 | optional bool bias_term = 2 [default = true]; // whether to have bias terms 566 | optional FillerParameter weight_filler = 3; // The filler for the weight 567 | optional FillerParameter bias_filler = 4; // The filler for the bias 568 | } 569 | 570 | // Message that stores parameters used by LRNLayer 571 | message LRNParameter { 572 | optional uint32 local_size = 1 [default = 5]; 573 | optional float alpha = 2 [default = 1.]; 574 | optional float beta = 3 [default = 0.75]; 575 | enum NormRegion { 576 | ACROSS_CHANNELS = 0; 577 | WITHIN_CHANNEL = 1; 578 | } 579 | optional NormRegion norm_region = 4 [default = ACROSS_CHANNELS]; 580 | } 581 | 582 | // Message that stores parameters used by MemoryDataLayer 583 | message MemoryDataParameter { 584 | optional uint32 batch_size = 1; 585 | optional uint32 channels = 2; 586 | optional uint32 height = 3; 587 | optional uint32 width = 4; 588 | } 589 | 590 | // Message that stores parameters used by MVNLayer 591 | message MVNParameter { 592 | // This parameter can be set to false to normalize mean only 593 | optional bool normalize_variance = 1 [default = true]; 594 | 595 | // This parameter can be set to true to perform DNN-like MVN 596 | optional bool across_channels = 2 [default = false]; 597 | } 598 | 599 | // Message that stores parameters used by BNLayer 600 | message BNParameter { 601 | optional FillerParameter scale_filler = 1; // The filler for the scale 602 | optional FillerParameter shift_filler = 2; // The filler for the shift 603 | } 604 | 605 | // Message that stores parameters used by PoolingLayer 606 | message PoolingParameter { 607 | enum PoolMethod { 608 | MAX = 0; 609 | AVE = 1; 610 | STOCHASTIC = 2; 611 | } 612 | optional PoolMethod pool = 1 [default = MAX]; // The pooling method 613 | // Pad, kernel size, and stride are all given as a single value for equal 614 | // dimensions in height and width or as Y, X pairs. 615 | optional uint32 pad = 4 [default = 0]; // The padding size (equal in Y, X) 616 | optional uint32 pad_h = 9 [default = 0]; // The padding height 617 | optional uint32 pad_w = 10 [default = 0]; // The padding width 618 | optional uint32 kernel_size = 2; // The kernel size (square) 619 | optional uint32 kernel_h = 5; // The kernel height 620 | optional uint32 kernel_w = 6; // The kernel width 621 | optional uint32 stride = 3 [default = 1]; // The stride (equal in Y, X) 622 | optional uint32 stride_h = 7; // The stride height 623 | optional uint32 stride_w = 8; // The stride width 624 | enum Engine { 625 | DEFAULT = 0; 626 | CAFFE = 1; 627 | CUDNN = 2; 628 | } 629 | optional Engine engine = 11 [default = DEFAULT]; 630 | } 631 | 632 | // Message that stores parameters used by PowerLayer 633 | message PowerParameter { 634 | // PowerLayer computes outputs y = (shift + scale * x) ^ power. 635 | optional float power = 1 [default = 1.0]; 636 | optional float scale = 2 [default = 1.0]; 637 | optional float shift = 3 [default = 0.0]; 638 | } 639 | 640 | // Message that stores parameters used by PReLULayer 641 | message PReLUParameter { 642 | // Parametric ReLU described in K. He et al, Delving Deep into Rectifiers: 643 | // Surpassing Human-Level Performance on ImageNet Classification, 2015. 644 | 645 | // Initial value of a_i. Set as a_i=init_value for all channel i. 646 | optional float init_value = 1 [default = 0.25]; 647 | // Whether or not slope paramters are shared across channels. 648 | optional bool channel_shared = 2 [default = false]; 649 | } 650 | 651 | // Message that stores parameters used by ReLULayer 652 | message ReLUParameter { 653 | // Allow non-zero slope for negative inputs to speed up optimization 654 | // Described in: 655 | // Maas, A. L., Hannun, A. Y., & Ng, A. Y. (2013). Rectifier nonlinearities 656 | // improve neural network acoustic models. In ICML Workshop on Deep Learning 657 | // for Audio, Speech, and Language Processing. 658 | optional float negative_slope = 1 [default = 0]; 659 | enum Engine { 660 | DEFAULT = 0; 661 | CAFFE = 1; 662 | CUDNN = 2; 663 | } 664 | optional Engine engine = 2 [default = DEFAULT]; 665 | } 666 | 667 | // Message that stores parameters used by SigmoidLayer 668 | message SigmoidParameter { 669 | enum Engine { 670 | DEFAULT = 0; 671 | CAFFE = 1; 672 | CUDNN = 2; 673 | } 674 | optional Engine engine = 1 [default = DEFAULT]; 675 | } 676 | 677 | // Message that stores parameters used by SliceLayer 678 | message SliceParameter { 679 | // SliceLayer needs to know which dimension to slice across. 680 | // Currently, SliceLayer only supports slicing across num (dim 0) 681 | // and channels (dim 1). 682 | // By default, SliceLayer slices across channels. 683 | optional uint32 slice_dim = 1 [default = 1]; 684 | repeated uint32 slice_point = 2; 685 | } 686 | 687 | // Message that stores parameters used by SoftmaxLayer, SoftMaxWithLossLayer 688 | message SoftmaxParameter { 689 | enum Engine { 690 | DEFAULT = 0; 691 | CAFFE = 1; 692 | CUDNN = 2; 693 | } 694 | optional Engine engine = 1 [default = DEFAULT]; 695 | } 696 | 697 | // Message that stores parameters used by SigmoidLayer 698 | message TanHParameter { 699 | enum Engine { 700 | DEFAULT = 0; 701 | CAFFE = 1; 702 | CUDNN = 2; 703 | } 704 | optional Engine engine = 1 [default = DEFAULT]; 705 | } 706 | 707 | // Message that stores parameters used by WindowDataLayer 708 | message WindowDataParameter { 709 | // Specify the data source. 710 | optional string source = 1; 711 | // For data pre-processing, we can do simple scaling and subtracting the 712 | // data mean, if provided. Note that the mean subtraction is always carried 713 | // out before scaling. 714 | optional float scale = 2 [default = 1]; 715 | optional string mean_file = 3; 716 | // Specify the batch size. 717 | optional uint32 batch_size = 4; 718 | // Specify if we would like to randomly crop an image. 719 | optional uint32 crop_size = 5 [default = 0]; 720 | // Specify if we want to randomly mirror data. 721 | optional bool mirror = 6 [default = false]; 722 | // Foreground (object) overlap threshold 723 | optional float fg_threshold = 7 [default = 0.5]; 724 | // Background (non-object) overlap threshold 725 | optional float bg_threshold = 8 [default = 0.5]; 726 | // Fraction of batch that should be foreground objects 727 | optional float fg_fraction = 9 [default = 0.25]; 728 | // Amount of contextual padding to add around a window 729 | // (used only by the window_data_layer) 730 | optional uint32 context_pad = 10 [default = 0]; 731 | // Mode for cropping out a detection window 732 | // warp: cropped window is warped to a fixed size and aspect ratio 733 | // square: the tightest square around the window is cropped 734 | optional string crop_mode = 11 [default = "warp"]; 735 | } 736 | 737 | // DEPRECATED: V0LayerParameter is the old way of specifying layer parameters 738 | // in Caffe. We keep this message type around for legacy support. 739 | message V0LayerParameter { 740 | optional string name = 1; // the layer name 741 | optional string type = 2; // the string to specify the layer type 742 | 743 | // Parameters to specify layers with inner products. 744 | optional uint32 num_output = 3; // The number of outputs for the layer 745 | optional bool biasterm = 4 [default = true]; // whether to have bias terms 746 | optional FillerParameter weight_filler = 5; // The filler for the weight 747 | optional FillerParameter bias_filler = 6; // The filler for the bias 748 | 749 | optional uint32 pad = 7 [default = 0]; // The padding size 750 | optional uint32 kernelsize = 8; // The kernel size 751 | optional uint32 group = 9 [default = 1]; // The group size for group conv 752 | optional uint32 stride = 10 [default = 1]; // The stride 753 | enum PoolMethod { 754 | MAX = 0; 755 | AVE = 1; 756 | STOCHASTIC = 2; 757 | } 758 | optional PoolMethod pool = 11 [default = MAX]; // The pooling method 759 | optional float dropout_ratio = 12 [default = 0.5]; // dropout ratio 760 | 761 | optional uint32 local_size = 13 [default = 5]; // for local response norm 762 | optional float alpha = 14 [default = 1.]; // for local response norm 763 | optional float beta = 15 [default = 0.75]; // for local response norm 764 | 765 | // For data layers, specify the data source 766 | optional string source = 16; 767 | // For data pre-processing, we can do simple scaling and subtracting the 768 | // data mean, if provided. Note that the mean subtraction is always carried 769 | // out before scaling. 770 | optional float scale = 17 [default = 1]; 771 | optional string meanfile = 18; 772 | // For data layers, specify the batch size. 773 | optional uint32 batchsize = 19; 774 | // For data layers, specify if we would like to randomly crop an image. 775 | optional uint32 cropsize = 20 [default = 0]; 776 | // For data layers, specify if we want to randomly mirror data. 777 | optional bool mirror = 21 [default = false]; 778 | 779 | // The blobs containing the numeric parameters of the layer 780 | repeated BlobProto blobs = 50; 781 | // The ratio that is multiplied on the global learning rate. If you want to 782 | // set the learning ratio for one blob, you need to set it for all blobs. 783 | repeated float blobs_lr = 51; 784 | // The weight decay that is multiplied on the global weight decay. 785 | repeated float weight_decay = 52; 786 | 787 | // The rand_skip variable is for the data layer to skip a few data points 788 | // to avoid all asynchronous sgd clients to start at the same point. The skip 789 | // point would be set as rand_skip * rand(0,1). Note that rand_skip should not 790 | // be larger than the number of keys in the leveldb. 791 | optional uint32 rand_skip = 53 [default = 0]; 792 | 793 | // Fields related to detection (det_*) 794 | // foreground (object) overlap threshold 795 | optional float det_fg_threshold = 54 [default = 0.5]; 796 | // background (non-object) overlap threshold 797 | optional float det_bg_threshold = 55 [default = 0.5]; 798 | // Fraction of batch that should be foreground objects 799 | optional float det_fg_fraction = 56 [default = 0.25]; 800 | 801 | // optional bool OBSOLETE_can_clobber = 57 [default = true]; 802 | 803 | // Amount of contextual padding to add around a window 804 | // (used only by the window_data_layer) 805 | optional uint32 det_context_pad = 58 [default = 0]; 806 | 807 | // Mode for cropping out a detection window 808 | // warp: cropped window is warped to a fixed size and aspect ratio 809 | // square: the tightest square around the window is cropped 810 | optional string det_crop_mode = 59 [default = "warp"]; 811 | 812 | // For ReshapeLayer, one needs to specify the new dimensions. 813 | optional int32 new_num = 60 [default = 0]; 814 | optional int32 new_channels = 61 [default = 0]; 815 | optional int32 new_height = 62 [default = 0]; 816 | optional int32 new_width = 63 [default = 0]; 817 | 818 | // Whether or not ImageLayer should shuffle the list of files at every epoch. 819 | // It will also resize images if new_height or new_width are not zero. 820 | optional bool shuffle_images = 64 [default = false]; 821 | 822 | // For ConcatLayer, one needs to specify the dimension for concatenation, and 823 | // the other dimensions must be the same for all the bottom blobs. 824 | // By default it will concatenate blobs along the channels dimension. 825 | optional uint32 concat_dim = 65 [default = 1]; 826 | 827 | optional HDF5OutputParameter hdf5_output_param = 1001; 828 | } 829 | -------------------------------------------------------------------------------- /common_layers.hpp: -------------------------------------------------------------------------------- 1 | #ifndef CAFFE_COMMON_LAYERS_HPP_ 2 | #define CAFFE_COMMON_LAYERS_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "caffe/blob.hpp" 9 | #include "caffe/common.hpp" 10 | #include "caffe/data_layers.hpp" 11 | #include "caffe/layer.hpp" 12 | #include "caffe/loss_layers.hpp" 13 | #include "caffe/neuron_layers.hpp" 14 | #include "caffe/proto/caffe.pb.h" 15 | 16 | namespace caffe { 17 | 18 | /** 19 | * @brief Compute the index of the @f$ K @f$ max values for each datum across 20 | * all dimensions @f$ (C \times H \times W) @f$. 21 | * 22 | * Intended for use after a classification layer to produce a prediction. 23 | * If parameter out_max_val is set to true, output is a vector of pairs 24 | * (max_ind, max_val) for each image. 25 | * 26 | * NOTE: does not implement Backwards operation. 27 | */ 28 | template 29 | class ArgMaxLayer : public Layer { 30 | public: 31 | /** 32 | * @param param provides ArgMaxParameter argmax_param, 33 | * with ArgMaxLayer options: 34 | * - top_k (\b optional uint, default 1). 35 | * the number @f$ K @f$ of maximal items to output. 36 | * - out_max_val (\b optional bool, default false). 37 | * if set, output a vector of pairs (max_ind, max_val) for each image. 38 | */ 39 | explicit ArgMaxLayer(const LayerParameter& param) 40 | : Layer(param) {} 41 | virtual void LayerSetUp(const vector*>& bottom, 42 | vector*>* top); 43 | 44 | virtual inline LayerParameter_LayerType type() const { 45 | return LayerParameter_LayerType_ARGMAX; 46 | } 47 | virtual inline int ExactNumBottomBlobs() const { return 1; } 48 | virtual inline int ExactNumTopBlobs() const { return 1; } 49 | 50 | protected: 51 | /** 52 | * @param bottom input Blob vector (length 1) 53 | * -# @f$ (N \times C \times H \times W) @f$ 54 | * the inputs @f$ x @f$ 55 | * @param top output Blob vector (length 1) 56 | * -# @f$ (N \times 1 \times K \times 1) @f$ or, if out_max_val 57 | * @f$ (N \times 2 \times K \times 1) @f$ 58 | * the computed outputs @f$ 59 | * y_n = \arg\max\limits_i x_{ni} 60 | * @f$ (for @f$ K = 1 @f$). 61 | */ 62 | virtual void Forward_cpu(const vector*>& bottom, 63 | vector*>* top); 64 | /// @brief Not implemented (non-differentiable function) 65 | virtual void Backward_cpu(const vector*>& top, 66 | const vector& propagate_down, vector*>* bottom) { 67 | NOT_IMPLEMENTED; 68 | } 69 | bool out_max_val_; 70 | size_t top_k_; 71 | }; 72 | 73 | /** 74 | * @brief Batch Normalization per-channel with scale & shift linear transform. 75 | * 76 | */ 77 | template 78 | class BNLayer : public Layer { 79 | public: 80 | explicit BNLayer(const LayerParameter& param) 81 | : Layer(param) {} 82 | virtual void LayerSetUp(const vector*>& bottom, 83 | vector*>* top); 84 | 85 | virtual inline LayerParameter_LayerType type() const { 86 | return LayerParameter_LayerType_BN; 87 | } 88 | virtual inline int ExactNumBottomBlobs() const { return 1; } 89 | virtual inline int ExactNumTopBlobs() const { return 1; } 90 | 91 | protected: 92 | virtual void Forward_cpu(const vector*>& bottom, 93 | vector*>* top); 94 | virtual void Forward_gpu(const vector*>& bottom, 95 | vector*>* top); 96 | virtual void Backward_cpu(const vector*>& top, 97 | const vector& propagate_down, vector*>* bottom); 98 | virtual void Backward_gpu(const vector*>& top, 99 | const vector& propagate_down, vector*>* bottom); 100 | 101 | // spatial mean & variance 102 | Blob spatial_mean_, spatial_variance_; 103 | // batch mean & variance 104 | Blob batch_mean_, batch_variance_; 105 | // buffer blob 106 | Blob buffer_blob_; 107 | // x_norm 108 | Blob x_norm_; 109 | 110 | // x_sum_multiplier is used to carry out sum using BLAS 111 | Blob spatial_sum_multiplier_, batch_sum_multiplier_; 112 | 113 | // dimension 114 | int N_; 115 | int C_; 116 | int H_; 117 | int W_; 118 | // eps 119 | Dtype var_eps_; 120 | 121 | }; 122 | 123 | /** 124 | * @brief Takes at least two Blob%s and concatenates them along either the num 125 | * or channel dimension, outputting the result. 126 | */ 127 | template 128 | class ConcatLayer : public Layer { 129 | public: 130 | explicit ConcatLayer(const LayerParameter& param) 131 | : Layer(param) {} 132 | virtual void LayerSetUp(const vector*>& bottom, 133 | vector*>* top); 134 | 135 | virtual inline LayerParameter_LayerType type() const { 136 | return LayerParameter_LayerType_CONCAT; 137 | } 138 | virtual inline int MinBottomBlobs() const { return 2; } 139 | virtual inline int ExactNumTopBlobs() const { return 1; } 140 | 141 | protected: 142 | /** 143 | * @param bottom input Blob vector (length 2+) 144 | * -# @f$ (N \times C \times H \times W) @f$ 145 | * the inputs @f$ x_1 @f$ 146 | * -# @f$ (N \times C \times H \times W) @f$ 147 | * the inputs @f$ x_2 @f$ 148 | * -# ... 149 | * - K @f$ (N \times C \times H \times W) @f$ 150 | * the inputs @f$ x_K @f$ 151 | * @param top output Blob vector (length 1) 152 | * -# @f$ (KN \times C \times H \times W) @f$ if concat_dim == 0, or 153 | * @f$ (N \times KC \times H \times W) @f$ if concat_dim == 1: 154 | * the concatenated output @f$ 155 | * y = [\begin{array}{cccc} x_1 & x_2 & ... & x_K \end{array}] 156 | * @f$ 157 | */ 158 | virtual void Forward_cpu(const vector*>& bottom, 159 | vector*>* top); 160 | virtual void Forward_gpu(const vector*>& bottom, 161 | vector*>* top); 162 | 163 | /** 164 | * @brief Computes the error gradient w.r.t. the concatenate inputs. 165 | * 166 | * @param top output Blob vector (length 1), providing the error gradient with 167 | * respect to the outputs 168 | * -# @f$ (KN \times C \times H \times W) @f$ if concat_dim == 0, or 169 | * @f$ (N \times KC \times H \times W) @f$ if concat_dim == 1: 170 | * containing error gradients @f$ \frac{\partial E}{\partial y} @f$ 171 | * with respect to concatenated outputs @f$ y @f$ 172 | * @param propagate_down see Layer::Backward. 173 | * @param bottom input Blob vector (length K), into which the top gradient 174 | * @f$ \frac{\partial E}{\partial y} @f$ is deconcatenated back to the 175 | * inputs @f$ 176 | * \left[ \begin{array}{cccc} 177 | * \frac{\partial E}{\partial x_1} & 178 | * \frac{\partial E}{\partial x_2} & 179 | * ... & 180 | * \frac{\partial E}{\partial x_K} 181 | * \end{array} \right] = 182 | * \frac{\partial E}{\partial y} 183 | * @f$ 184 | */ 185 | virtual void Backward_cpu(const vector*>& top, 186 | const vector& propagate_down, vector*>* bottom); 187 | virtual void Backward_gpu(const vector*>& top, 188 | const vector& propagate_down, vector*>* bottom); 189 | 190 | Blob col_bob_; 191 | int count_; 192 | int num_; 193 | int channels_; 194 | int height_; 195 | int width_; 196 | int concat_dim_; 197 | }; 198 | 199 | /** 200 | * @brief Compute elementwise operations, such as product and sum, 201 | * along multiple input Blobs. 202 | * 203 | * TODO(dox): thorough documentation for Forward, Backward, and proto params. 204 | */ 205 | template 206 | class EltwiseLayer : public Layer { 207 | public: 208 | explicit EltwiseLayer(const LayerParameter& param) 209 | : Layer(param) {} 210 | virtual void LayerSetUp(const vector*>& bottom, 211 | vector*>* top); 212 | 213 | virtual inline LayerParameter_LayerType type() const { 214 | return LayerParameter_LayerType_ELTWISE; 215 | } 216 | virtual inline int MinBottomBlobs() const { return 2; } 217 | virtual inline int ExactNumTopBlobs() const { return 1; } 218 | 219 | protected: 220 | virtual void Forward_cpu(const vector*>& bottom, 221 | vector*>* top); 222 | virtual void Forward_gpu(const vector*>& bottom, 223 | vector*>* top); 224 | virtual void Backward_cpu(const vector*>& top, 225 | const vector& propagate_down, vector*>* bottom); 226 | virtual void Backward_gpu(const vector*>& top, 227 | const vector& propagate_down, vector*>* bottom); 228 | 229 | EltwiseParameter_EltwiseOp op_; 230 | vector coeffs_; 231 | Blob max_idx_; 232 | 233 | bool stable_prod_grad_; 234 | }; 235 | 236 | /** 237 | * @brief Reshapes the input Blob into flat vectors. 238 | * 239 | * Note: because this layer does not change the input values -- merely the 240 | * dimensions -- it can simply copy the input. The copy happens "virtually" 241 | * (thus taking effectively 0 real time) by setting, in Forward, the data 242 | * pointer of the top Blob to that of the bottom Blob (see Blob::ShareData), 243 | * and in Backward, the diff pointer of the bottom Blob to that of the top Blob 244 | * (see Blob::ShareDiff). 245 | */ 246 | template 247 | class FlattenLayer : public Layer { 248 | public: 249 | explicit FlattenLayer(const LayerParameter& param) 250 | : Layer(param) {} 251 | virtual void LayerSetUp(const vector*>& bottom, 252 | vector*>* top); 253 | 254 | virtual inline LayerParameter_LayerType type() const { 255 | return LayerParameter_LayerType_FLATTEN; 256 | } 257 | virtual inline int ExactNumBottomBlobs() const { return 1; } 258 | virtual inline int ExactNumTopBlobs() const { return 1; } 259 | 260 | protected: 261 | /** 262 | * @param bottom input Blob vector (length 2+) 263 | * -# @f$ (N \times C \times H \times W) @f$ 264 | * the inputs 265 | * @param top output Blob vector (length 1) 266 | * -# @f$ (N \times CHW \times 1 \times 1) @f$ 267 | * the outputs -- i.e., the (virtually) copied, flattened inputs 268 | */ 269 | virtual void Forward_cpu(const vector*>& bottom, 270 | vector*>* top); 271 | virtual void Forward_gpu(const vector*>& bottom, 272 | vector*>* top); 273 | 274 | /** 275 | * @brief Computes the error gradient w.r.t. the concatenate inputs. 276 | * 277 | * @param top output Blob vector (length 1), providing the error gradient with 278 | * respect to the outputs 279 | * @param propagate_down see Layer::Backward. 280 | * @param bottom input Blob vector (length K), into which the top error 281 | * gradient is (virtually) copied 282 | */ 283 | virtual void Backward_cpu(const vector*>& top, 284 | const vector& propagate_down, vector*>* bottom); 285 | virtual void Backward_gpu(const vector*>& top, 286 | const vector& propagate_down, vector*>* bottom); 287 | 288 | int count_; 289 | }; 290 | 291 | /** 292 | * @brief Also known as a "fully-connected" layer, computes an inner product 293 | * with a set of learned weights, and (optionally) adds biases. 294 | * 295 | * TODO(dox): thorough documentation for Forward, Backward, and proto params. 296 | */ 297 | template 298 | class InnerProductLayer : public Layer { 299 | public: 300 | explicit InnerProductLayer(const LayerParameter& param) 301 | : Layer(param) {} 302 | virtual void LayerSetUp(const vector*>& bottom, 303 | vector*>* top); 304 | 305 | virtual inline LayerParameter_LayerType type() const { 306 | return LayerParameter_LayerType_INNER_PRODUCT; 307 | } 308 | virtual inline int ExactNumBottomBlobs() const { return 1; } 309 | virtual inline int ExactNumTopBlobs() const { return 1; } 310 | 311 | protected: 312 | virtual void Forward_cpu(const vector*>& bottom, 313 | vector*>* top); 314 | virtual void Forward_gpu(const vector*>& bottom, 315 | vector*>* top); 316 | virtual void Backward_cpu(const vector*>& top, 317 | const vector& propagate_down, vector*>* bottom); 318 | virtual void Backward_gpu(const vector*>& top, 319 | const vector& propagate_down, vector*>* bottom); 320 | 321 | int M_; 322 | int K_; 323 | int N_; 324 | bool bias_term_; 325 | Blob bias_multiplier_; 326 | }; 327 | 328 | 329 | /** 330 | * @brief Normalizes the input to have 0-mean and/or unit (1) variance. 331 | * 332 | * TODO(dox): thorough documentation for Forward, Backward, and proto params. 333 | */ 334 | template 335 | class MVNLayer : public Layer { 336 | public: 337 | explicit MVNLayer(const LayerParameter& param) 338 | : Layer(param) {} 339 | virtual void LayerSetUp(const vector*>& bottom, 340 | vector*>* top); 341 | 342 | virtual inline LayerParameter_LayerType type() const { 343 | return LayerParameter_LayerType_MVN; 344 | } 345 | virtual inline int ExactNumBottomBlobs() const { return 1; } 346 | virtual inline int ExactNumTopBlobs() const { return 1; } 347 | 348 | protected: 349 | virtual void Forward_cpu(const vector*>& bottom, 350 | vector*>* top); 351 | virtual void Forward_gpu(const vector*>& bottom, 352 | vector*>* top); 353 | virtual void Backward_cpu(const vector*>& top, 354 | const vector& propagate_down, vector*>* bottom); 355 | virtual void Backward_gpu(const vector*>& top, 356 | const vector& propagate_down, vector*>* bottom); 357 | 358 | Blob mean_, variance_, temp_; 359 | 360 | /// sum_multiplier is used to carry out sum using BLAS 361 | Blob sum_multiplier_; 362 | }; 363 | 364 | /** 365 | * @brief Ignores bottom blobs while producing no top blobs. (This is useful 366 | * to suppress outputs during testing.) 367 | */ 368 | template 369 | class SilenceLayer : public Layer { 370 | public: 371 | explicit SilenceLayer(const LayerParameter& param) 372 | : Layer(param) {} 373 | virtual void LayerSetUp(const vector*>& bottom, 374 | vector*>* top) {} 375 | 376 | virtual inline LayerParameter_LayerType type() const { 377 | return LayerParameter_LayerType_SILENCE; 378 | } 379 | virtual inline int MinBottomBlobs() const { return 1; } 380 | virtual inline int ExactNumTopBlobs() const { return 0; } 381 | 382 | protected: 383 | virtual void Forward_cpu(const vector*>& bottom, 384 | vector*>* top) {} 385 | // We can't define Forward_gpu here, since STUB_GPU will provide 386 | // its own definition for CPU_ONLY mode. 387 | virtual void Forward_gpu(const vector*>& bottom, 388 | vector*>* top); 389 | virtual void Backward_cpu(const vector*>& top, 390 | const vector& propagate_down, vector*>* bottom); 391 | virtual void Backward_gpu(const vector*>& top, 392 | const vector& propagate_down, vector*>* bottom); 393 | }; 394 | 395 | /** 396 | * @brief Computes the softmax function. 397 | * 398 | * TODO(dox): thorough documentation for Forward, Backward, and proto params. 399 | */ 400 | template 401 | class SoftmaxLayer : public Layer { 402 | public: 403 | explicit SoftmaxLayer(const LayerParameter& param) 404 | : Layer(param) {} 405 | virtual void LayerSetUp(const vector*>& bottom, 406 | vector*>* top); 407 | 408 | virtual inline LayerParameter_LayerType type() const { 409 | return LayerParameter_LayerType_SOFTMAX; 410 | } 411 | virtual inline int ExactNumBottomBlobs() const { return 1; } 412 | virtual inline int ExactNumTopBlobs() const { return 1; } 413 | 414 | protected: 415 | virtual void Forward_cpu(const vector*>& bottom, 416 | vector*>* top); 417 | virtual void Forward_gpu(const vector*>& bottom, 418 | vector*>* top); 419 | virtual void Backward_cpu(const vector*>& top, 420 | const vector& propagate_down, vector*>* bottom); 421 | virtual void Backward_gpu(const vector*>& top, 422 | const vector& propagate_down, vector*>* bottom); 423 | 424 | /// sum_multiplier is used to carry out sum using BLAS 425 | Blob sum_multiplier_; 426 | /// scale is an intermediate Blob to hold temporary results. 427 | Blob scale_; 428 | }; 429 | 430 | #ifdef USE_CUDNN 431 | /** 432 | * @brief cuDNN implementation of SoftmaxLayer. 433 | * Fallback to SoftmaxLayer for CPU mode. 434 | */ 435 | template 436 | class CuDNNSoftmaxLayer : public SoftmaxLayer { 437 | public: 438 | explicit CuDNNSoftmaxLayer(const LayerParameter& param) 439 | : SoftmaxLayer(param) {} 440 | virtual void LayerSetUp(const vector*>& bottom, 441 | vector*>* top); 442 | virtual ~CuDNNSoftmaxLayer(); 443 | 444 | protected: 445 | virtual void Forward_gpu(const vector*>& bottom, 446 | vector*>* top); 447 | virtual void Backward_gpu(const vector*>& top, 448 | const vector& propagate_down, vector*>* bottom); 449 | 450 | cudnnHandle_t handle_; 451 | cudnnTensor4dDescriptor_t bottom_desc_; 452 | cudnnTensor4dDescriptor_t top_desc_; 453 | }; 454 | #endif 455 | 456 | /** 457 | * @brief Creates a "split" path in the network by copying the bottom Blob 458 | * into multiple top Blob%s to be used by multiple consuming layers. 459 | * 460 | * TODO(dox): thorough documentation for Forward, Backward, and proto params. 461 | */ 462 | template 463 | class SplitLayer : public Layer { 464 | public: 465 | explicit SplitLayer(const LayerParameter& param) 466 | : Layer(param) {} 467 | virtual void LayerSetUp(const vector*>& bottom, 468 | vector*>* top); 469 | 470 | virtual inline LayerParameter_LayerType type() const { 471 | return LayerParameter_LayerType_SPLIT; 472 | } 473 | virtual inline int ExactNumBottomBlobs() const { return 1; } 474 | virtual inline int MinTopBlobs() const { return 1; } 475 | 476 | protected: 477 | virtual void Forward_cpu(const vector*>& bottom, 478 | vector*>* top); 479 | virtual void Forward_gpu(const vector*>& bottom, 480 | vector*>* top); 481 | virtual void Backward_cpu(const vector*>& top, 482 | const vector& propagate_down, vector*>* bottom); 483 | virtual void Backward_gpu(const vector*>& top, 484 | const vector& propagate_down, vector*>* bottom); 485 | 486 | int count_; 487 | }; 488 | 489 | /** 490 | * @brief Takes a Blob and slices it along either the num or channel dimension, 491 | * outputting multiple sliced Blob results. 492 | * 493 | * TODO(dox): thorough documentation for Forward, Backward, and proto params. 494 | */ 495 | template 496 | class SliceLayer : public Layer { 497 | public: 498 | explicit SliceLayer(const LayerParameter& param) 499 | : Layer(param) {} 500 | virtual void LayerSetUp(const vector*>& bottom, 501 | vector*>* top); 502 | 503 | virtual inline LayerParameter_LayerType type() const { 504 | return LayerParameter_LayerType_SLICE; 505 | } 506 | virtual inline int ExactNumBottomBlobs() const { return 1; } 507 | virtual inline int MinTopBlobs() const { return 2; } 508 | 509 | protected: 510 | virtual void Forward_cpu(const vector*>& bottom, 511 | vector*>* top); 512 | virtual void Forward_gpu(const vector*>& bottom, 513 | vector*>* top); 514 | virtual void Backward_cpu(const vector*>& top, 515 | const vector& propagate_down, vector*>* bottom); 516 | virtual void Backward_gpu(const vector*>& top, 517 | const vector& propagate_down, vector*>* bottom); 518 | 519 | Blob col_bob_; 520 | int count_; 521 | int num_; 522 | int channels_; 523 | int height_; 524 | int width_; 525 | int slice_dim_; 526 | vector slice_point_; 527 | }; 528 | 529 | } // namespace caffe 530 | 531 | #endif // CAFFE_COMMON_LAYERS_HPP_ 532 | -------------------------------------------------------------------------------- /layer_factory.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "caffe/layer.hpp" 4 | #include "caffe/proto/caffe.pb.h" 5 | #include "caffe/vision_layers.hpp" 6 | 7 | namespace caffe { 8 | 9 | // GetLayer() defines the overall layer factory. The Get*Layer() functions 10 | // define factories for layers with multiple computational engines. 11 | 12 | // Get convolution layer according to engine. 13 | template 14 | ConvolutionLayer* GetConvolutionLayer(const string& name, 15 | const LayerParameter& param) { 16 | ConvolutionParameter_Engine engine = param.convolution_param().engine(); 17 | if (engine == ConvolutionParameter_Engine_DEFAULT) { 18 | engine = ConvolutionParameter_Engine_CAFFE; 19 | #ifdef USE_CUDNN 20 | engine = ConvolutionParameter_Engine_CUDNN; 21 | #endif 22 | } 23 | if (engine == ConvolutionParameter_Engine_CAFFE) { 24 | return new ConvolutionLayer(param); 25 | #ifdef USE_CUDNN 26 | } else if (engine == ConvolutionParameter_Engine_CUDNN) { 27 | return new CuDNNConvolutionLayer(param); 28 | #endif 29 | } else { 30 | LOG(FATAL) << "Layer " << name << " has unknown engine."; 31 | } 32 | } 33 | 34 | template ConvolutionLayer* GetConvolutionLayer(const string& name, 35 | const LayerParameter& param); 36 | template ConvolutionLayer* GetConvolutionLayer(const string& name, 37 | const LayerParameter& param); 38 | 39 | // Get pooling layer according to engine. 40 | template 41 | PoolingLayer* GetPoolingLayer(const string& name, 42 | const LayerParameter& param) { 43 | PoolingParameter_Engine engine = param.pooling_param().engine(); 44 | if (engine == PoolingParameter_Engine_DEFAULT) { 45 | engine = PoolingParameter_Engine_CAFFE; 46 | #ifdef USE_CUDNN 47 | engine = PoolingParameter_Engine_CUDNN; 48 | #endif 49 | } 50 | if (engine == PoolingParameter_Engine_CAFFE) { 51 | return new PoolingLayer(param); 52 | #ifdef USE_CUDNN 53 | } else if (engine == PoolingParameter_Engine_CUDNN) { 54 | return new CuDNNPoolingLayer(param); 55 | #endif 56 | } else { 57 | LOG(FATAL) << "Layer " << name << " has unknown engine."; 58 | } 59 | } 60 | 61 | template PoolingLayer* GetPoolingLayer(const string& name, 62 | const LayerParameter& param); 63 | template PoolingLayer* GetPoolingLayer(const string& name, 64 | const LayerParameter& param); 65 | 66 | // Get relu layer according to engine. 67 | template 68 | ReLULayer* GetReLULayer(const string& name, 69 | const LayerParameter& param) { 70 | ReLUParameter_Engine engine = param.relu_param().engine(); 71 | if (engine == ReLUParameter_Engine_DEFAULT) { 72 | engine = ReLUParameter_Engine_CAFFE; 73 | #ifdef USE_CUDNN 74 | engine = ReLUParameter_Engine_CUDNN; 75 | #endif 76 | } 77 | if (engine == ReLUParameter_Engine_CAFFE) { 78 | return new ReLULayer(param); 79 | #ifdef USE_CUDNN 80 | } else if (engine == ReLUParameter_Engine_CUDNN) { 81 | return new CuDNNReLULayer(param); 82 | #endif 83 | } else { 84 | LOG(FATAL) << "Layer " << name << " has unknown engine."; 85 | } 86 | } 87 | 88 | template ReLULayer* GetReLULayer(const string& name, 89 | const LayerParameter& param); 90 | template ReLULayer* GetReLULayer(const string& name, 91 | const LayerParameter& param); 92 | 93 | // Get sigmoid layer according to engine. 94 | template 95 | SigmoidLayer* GetSigmoidLayer(const string& name, 96 | const LayerParameter& param) { 97 | SigmoidParameter_Engine engine = param.sigmoid_param().engine(); 98 | if (engine == SigmoidParameter_Engine_DEFAULT) { 99 | engine = SigmoidParameter_Engine_CAFFE; 100 | #ifdef USE_CUDNN 101 | engine = SigmoidParameter_Engine_CUDNN; 102 | #endif 103 | } 104 | if (engine == SigmoidParameter_Engine_CAFFE) { 105 | return new SigmoidLayer(param); 106 | #ifdef USE_CUDNN 107 | } else if (engine == SigmoidParameter_Engine_CUDNN) { 108 | return new CuDNNSigmoidLayer(param); 109 | #endif 110 | } else { 111 | LOG(FATAL) << "Layer " << name << " has unknown engine."; 112 | } 113 | } 114 | 115 | template SigmoidLayer* GetSigmoidLayer(const string& name, 116 | const LayerParameter& param); 117 | template SigmoidLayer* GetSigmoidLayer(const string& name, 118 | const LayerParameter& param); 119 | 120 | // Get tanh layer according to engine. 121 | template 122 | TanHLayer* GetTanHLayer(const string& name, 123 | const LayerParameter& param) { 124 | TanHParameter_Engine engine = param.tanh_param().engine(); 125 | if (engine == TanHParameter_Engine_DEFAULT) { 126 | engine = TanHParameter_Engine_CAFFE; 127 | #ifdef USE_CUDNN 128 | engine = TanHParameter_Engine_CUDNN; 129 | #endif 130 | } 131 | if (engine == TanHParameter_Engine_CAFFE) { 132 | return new TanHLayer(param); 133 | #ifdef USE_CUDNN 134 | } else if (engine == TanHParameter_Engine_CUDNN) { 135 | return new CuDNNTanHLayer(param); 136 | #endif 137 | } else { 138 | LOG(FATAL) << "Layer " << name << " has unknown engine."; 139 | } 140 | } 141 | 142 | template TanHLayer* GetTanHLayer(const string& name, 143 | const LayerParameter& param); 144 | template TanHLayer* GetTanHLayer(const string& name, 145 | const LayerParameter& param); 146 | 147 | // Get softmax layer according to engine. 148 | template 149 | SoftmaxLayer* GetSoftmaxLayer(const string& name, 150 | const LayerParameter& param) { 151 | SoftmaxParameter_Engine engine = param.softmax_param().engine(); 152 | if (engine == SoftmaxParameter_Engine_DEFAULT) { 153 | engine = SoftmaxParameter_Engine_CAFFE; 154 | #ifdef USE_CUDNN 155 | engine = SoftmaxParameter_Engine_CUDNN; 156 | #endif 157 | } 158 | if (engine == SoftmaxParameter_Engine_CAFFE) { 159 | return new SoftmaxLayer(param); 160 | #ifdef USE_CUDNN 161 | } else if (engine == SoftmaxParameter_Engine_CUDNN) { 162 | return new CuDNNSoftmaxLayer(param); 163 | #endif 164 | } else { 165 | LOG(FATAL) << "Layer " << name << " has unknown engine."; 166 | } 167 | } 168 | 169 | template SoftmaxLayer* GetSoftmaxLayer(const string& name, 170 | const LayerParameter& param); 171 | template SoftmaxLayer* GetSoftmaxLayer(const string& name, 172 | const LayerParameter& param); 173 | 174 | // A function to get a specific layer from the specification given in 175 | // LayerParameter. Ideally this would be replaced by a factory pattern, 176 | // but we will leave it this way for now. 177 | template 178 | Layer* GetLayer(const LayerParameter& param) { 179 | const string& name = param.name(); 180 | const LayerParameter_LayerType& type = param.type(); 181 | switch (type) { 182 | case LayerParameter_LayerType_ACCURACY: 183 | return new AccuracyLayer(param); 184 | case LayerParameter_LayerType_ABSVAL: 185 | return new AbsValLayer(param); 186 | case LayerParameter_LayerType_ARGMAX: 187 | return new ArgMaxLayer(param); 188 | case LayerParameter_LayerType_BN: 189 | return new BNLayer(param); 190 | case LayerParameter_LayerType_BNLL: 191 | return new BNLLLayer(param); 192 | case LayerParameter_LayerType_COMPACT_DATA: 193 | return new CompactDataLayer(param); 194 | case LayerParameter_LayerType_CONCAT: 195 | return new ConcatLayer(param); 196 | case LayerParameter_LayerType_CONVOLUTION: 197 | return GetConvolutionLayer(name, param); 198 | case LayerParameter_LayerType_DATA: 199 | return new DataLayer(param); 200 | case LayerParameter_LayerType_DROPOUT: 201 | return new DropoutLayer(param); 202 | case LayerParameter_LayerType_DUMMY_DATA: 203 | return new DummyDataLayer(param); 204 | case LayerParameter_LayerType_EUCLIDEAN_LOSS: 205 | return new EuclideanLossLayer(param); 206 | case LayerParameter_LayerType_ELTWISE: 207 | return new EltwiseLayer(param); 208 | case LayerParameter_LayerType_FLATTEN: 209 | return new FlattenLayer(param); 210 | case LayerParameter_LayerType_HDF5_DATA: 211 | return new HDF5DataLayer(param); 212 | case LayerParameter_LayerType_HDF5_OUTPUT: 213 | return new HDF5OutputLayer(param); 214 | case LayerParameter_LayerType_HINGE_LOSS: 215 | return new HingeLossLayer(param); 216 | case LayerParameter_LayerType_IMAGE_DATA: 217 | return new ImageDataLayer(param); 218 | case LayerParameter_LayerType_IM2COL: 219 | return new Im2colLayer(param); 220 | case LayerParameter_LayerType_INFOGAIN_LOSS: 221 | return new InfogainLossLayer(param); 222 | case LayerParameter_LayerType_INNER_PRODUCT: 223 | return new InnerProductLayer(param); 224 | case LayerParameter_LayerType_LRN: 225 | return new LRNLayer(param); 226 | case LayerParameter_LayerType_MEMORY_DATA: 227 | return new MemoryDataLayer(param); 228 | case LayerParameter_LayerType_MVN: 229 | return new MVNLayer(param); 230 | case LayerParameter_LayerType_MULTINOMIAL_LOGISTIC_LOSS: 231 | return new MultinomialLogisticLossLayer(param); 232 | case LayerParameter_LayerType_POOLING: 233 | return GetPoolingLayer(name, param); 234 | case LayerParameter_LayerType_POWER: 235 | return new PowerLayer(param); 236 | case LayerParameter_LayerType_PRELU: 237 | return new PReLULayer(param); 238 | case LayerParameter_LayerType_RELU: 239 | return GetReLULayer(name, param); 240 | case LayerParameter_LayerType_SILENCE: 241 | return new SilenceLayer(param); 242 | case LayerParameter_LayerType_SIGMOID: 243 | return GetSigmoidLayer(name, param); 244 | case LayerParameter_LayerType_SIGMOID_CROSS_ENTROPY_LOSS: 245 | return new SigmoidCrossEntropyLossLayer(param); 246 | case LayerParameter_LayerType_SLICE: 247 | return new SliceLayer(param); 248 | case LayerParameter_LayerType_SOFTMAX: 249 | return GetSoftmaxLayer(name, param); 250 | case LayerParameter_LayerType_SOFTMAX_LOSS: 251 | return new SoftmaxWithLossLayer(param); 252 | case LayerParameter_LayerType_SPLIT: 253 | return new SplitLayer(param); 254 | case LayerParameter_LayerType_TANH: 255 | return GetTanHLayer(name, param); 256 | case LayerParameter_LayerType_WINDOW_DATA: 257 | return new WindowDataLayer(param); 258 | case LayerParameter_LayerType_NONE: 259 | LOG(FATAL) << "Layer " << name << " has unspecified type."; 260 | default: 261 | LOG(FATAL) << "Layer " << name << " has unknown type " << type; 262 | } 263 | // just to suppress old compiler warnings. 264 | return (Layer*)(NULL); 265 | } 266 | 267 | template Layer* GetLayer(const LayerParameter& param); 268 | template Layer* GetLayer(const LayerParameter& param); 269 | 270 | } // namespace caffe 271 | -------------------------------------------------------------------------------- /lenet_BN_sgd.log: -------------------------------------------------------------------------------- 1 | I0219 20:24:25.883813 101920 caffe.cpp:108] Use GPU with device ID 0 2 | I0219 20:24:27.258891 101920 common.cpp:22] System entropy source not available, using fallback algorithm to generate seed instead. 3 | I0219 20:24:27.258891 101920 caffe.cpp:116] Starting Optimization 4 | I0219 20:24:27.258891 101920 solver.cpp:32] Initializing solver from parameters: 5 | test_iter: 100 6 | test_interval: 500 7 | base_lr: 0.01 8 | display: 100 9 | max_iter: 10000 10 | lr_policy: "inv" 11 | gamma: 0.0001 12 | power: 0.75 13 | momentum: 0.9 14 | weight_decay: 0.0005 15 | snapshot: 5000 16 | snapshot_prefix: "lenet" 17 | solver_mode: GPU 18 | random_seed: 1234 19 | net: "lenet_BN_train_valid.prototxt" 20 | solver_type: SGD 21 | delta: 1e-008 22 | I0219 20:24:27.258891 101920 solver.cpp:77] Creating training net from net file: lenet_BN_train_valid.prototxt 23 | E0219 20:24:27.259891 101920 upgrade_proto.cpp:603] Attempting to upgrade input file specified using deprecated transformation parameters: lenet_BN_train_valid.prototxt 24 | I0219 20:24:27.259891 101920 upgrade_proto.cpp:606] Successfully upgraded file specified using deprecated data transformation parameters. 25 | E0219 20:24:27.259891 101920 upgrade_proto.cpp:608] Note that future Caffe releases will only support transform_param messages for transformation fields. 26 | I0219 20:24:27.259891 101920 net.cpp:275] The NetState phase (0) differed from the phase (1) specified by a rule in layer mnist 27 | I0219 20:24:27.259891 101920 net.cpp:275] The NetState phase (0) differed from the phase (1) specified by a rule in layer prob 28 | I0219 20:24:27.259891 101920 net.cpp:275] The NetState phase (0) differed from the phase (1) specified by a rule in layer accuracy 29 | I0219 20:24:27.259891 101920 net.cpp:39] Initializing net from parameters: 30 | name: "LeNet" 31 | layers { 32 | top: "data" 33 | top: "label" 34 | name: "mnist" 35 | type: DATA 36 | data_param { 37 | source: "mnist-train-leveldb" 38 | batch_size: 100 39 | } 40 | include { 41 | phase: TRAIN 42 | } 43 | transform_param { 44 | scale: 0.00390625 45 | } 46 | } 47 | layers { 48 | bottom: "data" 49 | top: "conv1" 50 | name: "conv1" 51 | type: CONVOLUTION 52 | blobs_lr: 1 53 | blobs_lr: 2 54 | convolution_param { 55 | num_output: 20 56 | kernel_size: 5 57 | stride: 1 58 | weight_filler { 59 | type: "xavier" 60 | } 61 | bias_filler { 62 | type: "constant" 63 | } 64 | } 65 | } 66 | layers { 67 | bottom: "conv1" 68 | top: "conv1_bn" 69 | name: "conv1_bn" 70 | type: BN 71 | blobs_lr: 1 72 | blobs_lr: 1 73 | weight_decay: 0 74 | weight_decay: 0 75 | bn_param { 76 | scale_filler { 77 | type: "constant" 78 | value: 1 79 | } 80 | shift_filler { 81 | type: "constant" 82 | value: 0 83 | } 84 | } 85 | } 86 | layers { 87 | bottom: "conv1_bn" 88 | top: "pool1" 89 | name: "pool1" 90 | type: POOLING 91 | pooling_param { 92 | pool: MAX 93 | kernel_size: 2 94 | stride: 2 95 | } 96 | } 97 | layers { 98 | bottom: "pool1" 99 | top: "conv2" 100 | name: "conv2" 101 | type: CONVOLUTION 102 | blobs_lr: 1 103 | blobs_lr: 2 104 | convolution_param { 105 | num_output: 50 106 | kernel_size: 5 107 | stride: 1 108 | weight_filler { 109 | type: "xavier" 110 | } 111 | bias_filler { 112 | type: "constant" 113 | } 114 | } 115 | } 116 | layers { 117 | bottom: "conv2" 118 | top: "conv2_bn" 119 | name: "conv2_bn" 120 | type: BN 121 | blobs_lr: 1 122 | blobs_lr: 1 123 | weight_decay: 0 124 | weight_decay: 0 125 | bn_param { 126 | scale_filler { 127 | type: "constant" 128 | value: 1 129 | } 130 | shift_filler { 131 | type: "constant" 132 | value: 0 133 | } 134 | } 135 | } 136 | layers { 137 | bottom: "conv2_bn" 138 | top: "pool2" 139 | name: "pool2" 140 | type: POOLING 141 | pooling_param { 142 | pool: MAX 143 | kernel_size: 2 144 | stride: 2 145 | } 146 | } 147 | layers { 148 | bottom: "pool2" 149 | top: "ip1" 150 | name: "ip1" 151 | type: INNER_PRODUCT 152 | blobs_lr: 1 153 | blobs_lr: 2 154 | inner_product_param { 155 | num_output: 500 156 | weight_filler { 157 | type: "xavier" 158 | } 159 | bias_filler { 160 | type: "constant" 161 | } 162 | } 163 | } 164 | layers { 165 | bottom: "ip1" 166 | top: "ip1" 167 | name: "relu1" 168 | type: RELU 169 | } 170 | layers { 171 | bottom: "ip1" 172 | top: "ip2" 173 | name: "ip2" 174 | type: INNER_PRODUCT 175 | blobs_lr: 1 176 | blobs_lr: 2 177 | inner_product_param { 178 | num_output: 10 179 | weight_filler { 180 | type: "xavier" 181 | } 182 | bias_filler { 183 | type: "constant" 184 | } 185 | } 186 | } 187 | layers { 188 | bottom: "ip2" 189 | bottom: "label" 190 | name: "loss" 191 | type: SOFTMAX_LOSS 192 | include { 193 | phase: TRAIN 194 | } 195 | } 196 | state { 197 | phase: TRAIN 198 | } 199 | I0219 20:24:27.259891 101920 net.cpp:67] Creating Layer mnist 200 | I0219 20:24:27.259891 101920 net.cpp:356] mnist -> data 201 | I0219 20:24:27.259891 101920 net.cpp:356] mnist -> label 202 | I0219 20:24:27.259891 101920 net.cpp:96] Setting up mnist 203 | I0219 20:24:27.259891 101920 data_layer.cpp:46] Opening leveldb mnist-train-leveldb 204 | I0219 20:24:27.266892 101920 data_layer.cpp:130] output data size: 100,1,28,28 205 | I0219 20:24:27.266892 101920 net.cpp:103] Top shape: 100 1 28 28 (78400) 206 | I0219 20:24:27.266892 101920 net.cpp:103] Top shape: 100 1 1 1 (100) 207 | I0219 20:24:27.266892 101920 net.cpp:67] Creating Layer conv1 208 | I0219 20:24:27.266892 101920 net.cpp:394] conv1 <- data 209 | I0219 20:24:27.266892 101920 net.cpp:356] conv1 -> conv1 210 | I0219 20:24:27.266892 101920 net.cpp:96] Setting up conv1 211 | I0219 20:24:27.529907 101920 net.cpp:103] Top shape: 100 20 24 24 (1152000) 212 | I0219 20:24:27.529907 101920 net.cpp:67] Creating Layer conv1_bn 213 | I0219 20:24:27.529907 101920 net.cpp:394] conv1_bn <- conv1 214 | I0219 20:24:27.529907 101920 net.cpp:356] conv1_bn -> conv1_bn 215 | I0219 20:24:27.529907 101920 net.cpp:96] Setting up conv1_bn 216 | I0219 20:24:27.529907 101920 net.cpp:103] Top shape: 100 20 24 24 (1152000) 217 | I0219 20:24:27.529907 101920 net.cpp:67] Creating Layer pool1 218 | I0219 20:24:27.529907 101920 net.cpp:394] pool1 <- conv1_bn 219 | I0219 20:24:27.529907 101920 net.cpp:356] pool1 -> pool1 220 | I0219 20:24:27.529907 101920 net.cpp:96] Setting up pool1 221 | I0219 20:24:27.529907 101920 net.cpp:103] Top shape: 100 20 12 12 (288000) 222 | I0219 20:24:27.529907 101920 net.cpp:67] Creating Layer conv2 223 | I0219 20:24:27.529907 101920 net.cpp:394] conv2 <- pool1 224 | I0219 20:24:27.529907 101920 net.cpp:356] conv2 -> conv2 225 | I0219 20:24:27.529907 101920 net.cpp:96] Setting up conv2 226 | I0219 20:24:27.529907 101920 net.cpp:103] Top shape: 100 50 8 8 (320000) 227 | I0219 20:24:27.530907 101920 net.cpp:67] Creating Layer conv2_bn 228 | I0219 20:24:27.530907 101920 net.cpp:394] conv2_bn <- conv2 229 | I0219 20:24:27.530907 101920 net.cpp:356] conv2_bn -> conv2_bn 230 | I0219 20:24:27.530907 101920 net.cpp:96] Setting up conv2_bn 231 | I0219 20:24:27.530907 101920 net.cpp:103] Top shape: 100 50 8 8 (320000) 232 | I0219 20:24:27.530907 101920 net.cpp:67] Creating Layer pool2 233 | I0219 20:24:27.530907 101920 net.cpp:394] pool2 <- conv2_bn 234 | I0219 20:24:27.530907 101920 net.cpp:356] pool2 -> pool2 235 | I0219 20:24:27.530907 101920 net.cpp:96] Setting up pool2 236 | I0219 20:24:27.530907 101920 net.cpp:103] Top shape: 100 50 4 4 (80000) 237 | I0219 20:24:27.530907 101920 net.cpp:67] Creating Layer ip1 238 | I0219 20:24:27.530907 101920 net.cpp:394] ip1 <- pool2 239 | I0219 20:24:27.530907 101920 net.cpp:356] ip1 -> ip1 240 | I0219 20:24:27.530907 101920 net.cpp:96] Setting up ip1 241 | I0219 20:24:27.533907 101920 net.cpp:103] Top shape: 100 500 1 1 (50000) 242 | I0219 20:24:27.533907 101920 net.cpp:67] Creating Layer relu1 243 | I0219 20:24:27.533907 101920 net.cpp:394] relu1 <- ip1 244 | I0219 20:24:27.533907 101920 net.cpp:345] relu1 -> ip1 (in-place) 245 | I0219 20:24:27.533907 101920 net.cpp:96] Setting up relu1 246 | I0219 20:24:27.533907 101920 net.cpp:103] Top shape: 100 500 1 1 (50000) 247 | I0219 20:24:27.533907 101920 net.cpp:67] Creating Layer ip2 248 | I0219 20:24:27.534907 101920 net.cpp:394] ip2 <- ip1 249 | I0219 20:24:27.534907 101920 net.cpp:356] ip2 -> ip2 250 | I0219 20:24:27.534907 101920 net.cpp:96] Setting up ip2 251 | I0219 20:24:27.534907 101920 net.cpp:103] Top shape: 100 10 1 1 (1000) 252 | I0219 20:24:27.534907 101920 net.cpp:67] Creating Layer loss 253 | I0219 20:24:27.534907 101920 net.cpp:394] loss <- ip2 254 | I0219 20:24:27.534907 101920 net.cpp:394] loss <- label 255 | I0219 20:24:27.534907 101920 net.cpp:356] loss -> (automatic) 256 | I0219 20:24:27.534907 101920 net.cpp:96] Setting up loss 257 | I0219 20:24:27.534907 101920 net.cpp:103] Top shape: 1 1 1 1 (1) 258 | I0219 20:24:27.534907 101920 net.cpp:109] with loss weight 1 259 | I0219 20:24:27.534907 101920 net.cpp:170] loss needs backward computation. 260 | I0219 20:24:27.534907 101920 net.cpp:170] ip2 needs backward computation. 261 | I0219 20:24:27.534907 101920 net.cpp:170] relu1 needs backward computation. 262 | I0219 20:24:27.534907 101920 net.cpp:170] ip1 needs backward computation. 263 | I0219 20:24:27.534907 101920 net.cpp:170] pool2 needs backward computation. 264 | I0219 20:24:27.534907 101920 net.cpp:170] conv2_bn needs backward computation. 265 | I0219 20:24:27.534907 101920 net.cpp:170] conv2 needs backward computation. 266 | I0219 20:24:27.534907 101920 net.cpp:170] pool1 needs backward computation. 267 | I0219 20:24:27.534907 101920 net.cpp:170] conv1_bn needs backward computation. 268 | I0219 20:24:27.534907 101920 net.cpp:170] conv1 needs backward computation. 269 | I0219 20:24:27.534907 101920 net.cpp:172] mnist does not need backward computation. 270 | I0219 20:24:27.534907 101920 net.cpp:467] Collecting Learning Rate and Weight Decay. 271 | I0219 20:24:27.534907 101920 net.cpp:219] Network initialization done. 272 | I0219 20:24:27.534907 101920 net.cpp:220] Memory required for data: 13966004 273 | E0219 20:24:27.534907 101920 upgrade_proto.cpp:603] Attempting to upgrade input file specified using deprecated transformation parameters: lenet_BN_train_valid.prototxt 274 | I0219 20:24:27.534907 101920 upgrade_proto.cpp:606] Successfully upgraded file specified using deprecated data transformation parameters. 275 | E0219 20:24:27.534907 101920 upgrade_proto.cpp:608] Note that future Caffe releases will only support transform_param messages for transformation fields. 276 | I0219 20:24:27.534907 101920 solver.cpp:161] Creating test net (#0) specified by net file: lenet_BN_train_valid.prototxt 277 | I0219 20:24:27.535907 101920 net.cpp:275] The NetState phase (1) differed from the phase (0) specified by a rule in layer mnist 278 | I0219 20:24:27.535907 101920 net.cpp:275] The NetState phase (1) differed from the phase (0) specified by a rule in layer loss 279 | I0219 20:24:27.535907 101920 net.cpp:39] Initializing net from parameters: 280 | name: "LeNet" 281 | layers { 282 | top: "data" 283 | top: "label" 284 | name: "mnist" 285 | type: DATA 286 | data_param { 287 | source: "mnist-test-leveldb" 288 | batch_size: 100 289 | } 290 | include { 291 | phase: TEST 292 | } 293 | transform_param { 294 | scale: 0.00390625 295 | } 296 | } 297 | layers { 298 | bottom: "data" 299 | top: "conv1" 300 | name: "conv1" 301 | type: CONVOLUTION 302 | blobs_lr: 1 303 | blobs_lr: 2 304 | convolution_param { 305 | num_output: 20 306 | kernel_size: 5 307 | stride: 1 308 | weight_filler { 309 | type: "xavier" 310 | } 311 | bias_filler { 312 | type: "constant" 313 | } 314 | } 315 | } 316 | layers { 317 | bottom: "conv1" 318 | top: "conv1_bn" 319 | name: "conv1_bn" 320 | type: BN 321 | blobs_lr: 1 322 | blobs_lr: 1 323 | weight_decay: 0 324 | weight_decay: 0 325 | bn_param { 326 | scale_filler { 327 | type: "constant" 328 | value: 1 329 | } 330 | shift_filler { 331 | type: "constant" 332 | value: 0 333 | } 334 | } 335 | } 336 | layers { 337 | bottom: "conv1_bn" 338 | top: "pool1" 339 | name: "pool1" 340 | type: POOLING 341 | pooling_param { 342 | pool: MAX 343 | kernel_size: 2 344 | stride: 2 345 | } 346 | } 347 | layers { 348 | bottom: "pool1" 349 | top: "conv2" 350 | name: "conv2" 351 | type: CONVOLUTION 352 | blobs_lr: 1 353 | blobs_lr: 2 354 | convolution_param { 355 | num_output: 50 356 | kernel_size: 5 357 | stride: 1 358 | weight_filler { 359 | type: "xavier" 360 | } 361 | bias_filler { 362 | type: "constant" 363 | } 364 | } 365 | } 366 | layers { 367 | bottom: "conv2" 368 | top: "conv2_bn" 369 | name: "conv2_bn" 370 | type: BN 371 | blobs_lr: 1 372 | blobs_lr: 1 373 | weight_decay: 0 374 | weight_decay: 0 375 | bn_param { 376 | scale_filler { 377 | type: "constant" 378 | value: 1 379 | } 380 | shift_filler { 381 | type: "constant" 382 | value: 0 383 | } 384 | } 385 | } 386 | layers { 387 | bottom: "conv2_bn" 388 | top: "pool2" 389 | name: "pool2" 390 | type: POOLING 391 | pooling_param { 392 | pool: MAX 393 | kernel_size: 2 394 | stride: 2 395 | } 396 | } 397 | layers { 398 | bottom: "pool2" 399 | top: "ip1" 400 | name: "ip1" 401 | type: INNER_PRODUCT 402 | blobs_lr: 1 403 | blobs_lr: 2 404 | inner_product_param { 405 | num_output: 500 406 | weight_filler { 407 | type: "xavier" 408 | } 409 | bias_filler { 410 | type: "constant" 411 | } 412 | } 413 | } 414 | layers { 415 | bottom: "ip1" 416 | top: "ip1" 417 | name: "relu1" 418 | type: RELU 419 | } 420 | layers { 421 | bottom: "ip1" 422 | top: "ip2" 423 | name: "ip2" 424 | type: INNER_PRODUCT 425 | blobs_lr: 1 426 | blobs_lr: 2 427 | inner_product_param { 428 | num_output: 10 429 | weight_filler { 430 | type: "xavier" 431 | } 432 | bias_filler { 433 | type: "constant" 434 | } 435 | } 436 | } 437 | layers { 438 | bottom: "ip2" 439 | top: "prob" 440 | name: "prob" 441 | type: SOFTMAX 442 | include { 443 | phase: TEST 444 | } 445 | } 446 | layers { 447 | bottom: "prob" 448 | bottom: "label" 449 | top: "accuracy" 450 | name: "accuracy" 451 | type: ACCURACY 452 | include { 453 | phase: TEST 454 | } 455 | } 456 | state { 457 | phase: TEST 458 | } 459 | I0219 20:24:27.535907 101920 net.cpp:67] Creating Layer mnist 460 | I0219 20:24:27.571909 101920 net.cpp:356] mnist -> data 461 | I0219 20:24:27.571909 101920 net.cpp:356] mnist -> label 462 | I0219 20:24:27.571909 101920 net.cpp:96] Setting up mnist 463 | I0219 20:24:27.571909 101920 data_layer.cpp:46] Opening leveldb mnist-test-leveldb 464 | I0219 20:24:27.578910 101920 data_layer.cpp:130] output data size: 100,1,28,28 465 | I0219 20:24:27.578910 101920 net.cpp:103] Top shape: 100 1 28 28 (78400) 466 | I0219 20:24:27.578910 101920 net.cpp:103] Top shape: 100 1 1 1 (100) 467 | I0219 20:24:27.578910 101920 net.cpp:67] Creating Layer conv1 468 | I0219 20:24:27.578910 101920 net.cpp:394] conv1 <- data 469 | I0219 20:24:27.578910 101920 net.cpp:356] conv1 -> conv1 470 | I0219 20:24:27.578910 101920 net.cpp:96] Setting up conv1 471 | I0219 20:24:27.578910 101920 net.cpp:103] Top shape: 100 20 24 24 (1152000) 472 | I0219 20:24:27.578910 101920 net.cpp:67] Creating Layer conv1_bn 473 | I0219 20:24:27.578910 101920 net.cpp:394] conv1_bn <- conv1 474 | I0219 20:24:27.578910 101920 net.cpp:356] conv1_bn -> conv1_bn 475 | I0219 20:24:27.578910 101920 net.cpp:96] Setting up conv1_bn 476 | I0219 20:24:27.578910 101920 net.cpp:103] Top shape: 100 20 24 24 (1152000) 477 | I0219 20:24:27.578910 101920 net.cpp:67] Creating Layer pool1 478 | I0219 20:24:27.578910 101920 net.cpp:394] pool1 <- conv1_bn 479 | I0219 20:24:27.578910 101920 net.cpp:356] pool1 -> pool1 480 | I0219 20:24:27.578910 101920 net.cpp:96] Setting up pool1 481 | I0219 20:24:27.578910 101920 net.cpp:103] Top shape: 100 20 12 12 (288000) 482 | I0219 20:24:27.578910 101920 net.cpp:67] Creating Layer conv2 483 | I0219 20:24:27.578910 101920 net.cpp:394] conv2 <- pool1 484 | I0219 20:24:27.578910 101920 net.cpp:356] conv2 -> conv2 485 | I0219 20:24:27.578910 101920 net.cpp:96] Setting up conv2 486 | I0219 20:24:27.579910 101920 net.cpp:103] Top shape: 100 50 8 8 (320000) 487 | I0219 20:24:27.579910 101920 net.cpp:67] Creating Layer conv2_bn 488 | I0219 20:24:27.579910 101920 net.cpp:394] conv2_bn <- conv2 489 | I0219 20:24:27.579910 101920 net.cpp:356] conv2_bn -> conv2_bn 490 | I0219 20:24:27.579910 101920 net.cpp:96] Setting up conv2_bn 491 | I0219 20:24:27.579910 101920 net.cpp:103] Top shape: 100 50 8 8 (320000) 492 | I0219 20:24:27.579910 101920 net.cpp:67] Creating Layer pool2 493 | I0219 20:24:27.579910 101920 net.cpp:394] pool2 <- conv2_bn 494 | I0219 20:24:27.579910 101920 net.cpp:356] pool2 -> pool2 495 | I0219 20:24:27.579910 101920 net.cpp:96] Setting up pool2 496 | I0219 20:24:27.579910 101920 net.cpp:103] Top shape: 100 50 4 4 (80000) 497 | I0219 20:24:27.579910 101920 net.cpp:67] Creating Layer ip1 498 | I0219 20:24:27.579910 101920 net.cpp:394] ip1 <- pool2 499 | I0219 20:24:27.579910 101920 net.cpp:356] ip1 -> ip1 500 | I0219 20:24:27.579910 101920 net.cpp:96] Setting up ip1 501 | I0219 20:24:27.582911 101920 net.cpp:103] Top shape: 100 500 1 1 (50000) 502 | I0219 20:24:27.582911 101920 net.cpp:67] Creating Layer relu1 503 | I0219 20:24:27.582911 101920 net.cpp:394] relu1 <- ip1 504 | I0219 20:24:27.582911 101920 net.cpp:345] relu1 -> ip1 (in-place) 505 | I0219 20:24:27.582911 101920 net.cpp:96] Setting up relu1 506 | I0219 20:24:27.582911 101920 net.cpp:103] Top shape: 100 500 1 1 (50000) 507 | I0219 20:24:27.582911 101920 net.cpp:67] Creating Layer ip2 508 | I0219 20:24:27.582911 101920 net.cpp:394] ip2 <- ip1 509 | I0219 20:24:27.582911 101920 net.cpp:356] ip2 -> ip2 510 | I0219 20:24:27.582911 101920 net.cpp:96] Setting up ip2 511 | I0219 20:24:27.582911 101920 net.cpp:103] Top shape: 100 10 1 1 (1000) 512 | I0219 20:24:27.582911 101920 net.cpp:67] Creating Layer prob 513 | I0219 20:24:27.582911 101920 net.cpp:394] prob <- ip2 514 | I0219 20:24:27.582911 101920 net.cpp:356] prob -> prob 515 | I0219 20:24:27.582911 101920 net.cpp:96] Setting up prob 516 | I0219 20:24:27.582911 101920 net.cpp:103] Top shape: 100 10 1 1 (1000) 517 | I0219 20:24:27.582911 101920 net.cpp:67] Creating Layer accuracy 518 | I0219 20:24:27.582911 101920 net.cpp:394] accuracy <- prob 519 | I0219 20:24:27.582911 101920 net.cpp:394] accuracy <- label 520 | I0219 20:24:27.582911 101920 net.cpp:356] accuracy -> accuracy 521 | I0219 20:24:27.582911 101920 net.cpp:96] Setting up accuracy 522 | I0219 20:24:27.582911 101920 net.cpp:103] Top shape: 1 1 1 1 (1) 523 | I0219 20:24:27.582911 101920 net.cpp:172] accuracy does not need backward computation. 524 | I0219 20:24:27.582911 101920 net.cpp:172] prob does not need backward computation. 525 | I0219 20:24:27.635913 101920 net.cpp:172] ip2 does not need backward computation. 526 | I0219 20:24:27.635913 101920 net.cpp:172] relu1 does not need backward computation. 527 | I0219 20:24:27.635913 101920 net.cpp:172] ip1 does not need backward computation. 528 | I0219 20:24:27.635913 101920 net.cpp:172] pool2 does not need backward computation. 529 | I0219 20:24:27.635913 101920 net.cpp:172] conv2_bn does not need backward computation. 530 | I0219 20:24:27.635913 101920 net.cpp:172] conv2 does not need backward computation. 531 | I0219 20:24:27.635913 101920 net.cpp:172] pool1 does not need backward computation. 532 | I0219 20:24:27.635913 101920 net.cpp:172] conv1_bn does not need backward computation. 533 | I0219 20:24:27.635913 101920 net.cpp:172] conv1 does not need backward computation. 534 | I0219 20:24:27.635913 101920 net.cpp:172] mnist does not need backward computation. 535 | I0219 20:24:27.635913 101920 net.cpp:208] This network produces output accuracy 536 | I0219 20:24:27.635913 101920 net.cpp:467] Collecting Learning Rate and Weight Decay. 537 | I0219 20:24:27.635913 101920 net.cpp:219] Network initialization done. 538 | I0219 20:24:27.635913 101920 net.cpp:220] Memory required for data: 13970004 539 | I0219 20:24:27.635913 101920 solver.cpp:51] Solver scaffolding done. 540 | I0219 20:24:27.635913 101920 solver.cpp:170] Solving LeNet 541 | I0219 20:24:27.635913 101920 solver.cpp:287] Iteration 0, Testing net (#0) 542 | I0219 20:24:29.118999 101920 solver.cpp:338] Test net output #0: accuracy = 0.0916 543 | I0219 20:24:29.138000 101920 solver.cpp:231] Iteration 0, loss = 2.30219 544 | I0219 20:24:29.138000 101920 solver.cpp:442] Iteration 0, lr = 0.01 545 | I0219 20:24:32.218175 101920 solver.cpp:231] Iteration 100, loss = 0.323378 546 | I0219 20:24:32.218175 101920 solver.cpp:442] Iteration 100, lr = 0.00992565 547 | I0219 20:24:35.126341 101920 solver.cpp:231] Iteration 200, loss = 0.23759 548 | I0219 20:24:35.126341 101920 solver.cpp:442] Iteration 200, lr = 0.00985258 549 | I0219 20:24:38.196517 101920 solver.cpp:231] Iteration 300, loss = 0.149019 550 | I0219 20:24:38.196517 101920 solver.cpp:442] Iteration 300, lr = 0.00978075 551 | I0219 20:24:41.343698 101920 solver.cpp:231] Iteration 400, loss = 0.100363 552 | I0219 20:24:41.343698 101920 solver.cpp:442] Iteration 400, lr = 0.00971013 553 | I0219 20:24:44.105855 101920 solver.cpp:287] Iteration 500, Testing net (#0) 554 | I0219 20:24:45.333925 101920 solver.cpp:338] Test net output #0: accuracy = 0.9754 555 | I0219 20:24:45.338927 101920 solver.cpp:231] Iteration 500, loss = 0.0856673 556 | I0219 20:24:45.338927 101920 solver.cpp:442] Iteration 500, lr = 0.00964069 557 | I0219 20:24:48.222090 101920 solver.cpp:231] Iteration 600, loss = 0.137357 558 | I0219 20:24:48.222090 101920 solver.cpp:442] Iteration 600, lr = 0.0095724 559 | I0219 20:24:50.863242 101920 solver.cpp:231] Iteration 700, loss = 0.0835702 560 | I0219 20:24:50.863242 101920 solver.cpp:442] Iteration 700, lr = 0.00950522 561 | I0219 20:24:53.670403 101920 solver.cpp:231] Iteration 800, loss = 0.114954 562 | I0219 20:24:53.670403 101920 solver.cpp:442] Iteration 800, lr = 0.00943913 563 | I0219 20:24:56.469563 101920 solver.cpp:231] Iteration 900, loss = 0.0932496 564 | I0219 20:24:56.470562 101920 solver.cpp:442] Iteration 900, lr = 0.00937411 565 | I0219 20:24:59.068711 101920 solver.cpp:287] Iteration 1000, Testing net (#0) 566 | I0219 20:25:00.195775 101920 solver.cpp:338] Test net output #0: accuracy = 0.9816 567 | I0219 20:25:00.206776 101920 solver.cpp:231] Iteration 1000, loss = 0.0435796 568 | I0219 20:25:00.206776 101920 solver.cpp:442] Iteration 1000, lr = 0.00931012 569 | I0219 20:25:03.094941 101920 solver.cpp:231] Iteration 1100, loss = 0.0376325 570 | I0219 20:25:03.094941 101920 solver.cpp:442] Iteration 1100, lr = 0.00924715 571 | I0219 20:25:05.844099 101920 solver.cpp:231] Iteration 1200, loss = 0.0942016 572 | I0219 20:25:05.844099 101920 solver.cpp:442] Iteration 1200, lr = 0.00918515 573 | I0219 20:25:08.707262 101920 solver.cpp:231] Iteration 1300, loss = 0.0521515 574 | I0219 20:25:08.707262 101920 solver.cpp:442] Iteration 1300, lr = 0.00912412 575 | I0219 20:25:11.303411 101920 solver.cpp:231] Iteration 1400, loss = 0.0771083 576 | I0219 20:25:11.303411 101920 solver.cpp:442] Iteration 1400, lr = 0.00906403 577 | I0219 20:25:13.987565 101920 solver.cpp:287] Iteration 1500, Testing net (#0) 578 | I0219 20:25:15.120630 101920 solver.cpp:338] Test net output #0: accuracy = 0.985 579 | I0219 20:25:15.197633 101920 solver.cpp:231] Iteration 1500, loss = 0.06446 580 | I0219 20:25:15.197633 101920 solver.cpp:442] Iteration 1500, lr = 0.00900485 581 | I0219 20:25:17.791782 101920 solver.cpp:231] Iteration 1600, loss = 0.031064 582 | I0219 20:25:17.791782 101920 solver.cpp:442] Iteration 1600, lr = 0.00894657 583 | I0219 20:25:20.486937 101920 solver.cpp:231] Iteration 1700, loss = 0.0331084 584 | I0219 20:25:20.486937 101920 solver.cpp:442] Iteration 1700, lr = 0.00888916 585 | I0219 20:25:23.185091 101920 solver.cpp:231] Iteration 1800, loss = 0.0699748 586 | I0219 20:25:23.185091 101920 solver.cpp:442] Iteration 1800, lr = 0.0088326 587 | I0219 20:25:25.974251 101920 solver.cpp:231] Iteration 1900, loss = 0.0318849 588 | I0219 20:25:25.974251 101920 solver.cpp:442] Iteration 1900, lr = 0.00877687 589 | I0219 20:25:28.543397 101920 solver.cpp:287] Iteration 2000, Testing net (#0) 590 | I0219 20:25:29.640460 101920 solver.cpp:338] Test net output #0: accuracy = 0.9878 591 | I0219 20:25:29.661461 101920 solver.cpp:231] Iteration 2000, loss = 0.0629812 592 | I0219 20:25:29.661461 101920 solver.cpp:442] Iteration 2000, lr = 0.00872196 593 | I0219 20:25:32.453620 101920 solver.cpp:231] Iteration 2100, loss = 0.0518312 594 | I0219 20:25:32.453620 101920 solver.cpp:442] Iteration 2100, lr = 0.00866784 595 | I0219 20:25:35.086771 101920 solver.cpp:231] Iteration 2200, loss = 0.0227572 596 | I0219 20:25:35.086771 101920 solver.cpp:442] Iteration 2200, lr = 0.0086145 597 | I0219 20:25:37.841929 101920 solver.cpp:231] Iteration 2300, loss = 0.0244286 598 | I0219 20:25:37.841929 101920 solver.cpp:442] Iteration 2300, lr = 0.00856192 599 | I0219 20:25:40.717093 101920 solver.cpp:231] Iteration 2400, loss = 0.0507805 600 | I0219 20:25:40.717093 101920 solver.cpp:442] Iteration 2400, lr = 0.00851008 601 | I0219 20:25:43.349244 101920 solver.cpp:287] Iteration 2500, Testing net (#0) 602 | I0219 20:25:44.497309 101920 solver.cpp:338] Test net output #0: accuracy = 0.9859 603 | I0219 20:25:44.510310 101920 solver.cpp:231] Iteration 2500, loss = 0.0219777 604 | I0219 20:25:44.510310 101920 solver.cpp:442] Iteration 2500, lr = 0.00845897 605 | I0219 20:25:47.213465 101920 solver.cpp:231] Iteration 2600, loss = 0.0499731 606 | I0219 20:25:47.213465 101920 solver.cpp:442] Iteration 2600, lr = 0.00840857 607 | I0219 20:25:49.963623 101920 solver.cpp:231] Iteration 2700, loss = 0.0391577 608 | I0219 20:25:49.963623 101920 solver.cpp:442] Iteration 2700, lr = 0.00835886 609 | I0219 20:25:52.668777 101920 solver.cpp:231] Iteration 2800, loss = 0.0185332 610 | I0219 20:25:52.668777 101920 solver.cpp:442] Iteration 2800, lr = 0.00830984 611 | I0219 20:25:55.485939 101920 solver.cpp:231] Iteration 2900, loss = 0.0167396 612 | I0219 20:25:55.485939 101920 solver.cpp:442] Iteration 2900, lr = 0.00826148 613 | I0219 20:25:58.029083 101920 solver.cpp:287] Iteration 3000, Testing net (#0) 614 | I0219 20:25:59.158149 101920 solver.cpp:338] Test net output #0: accuracy = 0.987 615 | I0219 20:25:59.163148 101920 solver.cpp:231] Iteration 3000, loss = 0.0298753 616 | I0219 20:25:59.163148 101920 solver.cpp:442] Iteration 3000, lr = 0.00821377 617 | I0219 20:26:01.771297 101920 solver.cpp:231] Iteration 3100, loss = 0.0173053 618 | I0219 20:26:01.771297 101920 solver.cpp:442] Iteration 3100, lr = 0.0081667 619 | I0219 20:26:04.541456 101920 solver.cpp:231] Iteration 3200, loss = 0.0369976 620 | I0219 20:26:04.541456 101920 solver.cpp:442] Iteration 3200, lr = 0.00812025 621 | I0219 20:26:07.370617 101920 solver.cpp:231] Iteration 3300, loss = 0.0333898 622 | I0219 20:26:07.371618 101920 solver.cpp:442] Iteration 3300, lr = 0.00807442 623 | I0219 20:26:10.047771 101920 solver.cpp:231] Iteration 3400, loss = 0.015276 624 | I0219 20:26:10.048771 101920 solver.cpp:442] Iteration 3400, lr = 0.00802918 625 | I0219 20:26:12.734925 101920 solver.cpp:287] Iteration 3500, Testing net (#0) 626 | I0219 20:26:13.852988 101920 solver.cpp:338] Test net output #0: accuracy = 0.9898 627 | I0219 20:26:13.856988 101920 solver.cpp:231] Iteration 3500, loss = 0.00888279 628 | I0219 20:26:13.856988 101920 solver.cpp:442] Iteration 3500, lr = 0.00798454 629 | I0219 20:26:16.533143 101920 solver.cpp:231] Iteration 3600, loss = 0.0153337 630 | I0219 20:26:16.533143 101920 solver.cpp:442] Iteration 3600, lr = 0.00794046 631 | I0219 20:26:19.231297 101920 solver.cpp:231] Iteration 3700, loss = 0.0163696 632 | I0219 20:26:19.231297 101920 solver.cpp:442] Iteration 3700, lr = 0.00789695 633 | I0219 20:26:21.892448 101920 solver.cpp:231] Iteration 3800, loss = 0.0274265 634 | I0219 20:26:21.892448 101920 solver.cpp:442] Iteration 3800, lr = 0.007854 635 | I0219 20:26:24.654606 101920 solver.cpp:231] Iteration 3900, loss = 0.0280846 636 | I0219 20:26:24.654606 101920 solver.cpp:442] Iteration 3900, lr = 0.00781158 637 | I0219 20:26:27.394763 101920 solver.cpp:287] Iteration 4000, Testing net (#0) 638 | I0219 20:26:28.608832 101920 solver.cpp:338] Test net output #0: accuracy = 0.9892 639 | I0219 20:26:28.620833 101920 solver.cpp:231] Iteration 4000, loss = 0.0119719 640 | I0219 20:26:28.620833 101920 solver.cpp:442] Iteration 4000, lr = 0.00776969 641 | I0219 20:26:31.207981 101920 solver.cpp:231] Iteration 4100, loss = 0.00557243 642 | I0219 20:26:31.207981 101920 solver.cpp:442] Iteration 4100, lr = 0.00772833 643 | I0219 20:26:33.999141 101920 solver.cpp:231] Iteration 4200, loss = 0.00751942 644 | I0219 20:26:33.999141 101920 solver.cpp:442] Iteration 4200, lr = 0.00768748 645 | I0219 20:26:36.629292 101920 solver.cpp:231] Iteration 4300, loss = 0.0159704 646 | I0219 20:26:36.629292 101920 solver.cpp:442] Iteration 4300, lr = 0.00764712 647 | I0219 20:26:39.398449 101920 solver.cpp:231] Iteration 4400, loss = 0.0195673 648 | I0219 20:26:39.398449 101920 solver.cpp:442] Iteration 4400, lr = 0.00760726 649 | I0219 20:26:42.017599 101920 solver.cpp:287] Iteration 4500, Testing net (#0) 650 | I0219 20:26:43.123662 101920 solver.cpp:338] Test net output #0: accuracy = 0.9899 651 | I0219 20:26:43.137663 101920 solver.cpp:231] Iteration 4500, loss = 0.0226567 652 | I0219 20:26:43.137663 101920 solver.cpp:442] Iteration 4500, lr = 0.00756788 653 | I0219 20:26:45.888821 101920 solver.cpp:231] Iteration 4600, loss = 0.00898327 654 | I0219 20:26:45.888821 101920 solver.cpp:442] Iteration 4600, lr = 0.00752897 655 | I0219 20:26:48.785986 101920 solver.cpp:231] Iteration 4700, loss = 0.00437807 656 | I0219 20:26:48.786986 101920 solver.cpp:442] Iteration 4700, lr = 0.00749052 657 | I0219 20:26:51.433138 101920 solver.cpp:231] Iteration 4800, loss = 0.00555685 658 | I0219 20:26:51.433138 101920 solver.cpp:442] Iteration 4800, lr = 0.00745253 659 | I0219 20:26:54.230298 101920 solver.cpp:231] Iteration 4900, loss = 0.0127988 660 | I0219 20:26:54.230298 101920 solver.cpp:442] Iteration 4900, lr = 0.00741498 661 | I0219 20:26:56.887450 101920 solver.cpp:356] Snapshotting to lenet_iter_5000.caffemodel 662 | I0219 20:26:56.895450 101920 solver.cpp:363] Snapshotting solver state to lenet_iter_5000.caffemodel.solverstate 663 | I0219 20:26:56.899451 101920 solver.cpp:287] Iteration 5000, Testing net (#0) 664 | I0219 20:26:58.003515 101920 solver.cpp:338] Test net output #0: accuracy = 0.9908 665 | I0219 20:26:58.013514 101920 solver.cpp:231] Iteration 5000, loss = 0.015663 666 | I0219 20:26:58.013514 101920 solver.cpp:442] Iteration 5000, lr = 0.00737788 667 | I0219 20:27:00.788673 101920 solver.cpp:231] Iteration 5100, loss = 0.0169028 668 | I0219 20:27:00.788673 101920 solver.cpp:442] Iteration 5100, lr = 0.0073412 669 | I0219 20:27:03.455826 101920 solver.cpp:231] Iteration 5200, loss = 0.00675082 670 | I0219 20:27:03.455826 101920 solver.cpp:442] Iteration 5200, lr = 0.00730495 671 | I0219 20:27:06.232985 101920 solver.cpp:231] Iteration 5300, loss = 0.00372812 672 | I0219 20:27:06.232985 101920 solver.cpp:442] Iteration 5300, lr = 0.00726911 673 | I0219 20:27:08.875135 101920 solver.cpp:231] Iteration 5400, loss = 0.00410843 674 | I0219 20:27:08.875135 101920 solver.cpp:442] Iteration 5400, lr = 0.00723368 675 | I0219 20:27:11.657294 101920 solver.cpp:287] Iteration 5500, Testing net (#0) 676 | I0219 20:27:12.787359 101920 solver.cpp:338] Test net output #0: accuracy = 0.9887 677 | I0219 20:27:12.806360 101920 solver.cpp:231] Iteration 5500, loss = 0.0117428 678 | I0219 20:27:12.807360 101920 solver.cpp:442] Iteration 5500, lr = 0.00719865 679 | I0219 20:27:15.413509 101920 solver.cpp:231] Iteration 5600, loss = 0.0132238 680 | I0219 20:27:15.413509 101920 solver.cpp:442] Iteration 5600, lr = 0.00716402 681 | I0219 20:27:18.219671 101920 solver.cpp:231] Iteration 5700, loss = 0.0137579 682 | I0219 20:27:18.219671 101920 solver.cpp:442] Iteration 5700, lr = 0.00712977 683 | I0219 20:27:20.808818 101920 solver.cpp:231] Iteration 5800, loss = 0.00497267 684 | I0219 20:27:20.808818 101920 solver.cpp:442] Iteration 5800, lr = 0.0070959 685 | I0219 20:27:23.590977 101920 solver.cpp:231] Iteration 5900, loss = 0.00294243 686 | I0219 20:27:23.590977 101920 solver.cpp:442] Iteration 5900, lr = 0.0070624 687 | I0219 20:27:26.161124 101920 solver.cpp:287] Iteration 6000, Testing net (#0) 688 | I0219 20:27:27.281188 101920 solver.cpp:338] Test net output #0: accuracy = 0.9897 689 | I0219 20:27:27.287189 101920 solver.cpp:231] Iteration 6000, loss = 0.00346468 690 | I0219 20:27:27.287189 101920 solver.cpp:442] Iteration 6000, lr = 0.00702927 691 | I0219 20:27:30.089349 101920 solver.cpp:231] Iteration 6100, loss = 0.0104201 692 | I0219 20:27:30.089349 101920 solver.cpp:442] Iteration 6100, lr = 0.0069965 693 | I0219 20:27:32.643496 101920 solver.cpp:231] Iteration 6200, loss = 0.0109253 694 | I0219 20:27:32.643496 101920 solver.cpp:442] Iteration 6200, lr = 0.00696408 695 | I0219 20:27:35.436655 101920 solver.cpp:231] Iteration 6300, loss = 0.0113819 696 | I0219 20:27:35.436655 101920 solver.cpp:442] Iteration 6300, lr = 0.00693201 697 | I0219 20:27:38.090806 101920 solver.cpp:231] Iteration 6400, loss = 0.00391983 698 | I0219 20:27:38.090806 101920 solver.cpp:442] Iteration 6400, lr = 0.00690029 699 | I0219 20:27:40.816962 101920 solver.cpp:287] Iteration 6500, Testing net (#0) 700 | I0219 20:27:41.913025 101920 solver.cpp:338] Test net output #0: accuracy = 0.9903 701 | I0219 20:27:41.925026 101920 solver.cpp:231] Iteration 6500, loss = 0.0024418 702 | I0219 20:27:41.925026 101920 solver.cpp:442] Iteration 6500, lr = 0.0068689 703 | I0219 20:27:44.510174 101920 solver.cpp:231] Iteration 6600, loss = 0.00319171 704 | I0219 20:27:44.510174 101920 solver.cpp:442] Iteration 6600, lr = 0.00683784 705 | I0219 20:27:47.298333 101920 solver.cpp:231] Iteration 6700, loss = 0.00863161 706 | I0219 20:27:47.298333 101920 solver.cpp:442] Iteration 6700, lr = 0.00680711 707 | I0219 20:27:49.879482 101920 solver.cpp:231] Iteration 6800, loss = 0.00959494 708 | I0219 20:27:49.879482 101920 solver.cpp:442] Iteration 6800, lr = 0.0067767 709 | I0219 20:27:52.699642 101920 solver.cpp:231] Iteration 6900, loss = 0.0102416 710 | I0219 20:27:52.699642 101920 solver.cpp:442] Iteration 6900, lr = 0.0067466 711 | I0219 20:27:55.473801 101920 solver.cpp:287] Iteration 7000, Testing net (#0) 712 | I0219 20:27:56.587864 101920 solver.cpp:338] Test net output #0: accuracy = 0.9914 713 | I0219 20:27:56.663869 101920 solver.cpp:231] Iteration 7000, loss = 0.0030493 714 | I0219 20:27:56.663869 101920 solver.cpp:442] Iteration 7000, lr = 0.00671681 715 | I0219 20:27:59.254017 101920 solver.cpp:231] Iteration 7100, loss = 0.00213348 716 | I0219 20:27:59.254017 101920 solver.cpp:442] Iteration 7100, lr = 0.00668733 717 | I0219 20:28:02.059177 101920 solver.cpp:231] Iteration 7200, loss = 0.00286794 718 | I0219 20:28:02.059177 101920 solver.cpp:442] Iteration 7200, lr = 0.00665815 719 | I0219 20:28:04.666326 101920 solver.cpp:231] Iteration 7300, loss = 0.00784281 720 | I0219 20:28:04.666326 101920 solver.cpp:442] Iteration 7300, lr = 0.00662927 721 | I0219 20:28:07.445485 101920 solver.cpp:231] Iteration 7400, loss = 0.00841386 722 | I0219 20:28:07.445485 101920 solver.cpp:442] Iteration 7400, lr = 0.00660067 723 | I0219 20:28:09.988631 101920 solver.cpp:287] Iteration 7500, Testing net (#0) 724 | I0219 20:28:11.104696 101920 solver.cpp:338] Test net output #0: accuracy = 0.991 725 | I0219 20:28:11.110695 101920 solver.cpp:231] Iteration 7500, loss = 0.00848323 726 | I0219 20:28:11.110695 101920 solver.cpp:442] Iteration 7500, lr = 0.00657236 727 | I0219 20:28:13.898854 101920 solver.cpp:231] Iteration 7600, loss = 0.00240813 728 | I0219 20:28:13.898854 101920 solver.cpp:442] Iteration 7600, lr = 0.00654433 729 | I0219 20:28:16.455001 101920 solver.cpp:231] Iteration 7700, loss = 0.00191419 730 | I0219 20:28:16.455001 101920 solver.cpp:442] Iteration 7700, lr = 0.00651658 731 | I0219 20:28:19.070150 101920 solver.cpp:231] Iteration 7800, loss = 0.00295593 732 | I0219 20:28:19.070150 101920 solver.cpp:442] Iteration 7800, lr = 0.00648911 733 | I0219 20:28:21.787307 101920 solver.cpp:231] Iteration 7900, loss = 0.00673551 734 | I0219 20:28:21.787307 101920 solver.cpp:442] Iteration 7900, lr = 0.0064619 735 | I0219 20:28:24.451458 101920 solver.cpp:287] Iteration 8000, Testing net (#0) 736 | I0219 20:28:25.621526 101920 solver.cpp:338] Test net output #0: accuracy = 0.9913 737 | I0219 20:28:25.665527 101920 solver.cpp:231] Iteration 8000, loss = 0.00720231 738 | I0219 20:28:25.665527 101920 solver.cpp:442] Iteration 8000, lr = 0.00643496 739 | I0219 20:28:28.343682 101920 solver.cpp:231] Iteration 8100, loss = 0.00756929 740 | I0219 20:28:28.343682 101920 solver.cpp:442] Iteration 8100, lr = 0.00640827 741 | I0219 20:28:31.128840 101920 solver.cpp:231] Iteration 8200, loss = 0.00201314 742 | I0219 20:28:31.128840 101920 solver.cpp:442] Iteration 8200, lr = 0.00638185 743 | I0219 20:28:33.700988 101920 solver.cpp:231] Iteration 8300, loss = 0.00172469 744 | I0219 20:28:33.700988 101920 solver.cpp:442] Iteration 8300, lr = 0.00635568 745 | I0219 20:28:36.505147 101920 solver.cpp:231] Iteration 8400, loss = 0.00279293 746 | I0219 20:28:36.505147 101920 solver.cpp:442] Iteration 8400, lr = 0.00632975 747 | I0219 20:28:39.076295 101920 solver.cpp:287] Iteration 8500, Testing net (#0) 748 | I0219 20:28:40.196359 101920 solver.cpp:338] Test net output #0: accuracy = 0.9904 749 | I0219 20:28:40.201359 101920 solver.cpp:231] Iteration 8500, loss = 0.00631739 750 | I0219 20:28:40.201359 101920 solver.cpp:442] Iteration 8500, lr = 0.00630407 751 | I0219 20:28:42.982518 101920 solver.cpp:231] Iteration 8600, loss = 0.00666775 752 | I0219 20:28:42.982518 101920 solver.cpp:442] Iteration 8600, lr = 0.00627864 753 | I0219 20:28:45.588667 101920 solver.cpp:231] Iteration 8700, loss = 0.00658094 754 | I0219 20:28:45.588667 101920 solver.cpp:442] Iteration 8700, lr = 0.00625344 755 | I0219 20:28:48.405828 101920 solver.cpp:231] Iteration 8800, loss = 0.00169822 756 | I0219 20:28:48.406828 101920 solver.cpp:442] Iteration 8800, lr = 0.00622847 757 | I0219 20:28:51.037979 101920 solver.cpp:231] Iteration 8900, loss = 0.00170645 758 | I0219 20:28:51.037979 101920 solver.cpp:442] Iteration 8900, lr = 0.00620374 759 | I0219 20:28:53.760134 101920 solver.cpp:287] Iteration 9000, Testing net (#0) 760 | I0219 20:28:54.887199 101920 solver.cpp:338] Test net output #0: accuracy = 0.9902 761 | I0219 20:28:54.894199 101920 solver.cpp:231] Iteration 9000, loss = 0.00284398 762 | I0219 20:28:54.894199 101920 solver.cpp:442] Iteration 9000, lr = 0.00617924 763 | I0219 20:28:57.467346 101920 solver.cpp:231] Iteration 9100, loss = 0.00578002 764 | I0219 20:28:57.467346 101920 solver.cpp:442] Iteration 9100, lr = 0.00615496 765 | I0219 20:29:00.250506 101920 solver.cpp:231] Iteration 9200, loss = 0.00624887 766 | I0219 20:29:00.250506 101920 solver.cpp:442] Iteration 9200, lr = 0.0061309 767 | I0219 20:29:02.771651 101920 solver.cpp:231] Iteration 9300, loss = 0.00631996 768 | I0219 20:29:02.771651 101920 solver.cpp:442] Iteration 9300, lr = 0.00610706 769 | I0219 20:29:05.428802 101920 solver.cpp:231] Iteration 9400, loss = 0.00152145 770 | I0219 20:29:05.428802 101920 solver.cpp:442] Iteration 9400, lr = 0.00608343 771 | I0219 20:29:08.158958 101920 solver.cpp:287] Iteration 9500, Testing net (#0) 772 | I0219 20:29:09.268021 101920 solver.cpp:338] Test net output #0: accuracy = 0.9905 773 | I0219 20:29:09.274022 101920 solver.cpp:231] Iteration 9500, loss = 0.00170842 774 | I0219 20:29:09.274022 101920 solver.cpp:442] Iteration 9500, lr = 0.00606002 775 | I0219 20:29:11.904172 101920 solver.cpp:231] Iteration 9600, loss = 0.0025672 776 | I0219 20:29:11.904172 101920 solver.cpp:442] Iteration 9600, lr = 0.00603682 777 | I0219 20:29:14.641330 101920 solver.cpp:231] Iteration 9700, loss = 0.0057964 778 | I0219 20:29:14.641330 101920 solver.cpp:442] Iteration 9700, lr = 0.00601382 779 | I0219 20:29:17.284481 101920 solver.cpp:231] Iteration 9800, loss = 0.00612196 780 | I0219 20:29:17.284481 101920 solver.cpp:442] Iteration 9800, lr = 0.00599102 781 | I0219 20:29:20.012636 101920 solver.cpp:231] Iteration 9900, loss = 0.00597476 782 | I0219 20:29:20.012636 101920 solver.cpp:442] Iteration 9900, lr = 0.00596843 783 | I0219 20:29:22.656787 101920 solver.cpp:356] Snapshotting to lenet_iter_10000.caffemodel 784 | I0219 20:29:22.663789 101920 solver.cpp:363] Snapshotting solver state to lenet_iter_10000.caffemodel.solverstate 785 | I0219 20:29:22.673789 101920 solver.cpp:268] Iteration 10000, loss = 0.00146509 786 | I0219 20:29:22.673789 101920 solver.cpp:287] Iteration 10000, Testing net (#0) 787 | I0219 20:29:23.761850 101920 solver.cpp:338] Test net output #0: accuracy = 0.9909 788 | I0219 20:29:23.761850 101920 solver.cpp:273] Optimization Done. 789 | I0219 20:29:23.761850 101920 caffe.cpp:130] Optimization Done. 790 | -------------------------------------------------------------------------------- /lenet_BN_sgd_solver.prototxt: -------------------------------------------------------------------------------- 1 | # random seed to ensure reproducible results (seems don't effect GPU...) 2 | random_seed: 1234 3 | 4 | # model file 5 | net: "lenet_BN_train_valid.prototxt" 6 | 7 | # test_iter specifies how many forward passes the test should carry out. 8 | # In the case of MNIST, we have test batch size 100 and 100 test iterations, 9 | # covering the full 10,000 testing images. 10 | test_iter: 100 11 | # Carry out testing every 500 training iterations. 12 | test_interval: 500 13 | 14 | # Display every 100 iterations 15 | display: 100 16 | # The maximum number of iterations 17 | max_iter: 10000 18 | # snapshot intermediate results 19 | snapshot: 5000 20 | snapshot_prefix: "lenet" 21 | # solver mode: CPU or GPU 22 | solver_mode: GPU 23 | 24 | # solver type (NOTE: only SGD and NESTEROV support accumulate gradients) 25 | solver_type: SGD 26 | #solver_type: NESTEROV 27 | #solver_type: ADAGRAD 28 | #solver_type: ADADELTA 29 | 30 | # for ADADELTA to avoid dividing by zero 31 | delta: 1e-8 32 | 33 | # learning rate policy 34 | base_lr: 0.01 35 | momentum: 0.9 36 | # The learning rate policy 37 | lr_policy: "inv" 38 | gamma: 0.0001 39 | power: 0.75 40 | 41 | # weight decay 42 | weight_decay: 0.0005 -------------------------------------------------------------------------------- /lenet_BN_train_valid.prototxt: -------------------------------------------------------------------------------- 1 | name: "LeNet" 2 | # Training set 3 | layers { 4 | name: "mnist" 5 | type: DATA 6 | top: "data" 7 | top: "label" 8 | data_param { 9 | source: "mnist-train-leveldb" 10 | scale: 0.00390625 11 | batch_size: 100 12 | } 13 | include: { phase: TRAIN } 14 | } 15 | # Validation set 16 | layers { 17 | name: "mnist" 18 | type: DATA 19 | top: "data" 20 | top: "label" 21 | data_param { 22 | source: "mnist-test-leveldb" 23 | scale: 0.00390625 24 | batch_size: 100 25 | } 26 | include: { phase: TEST } 27 | } 28 | layers { 29 | name: "conv1" 30 | type: CONVOLUTION 31 | bottom: "data" 32 | top: "conv1" 33 | blobs_lr: 1 34 | blobs_lr: 2 35 | convolution_param { 36 | num_output: 20 37 | kernel_size: 5 38 | stride: 1 39 | weight_filler { 40 | type: "xavier" 41 | } 42 | bias_filler { 43 | type: "constant" 44 | } 45 | } 46 | } 47 | ## BN 48 | layers { 49 | bottom: "conv1" 50 | top: "conv1_bn" 51 | name: "conv1_bn" 52 | type: BN 53 | blobs_lr: 1 54 | blobs_lr: 1 55 | weight_decay: 0 56 | weight_decay: 0 57 | bn_param { 58 | scale_filler { 59 | type: "constant" 60 | value: 1 61 | } 62 | shift_filler { 63 | type: "constant" 64 | value: 0 65 | } 66 | } 67 | } 68 | layers { 69 | name: "pool1" 70 | type: POOLING 71 | bottom: "conv1_bn" 72 | top: "pool1" 73 | pooling_param { 74 | pool: MAX 75 | kernel_size: 2 76 | stride: 2 77 | } 78 | } 79 | layers { 80 | name: "conv2" 81 | type: CONVOLUTION 82 | bottom: "pool1" 83 | top: "conv2" 84 | blobs_lr: 1 85 | blobs_lr: 2 86 | convolution_param { 87 | num_output: 50 88 | kernel_size: 5 89 | stride: 1 90 | weight_filler { 91 | type: "xavier" 92 | } 93 | bias_filler { 94 | type: "constant" 95 | } 96 | } 97 | } 98 | ## BN 99 | layers { 100 | bottom: "conv2" 101 | top: "conv2_bn" 102 | name: "conv2_bn" 103 | type: BN 104 | blobs_lr: 1 105 | blobs_lr: 1 106 | weight_decay: 0 107 | weight_decay: 0 108 | bn_param { 109 | scale_filler { 110 | type: "constant" 111 | value: 1 112 | } 113 | shift_filler { 114 | type: "constant" 115 | value: 0 116 | } 117 | } 118 | } 119 | layers { 120 | name: "pool2" 121 | type: POOLING 122 | bottom: "conv2_bn" 123 | top: "pool2" 124 | pooling_param { 125 | pool: MAX 126 | kernel_size: 2 127 | stride: 2 128 | } 129 | } 130 | layers { 131 | name: "ip1" 132 | type: INNER_PRODUCT 133 | bottom: "pool2" 134 | top: "ip1" 135 | blobs_lr: 1 136 | blobs_lr: 2 137 | inner_product_param { 138 | num_output: 500 139 | weight_filler { 140 | type: "xavier" 141 | } 142 | bias_filler { 143 | type: "constant" 144 | } 145 | } 146 | } 147 | layers { 148 | name: "relu1" 149 | type: RELU 150 | bottom: "ip1" 151 | top: "ip1" 152 | } 153 | layers { 154 | name: "ip2" 155 | type: INNER_PRODUCT 156 | bottom: "ip1" 157 | top: "ip2" 158 | blobs_lr: 1 159 | blobs_lr: 2 160 | inner_product_param { 161 | num_output: 10 162 | weight_filler { 163 | type: "xavier" 164 | } 165 | bias_filler { 166 | type: "constant" 167 | } 168 | } 169 | } 170 | layers { 171 | name: "loss" 172 | type: SOFTMAX_LOSS 173 | bottom: "ip2" 174 | bottom: "label" 175 | include: { phase: TRAIN } 176 | } 177 | layers { 178 | name: "prob" 179 | type: SOFTMAX 180 | bottom: "ip2" 181 | top: "prob" 182 | include: { phase: TEST } 183 | } 184 | layers { 185 | name: "accuracy" 186 | type: ACCURACY 187 | bottom: "prob" 188 | bottom: "label" 189 | top: "accuracy" 190 | include: { phase: TEST } 191 | } 192 | --------------------------------------------------------------------------------