├── .gitignore ├── matconvnet-calvin ├── examples │ ├── e2s2 │ │ ├── .gitignore │ │ ├── README.md │ │ ├── e2s2_flipImageBoxes.m │ │ ├── e2s2_prepareImage.m │ │ ├── e2s2_storeSPGTOverlap.m │ │ └── e2s2_storeBlobMasks.m │ ├── fcn │ │ ├── README.md │ │ ├── pascalColors.m │ │ ├── fcnInitializeModel16sGeneric.m │ │ ├── fcnInitializeModel8sGeneric.m │ │ └── fcnTestGeneric.m │ └── frcn │ │ ├── README.md │ │ ├── VOC_modified │ │ ├── VOChash_lookup_modified.m │ │ └── VOChash_init_modified.m │ │ ├── boxes │ │ ├── BoxUnion.m │ │ ├── BoxRemoveDuplicates.m │ │ ├── BoxSize.m │ │ ├── BoxBestOverlapFastRcnn.m │ │ ├── BoxIntersection.m │ │ ├── BoxPascalOverlap.m │ │ ├── BoxRegresss.m │ │ ├── BoxRegresssGirshick.m │ │ ├── BoxRegresssLog.m │ │ ├── BoxOverlap.m │ │ ├── BoxNMS.m │ │ ├── BoxRegressionTarget.m │ │ ├── BoxRegressionTargetLog.m │ │ ├── BoxRegressionTargetGirshick.m │ │ └── selective_search_boxes.m │ │ ├── testClassification.m │ │ ├── misc │ │ ├── SelectIdleGpu.m │ │ └── Cell2Matrix.m │ │ ├── evalClassification.m │ │ ├── setupDataOpts.m │ │ ├── setupImdbDetection.m │ │ ├── DetectionToPascalVOCFiles.m │ │ ├── calvinNNDetection.m │ │ ├── calvinNNClassification.m │ │ ├── setupImdbClassification.m │ │ ├── GetGTAndSSBoxes.m │ │ ├── testDetection.m │ │ └── evalDetection.m ├── matlab │ ├── misc │ │ ├── varByteSize.m │ │ ├── cellRemoveEmptyEntries.m │ │ ├── flattenCellArray.m │ │ ├── appendNotEmpty.m │ │ ├── prependNotEmpty.m │ │ ├── invertPermutation.m │ │ ├── strEndsWith.m │ │ ├── matFileHasField.m │ │ ├── strStartsWith.m │ │ ├── sub2indFast.m │ │ ├── structOverwriteFields.m │ │ ├── struct2Varargin.m │ │ ├── isfieldRec.m │ │ ├── findIndexPermutation.m │ │ ├── rpGetFunc.m │ │ ├── filePathRemoveFile.m │ │ ├── rpAddPath.m │ │ ├── maskToBlob.m │ │ ├── rpGroundTruth.m │ │ ├── printProgress.m │ │ ├── varargToStr.m │ │ ├── freqClampMinimum.m │ │ ├── blobToImageSubs.m │ │ ├── evaluatePixAccHierarchy_loop.m │ │ ├── blobToImageInds.m │ │ ├── readLinesToCell.m │ │ ├── imageInsertText.m │ │ ├── computeBlobOverlapSum.m │ │ ├── indicesOfAInB.m │ │ ├── groundTruthExtractRegions.m │ │ ├── labelMapToColorImage.m │ │ ├── rpGetProposalsVars.m │ │ ├── AnnotationMC.m │ │ ├── computeBlobOverlapAny.m │ │ ├── evaluatePixAccHierarchy_preload.m │ │ ├── rpFelzenszwalb2004.m │ │ ├── evaluatePixAcc.m │ │ ├── computeBlobOverlapAnyPair.cpp │ │ ├── blobProbsToOutputMap.m │ │ ├── evaluatePixAccHierarchy.m │ │ ├── isEqualFuncs.m │ │ ├── scoreBlobIoUs.m │ │ ├── storeLabelListGT.m │ │ ├── rpUijlings2013.m │ │ ├── evaluateMeanIOU.m │ │ └── evaluatePixAccPaint.m │ ├── calvin_root.m │ ├── @CalvinNN │ │ ├── extractStats.m │ │ ├── loadState.m │ │ ├── findLastCheckpoint.m │ │ ├── saveState.m │ │ ├── accumulateStats.m │ │ ├── accumulateGradients.m │ │ ├── test.m │ │ ├── init.m │ │ ├── loadNetwork.m │ │ ├── CalvinNN.m │ │ ├── train.m │ │ ├── convertNetwork.m │ │ └── plotStats.m │ ├── dagnn_calvin │ │ ├── sortLayers.m │ │ ├── replaceVariables.m │ │ ├── getFreeVariable.m │ │ ├── replaceLayer.m │ │ └── insertLayer.m │ ├── confMatToAccuracies.m │ ├── roipool │ │ ├── roiPooling_freeform_backward.m │ │ └── roiPooling_freeform_forward.m │ ├── compatibility │ │ └── isScalar.h │ ├── vl_compilenn_calvin.m │ ├── setup │ │ ├── downloadNetwork.m │ │ ├── downloadSelectiveSearch.m │ │ ├── downloadSiftFlow.m │ │ ├── downloadModel.m │ │ ├── downloadVOC2010.m │ │ ├── exportModel.m │ │ ├── setupFastRcnnRegions.m │ │ └── setupE2S2Regions.m │ ├── +dagnn │ │ ├── RoiPoolingFreeform.m │ │ ├── SegmentationAccuracyFlexible.m │ │ ├── LabelPresence.m │ │ ├── GeneralSigmoid.m │ │ ├── SegmentationLossSemiSupervised.m │ │ └── LossWeighted.m │ ├── regiontopixel │ │ ├── regionToPixel_forward.m │ │ ├── regionToPixelSoft_forward.m │ │ ├── regionToPixel_backward.cpp │ │ └── regionToPixelSoft_backward.cpp │ ├── imdb │ │ └── ImdbCalvin.m │ ├── segmentationLossImage_extractMaxScores.m │ ├── labelpresence │ │ └── labelPresence_backward.cpp │ └── vl_nnloss_regress.m └── .gitignore ├── .gitmodules ├── demo_frcn.m ├── setup.m ├── demo_fcn.m ├── demo_e2s2_full.m ├── demo_e2s2_fast.m └── LICENSE.md /.gitignore: -------------------------------------------------------------------------------- 1 | /data 2 | *.m~ 3 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/e2s2/.gitignore: -------------------------------------------------------------------------------- 1 | /Experiments 2 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/e2s2/README.md: -------------------------------------------------------------------------------- 1 | Code used in the paper "Region-based semantic segmentation with end-to-end training" by Caesar et al., ECCV 2016. 2 | 3 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/fcn/README.md: -------------------------------------------------------------------------------- 1 | Code similar to the one used in the paper "Fully Convolutional Networks for Semantic Segmentation" by Long et al., CVPR 2015. 2 | 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "matconvnet"] 2 | path = matconvnet 3 | url = https://github.com/vlfeat/matconvnet.git 4 | [submodule "matconvnet-fcn"] 5 | path = matconvnet-fcn 6 | url = https://github.com/vlfeat/matconvnet-fcn.git 7 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/varByteSize.m: -------------------------------------------------------------------------------- 1 | function[size] = varByteSize(var) %#ok 2 | % [size] = varByteSize(var) 3 | % 4 | % Returns the size of a variable in bytes. 5 | % 6 | % Copyright by Holger Caesar, 2014 7 | 8 | s = whos('var'); 9 | size = s.bytes; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/calvin_root.m: -------------------------------------------------------------------------------- 1 | function[root] = calvin_root() 2 | % [root] = calvin_root() 3 | % 4 | % Returns the absolute path to the root directory of matconvnet-calvin. 5 | % 6 | % Copyright by Holger Caesar, 2016 7 | 8 | root = fileparts(fileparts(fileparts(mfilename('fullpath')))); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/cellRemoveEmptyEntries.m: -------------------------------------------------------------------------------- 1 | function[c] = cellRemoveEmptyEntries(c) 2 | % [c] = cellRemoveEmptyEntries(c) 3 | % 4 | % Remove empty entries of a cell. The output is the resulting cell vector. 5 | % 6 | % Copyright by Holger Caesar, 2014 7 | 8 | c = c(cellfun(@(x) ~isempty(x), c)); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/flattenCellArray.m: -------------------------------------------------------------------------------- 1 | function[cellArray] = flattenCellArray(cellArray) 2 | % [cellArray] = flattenCellArray(cellArray) 3 | % 4 | % Flattens a cell array of cell arrays into a simple cell array. 5 | % 6 | % Copyright by Holger Caesar, 2014 7 | 8 | cellArray = cat(1, cellArray{:}); -------------------------------------------------------------------------------- /matconvnet-calvin/.gitignore: -------------------------------------------------------------------------------- 1 | *.xcodeproj/*xcuserdata* 2 | *.xcodeproj/project.xcworkspace/*xcuserdata* 3 | *.xcodeproj/project.xcworkspace/xcshareddata/ 4 | mex/* 5 | mex 6 | data 7 | *.o 8 | *.pyc 9 | *~ 10 | index.html 11 | local 12 | 13 | # Visual C 14 | *.suo 15 | *.user 16 | *.sdf 17 | *.opensdf 18 | 19 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/appendNotEmpty.m: -------------------------------------------------------------------------------- 1 | function[res] = appendNotEmpty(str, suffix) 2 | % [res] = appendNotEmpty(str, suffix) 3 | % 4 | % Append a suffix to a string, if it is not empty. 5 | % 6 | % Copyrights by Holger Caesar, 2015 7 | 8 | if ~isempty(str), 9 | res = [str, suffix]; 10 | else 11 | res = ''; 12 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/README.md: -------------------------------------------------------------------------------- 1 | This is the classification/detection example on PASCAL VOC. 2 | This requires the PASCAL VOC devkit and selective search. 3 | For more info see: 4 | https://github.com/npinto/VOCdevkit 5 | http://homepages.inf.ed.ac.uk/juijling/index.php 6 | 7 | Copyright by Holger Caesar and Jasper Uijlings, 2016 8 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/prependNotEmpty.m: -------------------------------------------------------------------------------- 1 | function[res] = prependNotEmpty(str, prefix) 2 | % [res] = prependNotEmpty(str, prefix) 3 | % 4 | % Prepend a prefix to a string, if it is not empty. 5 | % 6 | % Copyrights by Holger Caesar, 2015 7 | 8 | if ~isempty(str), 9 | res = [prefix, str]; 10 | else 11 | res = ''; 12 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/VOC_modified/VOChash_lookup_modified.m: -------------------------------------------------------------------------------- 1 | function ind = VOChash_lookup_modified(hash,s) 2 | 3 | hsize=numel(hash.key); 4 | if length(s) == 6 5 | h=mod(str2double(s),hsize)+1; 6 | else 7 | h=mod(str2double(s([3:4 6:11 13:end])),hsize)+1; 8 | end 9 | ind=hash.val{h}(strmatch(s,hash.key{h},'exact')); 10 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/extractStats.m: -------------------------------------------------------------------------------- 1 | function stats = extractStats(net, ~) 2 | % stats = extractStats(net) 3 | % 4 | % Extract all losses from the network. 5 | 6 | sel = find(cellfun(@(x) isa(x, 'dagnn.Loss'), {net.layers.block})); 7 | stats = struct(); 8 | for i = 1:numel(sel) 9 | stats.(net.layers(sel(i)).outputs{1}) = net.layers(sel(i)).block.average; 10 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/loadState.m: -------------------------------------------------------------------------------- 1 | function[net, stats] = loadState(fileName) 2 | % [net, stats] = loadState(fileName) 3 | % 4 | % Loads a training snapshot of the network. 5 | % 6 | % Copyright by Matconvnet 7 | % Modified by Holger Caesar, 2015 8 | 9 | netStruct = load(fileName, 'net', 'stats'); 10 | net = dagnn.DagNN.loadobj(netStruct.net); 11 | stats = netStruct.stats; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/invertPermutation.m: -------------------------------------------------------------------------------- 1 | function[invPerm] = invertPermutation(perm) 2 | % [invPerm] = invertPermutation(perm) 3 | % 4 | % Invert a given permutation such that the following statement is true for row vectors: 5 | % isequal(perm(invertPermutation(perm)), 1:numel(perm)) 6 | % 7 | % Copyright by Holger Caesar, 2014 8 | 9 | invPerm = findIndexPermutation(perm, 1:numel(perm)); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/strEndsWith.m: -------------------------------------------------------------------------------- 1 | function[result] = strEndsWith(str, prefix) 2 | % [result] = strEndsWith(str, prefix) 3 | % 4 | % Check whether a given string end with a given prefix. 5 | % 6 | % Copyright by Holger Caesar, 2014 7 | 8 | result = false; 9 | assert(~isempty(prefix)); 10 | if numel(str) >= numel(prefix) && strcmp(str(end-numel(prefix)+1:end), prefix), 11 | result = true; 12 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/dagnn_calvin/sortLayers.m: -------------------------------------------------------------------------------- 1 | function sortLayers(net) 2 | % sortLayers(net) 3 | % 4 | % Takes a DAG and sorts it's layers by execution order. 5 | % For a linear chain this means that layers are sorted by occurrence. 6 | % This change is purely cosmetic. 7 | % 8 | % Copyright by Holger Caesar, 2016 9 | 10 | order = net.getLayerExecutionOrder(); 11 | net.layers = net.layers(order); 12 | net.rebuild(); -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxUnion.m: -------------------------------------------------------------------------------- 1 | function union = BoxUnion(a, b) 2 | % union = BoxUnion(a, b) 3 | % 4 | % Creates the union box of two bounding boxes. 5 | % 6 | % a: Input bonding box "a" 7 | % b: Input bounding box "b" 8 | % 9 | % union: Intersection of box a and b 10 | 11 | union = [min(a(:,1),b(:,1)) min(a(:,2),b(:,2)) ... 12 | max(a(:,3),b(:,3)) max(a(:,4),b(:,4))]; 13 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/matFileHasField.m: -------------------------------------------------------------------------------- 1 | function[res] = matFileHasField(path, fieldName) 2 | % [res] = matFileHasField(path, fieldName) 3 | % 4 | % Check whether a mat file includes a variable, without loading it. 5 | % Returns true if it is included or else false. 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | matObj = matfile(path, 'Writable', false); 10 | info = whos(matObj, fieldName); 11 | res = numel(info) == 1; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/testClassification.m: -------------------------------------------------------------------------------- 1 | function result = testClassification(~, ~, net, ~, batchInds) 2 | % result = testClassification(~, ~, net, ~, batchInds) 3 | % 4 | % Get classification scores. 5 | % 6 | % Copyright by Jasper Uijlings, 2015 7 | 8 | vI = net.getVarIndex('scores'); 9 | scoresStruct = net.vars(vI); 10 | scores = permute(scoresStruct.value, [4 3 2 1]); 11 | 12 | for i = numel(batchInds) : -1 : 1 13 | result(i).scores = gather(scores(i, :)); 14 | end 15 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/VOC_modified/VOChash_init_modified.m: -------------------------------------------------------------------------------- 1 | function hash = VOChash_init_modified(strs) 2 | 3 | hsize=4999; 4 | hash.key=cell(hsize,1); 5 | hash.val=cell(hsize,1); 6 | 7 | for i=1:numel(strs) 8 | s=strs{i}; 9 | if length(s) == 6 10 | h = mod(str2double(s),hsize)+1; 11 | else 12 | h=mod(str2double(s([3:4 6:11 13:end])),hsize)+1; 13 | end 14 | j=numel(hash.key{h})+1; 15 | hash.key{h}{j}=strs{i}; 16 | hash.val{h}(j)=i; 17 | end 18 | 19 | -------------------------------------------------------------------------------- /demo_frcn.m: -------------------------------------------------------------------------------- 1 | % demo_frcn() 2 | % 3 | % Trains and tests the Fast-RCNN object detection method on PASCAL VOC 20xx TBD. 4 | % 5 | % Copyright by Holger Caesar, 2016 6 | 7 | % Add folders to path 8 | setup(); 9 | 10 | % Download dataset 11 | downloadVOC2010(); 12 | 13 | % Download base network 14 | downloadNetwork(); 15 | 16 | % Download Selective Search 17 | downloadSelectiveSearch(); 18 | 19 | % Extract Selective Search regions 20 | setupFastRcnnRegions(); 21 | 22 | % Train and test network 23 | calvinNNDetection(); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/dagnn_calvin/replaceVariables.m: -------------------------------------------------------------------------------- 1 | function[variables] = replaceVariables(obj, variables) 2 | % [variables] = replaceVariables(obj, variables) 3 | % 4 | % Replace all default variables (x\d+) with free variables. 5 | % 6 | % Copyright by Holger Caesar, 2015 7 | 8 | for i = 1 : numel(variables), 9 | oldVariable = variables{i}; 10 | 11 | if regexp(oldVariable, 'x\d+'), 12 | freeVariable = getFreeVariable(obj); 13 | variables{i} = freeVariable; 14 | end; 15 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/findLastCheckpoint.m: -------------------------------------------------------------------------------- 1 | function epoch = findLastCheckpoint(modelDir) 2 | % epoch = findLastCheckpoint(modelDir) 3 | % 4 | % Find the last checkpoint to continue previous trainings. 5 | % 6 | % Copyright by Holger Caesar, 2015 7 | 8 | list = dir(fullfile(modelDir, 'net-epoch-*.mat')); 9 | tokens = regexp({list.name}, 'net-epoch-([\d]+).mat', 'tokens'); 10 | epoch = cellfun(@(x) sscanf(x{1}{1}, '%d'), tokens); 11 | if isempty(epoch) 12 | epoch = NaN; 13 | else 14 | epoch = max(epoch); 15 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/strStartsWith.m: -------------------------------------------------------------------------------- 1 | function[result] = strStartsWith(str, prefix) 2 | % [result] = strStartsWith(str, prefix) 3 | % 4 | % Check whether a given string starts with a given prefix. 5 | % 6 | % Copyright by Holger Caesar, 2014 7 | 8 | result = false; 9 | assert(~isempty(prefix)); 10 | 11 | if iscell(str), 12 | result = cellfun(@(x) strStartsWith(x, prefix), str); 13 | else 14 | if numel(str) >= numel(prefix) && strcmp(str(1:numel(prefix)), prefix), 15 | result = true; 16 | end; 17 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/sub2indFast.m: -------------------------------------------------------------------------------- 1 | function ndx = sub2indFast(siz, varargin) 2 | % ndx = sub2indFast(siz, varargin) 3 | % 4 | % Like sub2ind but without time-intensive checks. 5 | % 6 | % Copyright by Mathworks 7 | % Modified by Holger Caesar, 2014 8 | 9 | assert(all(numel(siz) == numel(varargin))); 10 | 11 | %Compute linear indices 12 | k = [1, cumprod(siz(1:end-1))]; 13 | ndx = 1; 14 | numOfIndInput = nargin-1; 15 | for i = 1:numOfIndInput 16 | v = varargin{i}; 17 | ndx = ndx + (double(v)-1) * k(i); 18 | end 19 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/structOverwriteFields.m: -------------------------------------------------------------------------------- 1 | function[oldStruct] = structOverwriteFields(oldStruct, newStruct) 2 | % [oldStruct] = structOverwriteFields(oldStruct, newStruct) 3 | % 4 | % Copies all fields from newStruct to oldStruct, regardless of whether they 5 | % existed before in oldStruct. 6 | % 7 | % Copyright by Holger Caesar, 2016 8 | 9 | fieldsNew = fieldnames(newStruct); 10 | 11 | for fieldIdx = 1 : numel(fieldsNew), 12 | fieldName = fieldsNew{fieldIdx}; 13 | oldStruct.(fieldName) = newStruct.(fieldName); 14 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxRemoveDuplicates.m: -------------------------------------------------------------------------------- 1 | function [boxesOut, uniqueIdx] = BoxRemoveDuplicates(boxesIn) 2 | % function boxOut = BoxRemoveDuplicates(boxIn) 3 | % 4 | % Removes duplicate boxes. Leaves the boxes in the same order 5 | % Keeps the first box of each kind. 6 | % 7 | % boxesIn: N x 4 array containing boxes 8 | % 9 | % boxexOut: M x 4 array of boxes witout duplicates. M <= N 10 | % uniqueIdx: Indices of retained boxes from boxesIn 11 | 12 | [~, uniqueIdx] = unique(boxesIn, 'rows', 'first'); 13 | uniqueIdx = sort(uniqueIdx); 14 | boxesOut = boxesIn(uniqueIdx,:); 15 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/dagnn_calvin/getFreeVariable.m: -------------------------------------------------------------------------------- 1 | function[var] = getFreeVariable(obj) 2 | % [var] = getFreeVariable(obj) 3 | % 4 | % Returns the name of a new free variable. 5 | % The name is xi where i is the smallest unused integer in the existing 6 | % variables. 7 | % 8 | % Copyright by Holger Caesar, 2015 9 | 10 | names = {obj.vars.name}; 11 | inds = nan(numel(names, 1)); 12 | 13 | for i = 1 : numel(names), 14 | [tok, ~] = regexp(names{i}, 'x(\d+)', 'tokens', 'match'); 15 | if ~isempty(tok), 16 | inds(i) = str2double(tok{1}); 17 | end; 18 | end; 19 | 20 | maxInd = max(inds); 21 | var = sprintf('x%d', maxInd+1); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/struct2Varargin.m: -------------------------------------------------------------------------------- 1 | function[res] = struct2Varargin(stru) 2 | % [res] = struct2Varargin(stru) 3 | % 4 | % Convert a struct to the varargin format of Matlab. 5 | % (alternating name and value of each field) 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | % Check inputs 10 | assert(numel(stru) == 1); 11 | 12 | % Get struct fields 13 | fields = fieldnames(stru); 14 | fieldCount = numel(fields); 15 | 16 | % Init result 17 | res = cell(1, fieldCount * 2); 18 | 19 | for fieldIdx = 1 : fieldCount, 20 | res{1 + ((fieldIdx-1) * 2)} = fields{fieldIdx}; 21 | res{2 + ((fieldIdx-1) * 2)} = stru.(fields{fieldIdx}); 22 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/isfieldRec.m: -------------------------------------------------------------------------------- 1 | function[answer] = isfieldRec(struct, varargin) 2 | % [answer] = isfieldRec(struct, varargin) 3 | % 4 | % Check if a struct has all the fields and subfields specified by their name strings. 5 | % I.e. to see if struct a has child b with element c, call isfieldRec(a, 'b', 'c') 6 | % 7 | % Copyright by Holger Caesar, 2014 8 | 9 | answer = true; 10 | nargs = numel(varargin); 11 | 12 | for i = 1 : nargs, 13 | fieldName = varargin{i}; 14 | if ~isfield(struct, fieldName), 15 | answer = false; 16 | return; 17 | elseif i < nargs, 18 | struct = getfield(struct, fieldName); 19 | end; 20 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/findIndexPermutation.m: -------------------------------------------------------------------------------- 1 | function[perm] = findIndexPermutation(from, to) 2 | % [perm] = findIndexPermutation(from, to) 3 | % 4 | % Find a permutation that maps indices from to to. 5 | % Make sure that both lists include exactly the same elements (but possibly 6 | % in different order). 7 | % 8 | % Copyright by Holger Caesar, 2014 9 | 10 | % Convert to column vectors 11 | from = from(:); 12 | to = to(:); 13 | 14 | % Check identical length 15 | assert(numel(from) == numel(to)); 16 | 17 | [fromS, a] = sort(from); 18 | [toS, b] = sort(to); 19 | 20 | assert(all(fromS == toS), 'Cannot find permutation between vectors with entirely different elements!'); 21 | 22 | perm = b(a); 23 | 24 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/fcn/pascalColors.m: -------------------------------------------------------------------------------- 1 | function cmap = pascalColors(colorCount) 2 | % cmap = pascalColors(colorCount) 3 | % 4 | % Color scheme for PASCAL VOC 5 | % First color is black for background. 6 | % 7 | % Copyright by MatConvNet 8 | 9 | cmap = zeros(colorCount, 3); 10 | for i = 1 : colorCount 11 | id = i-1; 12 | r = 0; 13 | g = 0; 14 | b = 0; 15 | for j=0:7 16 | r = bitor(r, bitshift(bitget(id, 1), 7 - j)); 17 | g = bitor(g, bitshift(bitget(id, 2), 7 - j)); 18 | b = bitor(b, bitshift(bitget(id, 3), 7 - j)); 19 | id = bitshift(id, -3); 20 | end 21 | cmap(i, 1) = r; 22 | cmap(i, 2) = g; 23 | cmap(i, 3) = b; 24 | end 25 | cmap = cmap / 255; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/rpGetFunc.m: -------------------------------------------------------------------------------- 1 | function[rpFunc] = rpGetFunc(proposalName) 2 | % [rpFunc] = rpGetFunc(proposalName) 3 | % 4 | % Define region proposal function. 5 | % 6 | % Copyright by Holger Caesar, 2015 7 | 8 | if strcmp(proposalName, 'Felzenszwalb2004'), 9 | rpFunc = @(dataset, imageName, image, varargin) rpFelzenszwalb2004(image, varargin{:}); 10 | elseif strcmp(proposalName, 'GroundTruth'), 11 | rpFunc = @(dataset, imageName, image, varargin) rpGroundTruth(dataset, imageName, varargin{:}); 12 | elseif strcmp(proposalName, 'Uijlings2013'), 13 | rpFunc = @(dataset, imageName, image, varargin) rpUijlings2013(image, varargin{:}); 14 | else 15 | error('Error: Unknown segmentation specified!'); 16 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxSize.m: -------------------------------------------------------------------------------- 1 | function [numRows, numColumns, area] = BoxSize(bbox) 2 | % [numRows numColumns Surface] = BoxSize(bbox) 3 | % 4 | % Retrieves number of rows, columns, and surface area from bounding box 5 | % 6 | % bbox: 4 x N Bounding box as [rowBegin colBegin rowEnd colEnd] 7 | % 8 | % numRows: Number of rows of boxes 9 | % numColumns: Number of columns of boxes 10 | % area: Area of boxes 11 | % 12 | % Jasper Uijlings, 2010 13 | 14 | % Box is empty 15 | if isempty(bbox) 16 | numRows = 0; 17 | numColumns = 0; 18 | area = 0; 19 | return 20 | end 21 | 22 | numRows = bbox(:,3) - bbox(:,1) + 1; 23 | numColumns = bbox(:,4) - bbox(:,2) + 1; 24 | area = numRows .* numColumns; 25 | 26 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/confMatToAccuracies.m: -------------------------------------------------------------------------------- 1 | function[pixelAccuracy, meanAccuracy, meanIU] = confMatToAccuracies(confusion) 2 | % [pixelAccuracy, meanAccuracy, meanIU] = confMatToAccuracies(confusion) 3 | % 4 | % Compute accuracies from a confusion matrix. 5 | % 6 | % Copyright by Holger Caesar, 2016 7 | 8 | % compute various statistics of the confusion matrix 9 | total = sum(confusion(:)); 10 | pos = sum(confusion, 2); 11 | res = sum(confusion, 1)'; 12 | tp = diag(confusion); 13 | IUs = tp ./ (pos + res - tp); 14 | missing = pos == 0; 15 | 16 | % Modified metrics to ignore classes for which we didn't see 17 | % any pixels yet in this epoch 18 | pixelAccuracy = sum(tp) / total; 19 | meanAccuracy = nanmean(tp ./ pos); 20 | meanIU = mean(IUs(~missing)); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/filePathRemoveFile.m: -------------------------------------------------------------------------------- 1 | function[path] = filePathRemoveFile(path) 2 | % [path] = filePathRemoveFile(path) 3 | % 4 | % Removes the file name from a file path to have just the path of the folder. 5 | % Files will be reduced to empty strings. 6 | % 7 | % Copyright by Holger Caesar, 2014 8 | 9 | delims = strfind(path, filesep); 10 | if iscell(path), 11 | % For cells 12 | empties = find(cellfun(@isempty, delims)); 13 | delims(empties) = repmat({1}, [numel(empties), 1]); 14 | path = cellfun(@(x, y) x(1:y(end)-1), path, delims, 'UniformOutput', false); 15 | else 16 | % For strings 17 | if isempty(delims), 18 | path = []; 19 | else 20 | path = path(1:delims(end)-1); 21 | end; 22 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/saveState.m: -------------------------------------------------------------------------------- 1 | function saveState(obj, fileName) 2 | % saveState(obj, fileName) 3 | % 4 | % Save network and statistics for the current epoch. 5 | % Depending on file size the net is saved in v7.3 format to allow nets with > 2GB size. 6 | % 7 | % Copyright by Holger Caesar, 2016 8 | 9 | % Get fields from CalvinNN 10 | net = obj.net; 11 | stats = obj.stats; %#ok 12 | 13 | % Extract the fields from the DAG class 14 | net = net.saveobj() ; %#ok 15 | 16 | % Determine whether to use new matfile format 17 | netInfo = whos('net'); 18 | netSize = netInfo.bytes; 19 | if netSize > 2e9, 20 | matVer = '-v7.3'; 21 | else 22 | matVer = '-v6'; 23 | end 24 | 25 | % Save to file 26 | save(fileName, 'net', 'stats', matVer); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/roipool/roiPooling_freeform_backward.m: -------------------------------------------------------------------------------- 1 | function[dzdxout] = roiPooling_freeform_backward(dzdx, combineFgBox) 2 | % [dzdxout] = roiPooling_freeform_backward(dzdx, combineFgBox) 3 | % 4 | % Freeform pooling backward pass. 5 | % 6 | % If specified, this function sums the gradients for the foreground 7 | % mask and the entire box. Otherwise it just passed through the gradients. 8 | % 9 | % Copyright by Holger Caesar, 2015 10 | 11 | if combineFgBox, 12 | % Sum the gradients from fg and entire box 13 | half = size(dzdx, 3) / 2; 14 | dzdxFg = dzdx(:, :, 1:half, :); 15 | dzdxBox = dzdx(:, :, half+1:end, :); 16 | dzdxout = dzdxFg + dzdxBox; 17 | else 18 | % Just pass through the gradients 19 | dzdxout = dzdx; 20 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/e2s2/e2s2_flipImageBoxes.m: -------------------------------------------------------------------------------- 1 | function[image, boxes, blobMasks] = e2s2_flipImageBoxes(image, boxes, oriImSize, blobMasks) 2 | % [image, boxes, blobMasks] = e2s2_flipImageBoxes(image, boxes, oriImSize, blobMasks) 3 | % 4 | % Flip the batch (image, blobs, masks) along the vertical axis. 5 | % 6 | % Copyright by Holger Caesar, 2015 7 | 8 | image = fliplr(image); 9 | boxes = [... 10 | boxes(:, 1), ... 11 | oriImSize(2) - boxes(:, 4) + 1, ... 12 | boxes(:, 3), ... 13 | oriImSize(2) - boxes(:, 2) + 1, ... 14 | ]; 15 | if exist('blobMasks', 'var'), 16 | blobMasks = cellfun(@(x) fliplr(x), blobMasks, 'UniformOutput', false); 17 | end; 18 | 19 | assert(all(boxes(:, 1) <= boxes(:, 3))); 20 | assert(all(boxes(:, 2) <= boxes(:, 4))); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/rpAddPath.m: -------------------------------------------------------------------------------- 1 | function rpAddPath(segmentationName) 2 | % rpAddPath(segmentationName) 3 | % 4 | % Adds the required folders for a region proposal method to the Matlab path. 5 | % 6 | % Copyright by Holger Caesar, 2014 7 | 8 | % Initialize region proposal functions 9 | if strStartsWith(segmentationName, 'GroundTruth'), 10 | % Do nothing 11 | elseif strcmp(segmentationName, 'Felzenszwalb2004') || strStartsWith(segmentationName, 'Uijlings2013'), 12 | % Add code to path 13 | global glBaseFolder; 14 | codeFolder = fullfile(glBaseFolder, 'Code', 'SelectiveSearchCodeIJCV'); 15 | assert(exist(codeFolder, 'dir') ~= 0) 16 | addpath(genpath(codeFolder)); 17 | else 18 | error('Error: Unknown segmentation specified!'); 19 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/compatibility/isScalar.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Workaround in case Matlab's mxIsScalar is not available. 3 | * Only works for <= 2 dimensional matrices, otherwise returns true. 4 | * 5 | * Copyright by Holger Caesar, 2016 6 | */ 7 | 8 | bool isScalar(const mxArray *arr) { 9 | int mrows = mxGetM(arr); 10 | int ncols = mxGetN(arr); 11 | 12 | // Check that this matrix has <= 2 dimensions 13 | if (mxGetNumberOfDimensions(arr) > 2) { 14 | return false; 15 | // mexErrMsgTxt("Error: Compatibility script isScalar() does not support matrices with >2 dimensions!"); 16 | } 17 | 18 | // Check format 19 | if (mrows == 1 && ncols == 1) { 20 | return true; 21 | } else { 22 | return false; 23 | } 24 | } -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/maskToBlob.m: -------------------------------------------------------------------------------- 1 | function[blob] = maskToBlob(iMask) 2 | % [blob] = maskToBlob(mask) 3 | % 4 | % Convert a mask (of the whole image) to a blob (only the relevant region). 5 | % See "help blob" for more information. 6 | % 7 | % Copyright by Holger Caesar, 2014 8 | 9 | [pixelIndsY, pixelIndsX] = find(iMask == true); 10 | 11 | minY = min(pixelIndsY); 12 | maxY = max(pixelIndsY); 13 | minX = min(pixelIndsX); 14 | maxX = max(pixelIndsX); 15 | 16 | sizeY = maxY - minY + 1; 17 | sizeX = maxX - minX + 1; 18 | 19 | rect = [minY, minX, maxY, maxX]; 20 | mask = false(sizeY, sizeX); 21 | inds = sub2ind(size(mask), pixelIndsY-minY+1, pixelIndsX-minX+1); 22 | mask(inds) = true; 23 | 24 | % Create struct 25 | blob.rect = rect; 26 | blob.mask = mask; 27 | blob.size = sum(mask(:)); -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxBestOverlapFastRcnn.m: -------------------------------------------------------------------------------- 1 | function [scores, index] = BoxBestOverlap(targetBoxes, testBoxes) 2 | % [scores, index] = BoxBestOverlap(targetBoxes, testBoxes) 3 | % 4 | % Get overlap scores (Pascal-wise) for testBoxes bounding boxes 5 | % 6 | % gtBoxes: N x 4 Target bounding boxes 7 | % testBoxes: M x 4 Test bounding boxes 8 | % 9 | % scores: M x 1 Highest overlap scores for each test bounding box 10 | % index: M x 1 Index for each testBoxes box which target box is best 11 | 12 | numGT = size(targetBoxes,1); 13 | numTest = size(testBoxes,1); 14 | 15 | scoreM = zeros(numTest, numGT); 16 | 17 | for j=1:numGT 18 | scoreM(:,j) = BoxPascalOverlap(targetBoxes(j,:), testBoxes); 19 | end 20 | 21 | [scores, index] = max(scoreM, [], 2); 22 | 23 | 24 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxIntersection.m: -------------------------------------------------------------------------------- 1 | function intersection = BoxIntersection(a, b) 2 | % intersection = BoxIntersection(a, b) 3 | % 4 | % Creates the intersection of two bounding box sets. Returns minus ones if 5 | % there is no intersection 6 | % 7 | % a: Input bonding boxes "a" 8 | % b: Input bounding boxes "b" 9 | % Note: size(a,1) == size(b,1) OR either size(a,1) = 1 or size(b,1) = 1 10 | % 11 | % intersection: Intersection of box a and b 12 | 13 | intersection = [max(a(:,1),b(:,1)) max(a(:,2),b(:,2)) ... 14 | min(a(:,3),b(:,3)) min(a(:,4),b(:,4))]; 15 | 16 | [numRows, numColumns] = BoxSize(intersection); 17 | 18 | % There is no intersection box 19 | negIds = numRows < 1 | numColumns < 1; 20 | intersection(negIds,:) = -1; 21 | 22 | 23 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/misc/SelectIdleGpu.m: -------------------------------------------------------------------------------- 1 | function id = SelectIdleGpu() 2 | % id = SelectIdleGpu() 3 | % 4 | % Selects idle GPU. 5 | % It's a bit hacky since it actually checks if less than 400 MB GPU memory is used. 6 | % There is no way yet in Matlab to check how active the GPU is currently. 7 | % 8 | % id: index. Call gpuDevice(id) afterwards (maybe not even necessary); 9 | 10 | persistent theId; 11 | if ~isempty(theId) 12 | id = theId; 13 | return; 14 | end 15 | 16 | numGpus = gpuDeviceCount; 17 | 18 | for id = numGpus : -1 : 1 19 | g = gpuDevice(id); 20 | 21 | if isprop(g, 'AvailableMemory') 22 | memUsed = g.TotalMemory - g.AvailableMemory; 23 | else 24 | memUsed = g.TotalMemory - g.FreeMemory; 25 | end 26 | 27 | if memUsed < 4e8 28 | theId = id; 29 | return 30 | end 31 | end 32 | 33 | warning('No GPU available'); 34 | id = []; 35 | -------------------------------------------------------------------------------- /setup.m: -------------------------------------------------------------------------------- 1 | function setup() 2 | % setup() 3 | % 4 | % Add Matconvnet, Matconvnet-FCN and Matconvnet-Calvin to Matlab path 5 | % and initialize global variables used by the demos. 6 | % 7 | % Copyright by Holger Caesar, 2016 8 | 9 | % Define paths 10 | root = fileparts(mfilename('fullpath')); 11 | matconvnetFolder = fullfile(root, 'matconvnet', 'matlab'); 12 | matconvnetFcnFolder = fullfile(root, 'matconvnet-fcn'); 13 | matconvnetCalvinFolder = fullfile(root, 'matconvnet-calvin'); 14 | 15 | % Add matconvnet 16 | addpath(matconvnetFolder); 17 | vl_setupnn(); 18 | 19 | % Add matconvnet-fcn 20 | addpath(matconvnetFcnFolder); 21 | 22 | % Add matconvnet-calvin 23 | addpath(genpath(matconvnetCalvinFolder)); 24 | 25 | % Define global variables 26 | global glBaseFolder glDatasetFolder glFeaturesFolder; 27 | glBaseFolder = fullfile(root, 'data'); 28 | glDatasetFolder = fullfile(glBaseFolder, 'Datasets'); 29 | glFeaturesFolder = fullfile(glBaseFolder, 'Features'); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/rpGroundTruth.m: -------------------------------------------------------------------------------- 1 | function[blobs] = rpGroundTruth(dataset, imageName, varargin) 2 | % [blobs] = rpGroundTruth(dataset, imageName, varargin) 3 | % 4 | % Create a list of region proposals (blobs) of an image using the 5 | % ground-truth labels. 6 | % 7 | % For information on the returned blobs see blob(). 8 | % 9 | % Copyright by Holger Caesar, 2014 10 | 11 | % Get label map 12 | labelMap = dataset.getImLabelMap(imageName); 13 | 14 | % Create segmentation 15 | regionMap = groundTruthExtractRegions(labelMap); 16 | 17 | % Initialize 18 | blobCount = max(regionMap(:)); 19 | blobs = cell(blobCount, 1); 20 | 21 | for blobIdx = 1 : blobCount, 22 | % Create blob 23 | iMask = regionMap == blobIdx; 24 | 25 | % Convert pixel mask to blob 26 | blob = maskToBlob(iMask); 27 | 28 | % Store result 29 | blobs{blobIdx} = blob; 30 | end; 31 | 32 | % Convert to col struct 33 | blobs = [blobs{:}]; 34 | blobs = blobs(:); -------------------------------------------------------------------------------- /demo_fcn.m: -------------------------------------------------------------------------------- 1 | % demo_fcn() 2 | % 3 | % Trains and tests a Fully Convolutional Network on SIFT Flow. 4 | % 5 | % Copyright by Holger Caesar, 2016 6 | 7 | % Add folders to path 8 | setup(); 9 | 10 | % Settings 11 | expNameAppend = 'testRelease'; 12 | 13 | % Download dataset 14 | downloadSiftFlow(); 15 | 16 | % Download base network 17 | downloadNetwork(); 18 | 19 | % Train network 20 | dataset = SiftFlowDatasetMC(); 21 | fcnTrainGeneric('expNameAppend', expNameAppend, 'dataset', dataset); 22 | 23 | % Test network 24 | stats = fcnTestGeneric('expNameAppend', expNameAppend, 'dataset', dataset); 25 | disp(stats); 26 | 27 | % Show example segmentation 28 | global glFeaturesFolder; 29 | labelingsFolder = fullfile(glFeaturesFolder, 'CNN-Models', 'FCN', dataset.name, sprintf('fcn16s-%s', expNameAppend), 'labelings-test-epoch50'); 30 | fileList = dirSubfolders(labelingsFolder); 31 | image = imread(fullfile(labelingsFolder, fileList{1})); 32 | figure(); 33 | imshow(image); 34 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/printProgress.m: -------------------------------------------------------------------------------- 1 | function printProgress(message, iterIdx, iterCount, iterStep) 2 | % printProgress(message, iterIdx, iterCount, [iterStep]) 3 | % 4 | % Prints out the current iteration index. 5 | % Only each (iterStep)th index will be printed to reduce logfile output. 6 | % 7 | % Copyright by Holger Caesar, 2014 8 | 9 | % Default arguments 10 | if ~exist('iterStep', 'var'), 11 | stepCount = 10; 12 | iterStep = floor(iterCount / stepCount); 13 | iterStep = max(1, min(iterCount, iterStep)); 14 | end; 15 | 16 | if iterIdx == 1, 17 | % Start first line 18 | if ~isempty(message), 19 | fprintf('%s (total: %d)... 1', message, iterCount); 20 | end; 21 | elseif iterIdx == iterCount, 22 | % Print final index 23 | fprintf(' %d', iterIdx); 24 | elseif mod(iterIdx, iterStep) == 0 25 | % Regular status update 26 | fprintf(' %d', iterIdx); 27 | end; 28 | 29 | % Finally add a newline symbol 30 | if iterIdx == iterCount, 31 | fprintf('\n'); 32 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/varargToStr.m: -------------------------------------------------------------------------------- 1 | function[res] = varargToStr(separator, varargin) 2 | % [res] = varargToStr([separator], varargin) 3 | % 4 | % Convert a cell of sth. to a single str. 5 | % Output is similar to evalc('disp(proposalsVars)'), but without spaces and 6 | % fancy characters. 7 | % 8 | % Copyright by Holger Caesar, 2015 9 | 10 | res = ''; 11 | if ~exist('separator', 'var') || isempty(separator), 12 | separator = ''; 13 | end; 14 | 15 | for i = 1 : numel(varargin), 16 | cur = varargin{i}; 17 | 18 | if ischar(cur), 19 | % Do nothing 20 | elseif isnumeric(cur), 21 | cur = num2str(cur); 22 | elseif iscell(cur), 23 | cur = varargToStr(separator, cur{:}); 24 | elseif isfunc(cur), 25 | cur = func2str(cur); 26 | else 27 | error('Error: Invalid type!'); 28 | end; 29 | 30 | % Concatenate result 31 | if mod(i, 2) == 1 && i > 1, 32 | res = [res, separator, cur]; 33 | else 34 | res = [res, cur]; 35 | end; 36 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxPascalOverlap.m: -------------------------------------------------------------------------------- 1 | function scores = BoxPascalOverlap(targetBox, testBoxes) 2 | % scores = PascalOverlap(targetBox, testBoxes) 3 | % 4 | % Function obtains the pascal overlap scores between the targetBox and 5 | % all testBoxes 6 | % 7 | % targetBox: 1 x 4 array containing target box 8 | % testBoxes: N x 4 array containing test boxes 9 | % 10 | % scores: N x 1 array containing for each testBox the pascal 11 | % overlap score. 12 | 13 | intersectBoxes = BoxIntersection(targetBox, testBoxes); 14 | overlapI = intersectBoxes(:,1) ~= -1; % Get which boxes overlap 15 | 16 | % Intersection size 17 | [~, ~, intersectionSize] = BoxSize(intersectBoxes(overlapI,:)); 18 | 19 | % Union size 20 | [~, ~, testBoxSize] = BoxSize(testBoxes(overlapI,:)); 21 | [~, ~, targetBoxSize] = BoxSize(targetBox); 22 | unionSize = testBoxSize + targetBoxSize - intersectionSize; 23 | 24 | scores = zeros(size(testBoxes,1),1); 25 | scores(overlapI) = intersectionSize ./ unionSize; 26 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/freqClampMinimum.m: -------------------------------------------------------------------------------- 1 | function[freqs] = freqClampMinimum(freqs, minRatio) 2 | % [freqs] = freqClampMinimum(freqs, minRatio) 3 | % 4 | % Takes absolute frequencies and clamps them such that the minimum ratio of 5 | % a single freq. to the sum of all freqs. is minRatio. 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | % Early abort if there's nothing to do 10 | if minRatio == 0 || isempty(minRatio), 11 | return; 12 | end; 13 | 14 | % Check inputs 15 | assert(minRatio * numel(freqs) <= 1); 16 | 17 | % Compute ratios 18 | totalCount = sum(freqs); 19 | ratios = freqs ./ totalCount; 20 | 21 | % Renormalize entries that are higher than minRatio 22 | valid = ratios >= minRatio; 23 | if any(~valid), 24 | nomalizationFactor = (1 - minRatio * sum(~valid)) / sum(ratios(valid)); 25 | ratios(valid) = ratios(valid) * nomalizationFactor; 26 | 27 | % Clamp entries that are smaller than minRatio 28 | ratios(~valid) = minRatio; 29 | 30 | % Go back to frequencies 31 | freqs = ratios * totalCount; 32 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/vl_compilenn_calvin.m: -------------------------------------------------------------------------------- 1 | function vl_compilenn_calvin() 2 | % vl_compilenn_calvin() 3 | % 4 | % Compile the C code provided in Matconvnet-calvin. 5 | % Matconvnet code needs to be compiled separately. 6 | % 7 | % Copyright by Holger Caesar, 2016 8 | 9 | root = calvin_root(); 10 | codeDir = fullfile(root, 'matconvnet-calvin', 'matlab'); 11 | mexDir = fullfile(codeDir, 'mex'); 12 | mexOpts = {'-largeArrayDims', '-outdir', sprintf('"%s"', mexDir)}; 13 | 14 | % E2S2-related 15 | mex(mexOpts{:}, fullfile(codeDir, 'labelpresence', 'labelPresence_backward.cpp')); 16 | mex(mexOpts{:}, fullfile(codeDir, 'regiontopixel', 'regionToPixel_backward.cpp')); 17 | mex(mexOpts{:}, fullfile(codeDir, 'regiontopixel', 'regionToPixelSoft_backward.cpp')); 18 | mex(mexOpts{:}, fullfile(codeDir, 'roipool', 'roiPooling_forward.cpp')); 19 | mex(mexOpts{:}, fullfile(codeDir, 'roipool', 'roiPooling_backward.cpp')); 20 | 21 | % Misc 22 | mex(mexOpts{:}, fullfile(codeDir, 'misc', 'computeBlobOverlapAnyPair.cpp')); 23 | mex(mexOpts{:}, fullfile(codeDir, 'misc', 'scoreBlobIoUs.cpp')); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/accumulateStats.m: -------------------------------------------------------------------------------- 1 | function stats = accumulateStats(obj, stats_) 2 | % stats = accumulateStats(obj, stats_) 3 | % 4 | % Goes through each GPUs struct stats_{g} and averages the values of all 5 | % stats fields. 6 | % 7 | % Copyright by Matconvnet 8 | % Modified by Holger Caesar, 2016 9 | 10 | stats = struct(); 11 | datasetMode = obj.imdb.datasetMode; 12 | total = 0; 13 | 14 | for g = 1 : numel(stats_) 15 | stats__ = stats_{g}; 16 | num__ = stats__.(datasetMode).num; 17 | total = total + num__; 18 | 19 | for field = setdiff(fieldnames(stats__.(datasetMode))', 'num') 20 | field = char(field); %#ok 21 | 22 | if g == 1 23 | stats.(datasetMode).(field) = 0; 24 | end 25 | stats.(datasetMode).(field) = stats.(datasetMode).(field) + stats__.(datasetMode).(field) * num__; 26 | 27 | if g == numel(stats_) 28 | stats.(datasetMode).(field) = stats.(datasetMode).(field) / total; 29 | end 30 | end 31 | end 32 | stats.(datasetMode).num = total; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/blobToImageSubs.m: -------------------------------------------------------------------------------- 1 | function[blobSubY, blobSubX] = blobToImageSubs(blob) 2 | % [blobSubY, blobSubX] = blobToImageSubs(blob) 3 | % 4 | % Convert a blob and an image to the indices of that blob in the image. 5 | % These indices are relative to the size of the whole image. We don't need 6 | % to know the image size for that! 7 | % See "help blob" for more information. 8 | % 9 | % Copyright by Holger Caesar, 2014 10 | 11 | % % For Matlab coder (not used) 12 | % assert(isa(blob, 'struct')); 13 | % assert(isa(blob.rect, 'double')); 14 | % assert(all(size(blob.rect) == [1, 4])); 15 | % assert(isa(blob.mask, 'logical')); 16 | % assert(all(size(blob.mask) >= [1, 1])); 17 | % assert(isa(blob.size, 'double')); 18 | % assert(all(size(blob.size) == [1, 1])); 19 | 20 | % Get all pix. coords for the mask 21 | [blobSubY, blobSubX] = find(blob.mask); 22 | 23 | % Make sure blobSub* are vectors (necessary when mask is a row vector) 24 | blobSubY = blobSubY(:); 25 | blobSubX = blobSubX(:); 26 | 27 | % Translate from blob coords to image coords 28 | blobSubY = blobSubY + blob.rect(1) - 1; 29 | blobSubX = blobSubX + blob.rect(2) - 1; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/setup/downloadNetwork.m: -------------------------------------------------------------------------------- 1 | function downloadNetwork(varargin) 2 | % downloadNetwork() 3 | % 4 | % Downloads the VGG-16 network model. 5 | % 6 | % Copyright by Holger Caesar, 2016 7 | 8 | % Parse input 9 | p = inputParser; 10 | addParameter(p, 'modelName', 'imagenet-vgg-verydeep-16'); 11 | addParameter(p, 'version', ''); % use latest version, otherwise betaXX 12 | parse(p, varargin{:}); 13 | 14 | modelName = p.Results.modelName; 15 | version = p.Results.version; 16 | 17 | % Settings 18 | url = sprintf('http://www.vlfeat.org/matconvnet/models%s/%s.mat', prependNotEmpty(version, '/'), modelName); 19 | rootFolder = calvin_root(); 20 | modelFolder = fullfile(rootFolder, 'data', 'Features', 'CNN-Models', 'matconvnet'); 21 | modelPath = fullfile(modelFolder, sprintf('%s%s.mat', modelName, prependNotEmpty(version, '_'))); 22 | 23 | % Download network 24 | if ~exist(modelPath, 'file') 25 | % Create folder 26 | if ~exist(modelFolder, 'dir') 27 | mkdir(modelFolder); 28 | end 29 | 30 | % Download model 31 | if ~exist(modelPath, 'dir') 32 | fprintf('Downloading model (0.5GB)...\n'); 33 | websave(modelPath, url); 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /demo_e2s2_full.m: -------------------------------------------------------------------------------- 1 | % demo_e2s2_full() 2 | % 3 | % Trains and tests a Region-based semantic segmentation with end-to-end training on SIFT Flow. 4 | % Requires the version of VGG-16 pretrained with MatConvNet's beta16, and does not 5 | % work with beta20. This is most likely due to exploding gradients, although the architecture is identical. 6 | % 7 | % Copyright by Holger Caesar, 2016 8 | 9 | % Add folders to path 10 | setup(); 11 | 12 | % Settings 13 | global glFeaturesFolder; % Define global variables to be used in all scripts 14 | labelingsFolder = fullfile(glFeaturesFolder, 'CNN-Models', 'E2S2', 'SiftFlow', 'Run1', sprintf('%s_e2s2_run1_exp2', 'SiftFlow'), 'labelings-test-epoch25'); 15 | 16 | % Download dataset 17 | downloadSiftFlow(); 18 | 19 | % Download base network 20 | downloadNetwork('version', 'beta16'); 21 | 22 | % Download Selective Search 23 | downloadSelectiveSearch(); 24 | 25 | % Extract region proposals and labels 26 | setupE2S2Regions(); 27 | 28 | % Train and test network 29 | e2s2_wrapper_SiftFlow_full(); 30 | 31 | % Show example segmentation 32 | fileList = dirSubfolders(labelingsFolder); 33 | image = imread(fullfile(labelingsFolder, fileList{1})); 34 | figure(); 35 | imshow(image); -------------------------------------------------------------------------------- /demo_e2s2_fast.m: -------------------------------------------------------------------------------- 1 | % demo_e2s2_fast() 2 | % 3 | % Trains and tests a Region-based semantic segmentation with end-to-end training on SIFT Flow. 4 | % Requires the version of VGG-16 pretrained with MatConvNet's beta16, and does not 5 | % work with newer versions. This is most likely due to exploding gradients, although the architecture is identical. 6 | % 7 | % Copyright by Holger Caesar, 2016 8 | 9 | % Add folders to path 10 | setup(); 11 | 12 | % Settings 13 | global glFeaturesFolder; % Define global variables to be used in all scripts 14 | labelingsFolder = fullfile(glFeaturesFolder, 'CNN-Models', 'E2S2', 'SiftFlow', 'Run1', sprintf('%s_e2s2_run1_exp1', 'SiftFlow'), 'labelings-test-epoch10'); 15 | 16 | % Download dataset 17 | downloadSiftFlow(); 18 | 19 | % Download base network 20 | downloadNetwork('version', 'beta16'); 21 | 22 | % Download Selective Search 23 | downloadSelectiveSearch(); 24 | 25 | % Extract region proposals and labels 26 | setupE2S2Regions(); 27 | 28 | % Train and test network 29 | e2s2_wrapper_SiftFlow_fast(); 30 | 31 | % Show example segmentation 32 | fileList = dirSubfolders(labelingsFolder); 33 | image = imread(fullfile(labelingsFolder, fileList{1})); 34 | figure(); 35 | imshow(image); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/evaluatePixAccHierarchy_loop.m: -------------------------------------------------------------------------------- 1 | function[pixCorrectHisto, pixTotalHisto] = evaluatePixAccHierarchy_loop(maxProbs, maxInds, overlapList, superPixelLabelHistos, pixCorrectHisto, pixTotalHisto) 2 | % [pixCorrectHisto, pixTotalHisto] = evaluatePixAccHierarchy_loop(maxProbs, maxInds, overlapList, superPixelLabelHistos, pixCorrectHisto, pixTotalHisto) 3 | % 4 | % Slow version of inner loop in evaluatePixAccHierarchy. 5 | % Use the mex version instead. 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | superPixelCount = size(overlapList, 2); 10 | 11 | % Compute maximum over regions (that contain a superpixel) and count pixels 12 | for superPixelIdx = 1 : superPixelCount, 13 | % Retrieve relevant regions that overlap with this superpixel and 14 | % find the one with the highest score 15 | relRegions = find(overlapList(:, superPixelIdx) == 1); 16 | [~, maxInd] = max(maxProbs(relRegions)); 17 | spLabel = maxInds(relRegions(maxInd)); 18 | spHisto = superPixelLabelHistos(:, superPixelIdx); 19 | 20 | % Increment correct and total pixel counts (per label) 21 | pixCorrectHisto(spLabel) = pixCorrectHisto(spLabel) + spHisto(spLabel); 22 | pixTotalHisto = pixTotalHisto + spHisto; 23 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/blobToImageInds.m: -------------------------------------------------------------------------------- 1 | function[blobInds] = blobToImageInds(blob, imageSize) 2 | % [blobInds] = blobToImageInds(blob, imageSize) 3 | % 4 | % Convert a blob and an image to the indices of that blob in the image. 5 | % These indices are relative to the size of the whole image. 6 | % We only use imageSize(1), other fields are irrelevant. 7 | % 8 | % See "help blob" for more information. 9 | % 10 | % Copyright by Holger Caesar, 2014 11 | 12 | % % For Matlab coder (not used) 13 | % assert(isa(blob, 'struct')); 14 | % assert(isa(blob.rect, 'double')); 15 | % assert(all(size(blob.rect) == [1, 4])); 16 | % assert(isa(blob.mask, 'logical')); 17 | % assert(all(size(blob.mask) >= [1, 1])); 18 | % assert(isa(blob.size, 'double')); 19 | % assert(all(size(blob.size) == [1, 1])); 20 | 21 | % Get all pix. coords for the mask 22 | [blobSubY, blobSubX] = find(blob.mask); 23 | 24 | % Make sure blobSub* are vectors (necessary when mask is a row vector) 25 | blobSubY = blobSubY(:); 26 | blobSubX = blobSubX(:); 27 | 28 | % Translate from blob coords to image coords 29 | blobSubY = blobSubY + blob.rect(1) - 1; 30 | blobSubX = blobSubX + blob.rect(2) - 1; 31 | 32 | % Convert subs to indices (fast alternative to sub2ind() 33 | blobInds = blobSubY + (blobSubX-1) * imageSize(1); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/readLinesToCell.m: -------------------------------------------------------------------------------- 1 | function[fileContent] = readLinesToCell(filePath, splitCols) 2 | % [fileContent] = readLinesToCell(filePath, [splitCols]) 3 | % 4 | % Read a text file and convert to cell. 5 | % Each row represents one entry. 6 | % If splitOnComma is true, the file will be treated like a csv with element 7 | % separator ',' and line separator '\n'. 8 | % 9 | % Copyright by Holger Caesar, 2014 10 | 11 | % Default arguments 12 | if ~exist('splitCols', 'var'), 13 | splitCols = false; 14 | end; 15 | rowDelim = '\n'; 16 | colDelim = ' '; 17 | 18 | % Check if file exists 19 | assert(exist(filePath, 'file') ~= 0, 'Error: File does not exist: %s', filePath); 20 | 21 | % Read input file 22 | fid = fopen(filePath, 'r'); 23 | fileContent = textscan(fid, '%s', 'Delimiter', {rowDelim}); 24 | fclose(fid); 25 | 26 | % Unpack 27 | fileContent = fileContent{1}; 28 | 29 | % Remove leading and trailing spaces 30 | fileContent = strtrim(fileContent); 31 | 32 | % Split further 33 | if splitCols, 34 | % Split each line 35 | fileContent = cellfun(@(x) strsplit(x, colDelim), fileContent, 'UniformOutput', false); 36 | 37 | % Rearrange to correct width and height and convert to number 38 | fileContent = str2double(cat(1, fileContent{:})); 39 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/evalClassification.m: -------------------------------------------------------------------------------- 1 | function evalClassification(imdb, stats, nnOpts) 2 | % evalClassification(imdb, stats, nnOpts) 3 | 4 | % Get scores and labels 5 | scores = zeros(size(imdb.misc.testLabs)); 6 | for i = 1 : length(stats.results) 7 | scores(i, :) = stats.results(i).scores; 8 | end 9 | testLabsAp = imdb.misc.testLabs; 10 | testLabsAp(testLabsAp == 0) = -1; 11 | 12 | % Compute AP 13 | ap = zeros(size(testLabsAp, 2), 1); 14 | for cI = 1:size(testLabsAp, 2) 15 | gt = testLabsAp(:, cI); 16 | out = scores(:, cI); 17 | 18 | [~, si] = sort(-out); 19 | tp = gt(si) > 0; 20 | fp = gt(si) < 0; 21 | 22 | fp = cumsum(fp); 23 | tp = cumsum(tp); 24 | rec = tp / sum(gt > 0); 25 | prec = tp ./ (fp + tp); 26 | 27 | ap(cI) = VOCap(rec, prec); 28 | end 29 | map = mean(ap); 30 | 31 | % Equivalent to the following (without writing to disk) 32 | % for classIdx = 1 : 20 33 | % className = DATAopts.classes{classIdx}; 34 | % [~, ~, ap(classIdx)] = VOCevalcls(DATAopts, 'FastRcnnMatconvnet', className, false); 35 | % end 36 | 37 | ap 38 | map 39 | 40 | % Save results to disk 41 | epoch = nnOpts.numEpochs; %#ok 42 | save([nnOpts.expDir, '/', 'resultsEpochFinalTest.mat'], 'nnOpts', 'stats', 'ap', 'map', 'epoch'); -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxRegresss.m: -------------------------------------------------------------------------------- 1 | function regressedBox = BoxRegresss(box, regressionFactor) 2 | % regressedBox = BoxRegresss(box, regressionFactor) 3 | % 4 | % Apply the regressionFactor to the box to get better bounding box: 5 | % regressedBox(4) = boxMiddle + 1/2 boxWidth + regressionFactor(4) * 1/2 boxWidth 6 | % = boxMiddle + (1+regressionFactor(4)) * 1/2 boxWidth 7 | % 8 | % box: N x 4 vector with BB coordinates 9 | % regressionFactor: N x 4 vector with regression factors 10 | % 11 | % regressedBox: N x 4 vector with updated BB coordinates 12 | % 13 | % Jasper Uijlings - 2015 14 | 15 | % regressedBox(N) = boxMiddle + (1+regressionFactor(N)) * 1/2 boxWidth 16 | regressionFactor = regressionFactor + 1; 17 | 18 | % Obtain middle 19 | middleOdd = (box(:,1) + box(:,3)) / 2; 20 | middleEven = (box(:,2) + box(:,4)) / 2; 21 | 22 | % Apply factors with respect to middle. 23 | % regress coord = middle + distance to middle .* F 24 | regressedBox(:,4) = middleEven + (box(:,4) - middleEven) .* regressionFactor(:,4); 25 | regressedBox(:,3) = middleOdd + (box(:,3) - middleOdd) .* regressionFactor(:,3); 26 | regressedBox(:,2) = middleEven + (box(:,2) - middleEven) .* regressionFactor(:,2); 27 | regressedBox(:,1) = middleOdd + (box(:,1) - middleOdd) .* regressionFactor(:,1); 28 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/+dagnn/RoiPoolingFreeform.m: -------------------------------------------------------------------------------- 1 | classdef RoiPoolingFreeform < dagnn.Layer 2 | % This layer has to be used AFTER the ROIPoolingFreeform. 3 | % It applies a mask to the roi-pooled activations, taking either fg, bg 4 | % or both of the image. 5 | % 6 | % inputs are: rois, masks, blobMasks 7 | % outputs are: rois 8 | % 9 | % Copyright by Holger Caesar, 2015 10 | 11 | properties 12 | combineFgBox = false; 13 | end 14 | 15 | properties (Transient) 16 | mask 17 | end 18 | 19 | methods 20 | function outputs = forward(obj, inputs, params) %#ok 21 | assert(numel(inputs) == 3); 22 | [outputs{1}, obj.mask] = roiPooling_freeform_forward(inputs{1}, inputs{2}, inputs{3}, obj.combineFgBox); 23 | end 24 | 25 | function [derInputs, derParams] = backward(obj, inputs, params, derOutputs) %#ok 26 | assert(numel(derOutputs) == 1); 27 | derInputs{1} = roiPooling_freeform_backward(derOutputs{1}, obj.combineFgBox); 28 | derInputs{2} = []; 29 | derInputs{3} = []; 30 | derParams = {} ; 31 | end 32 | 33 | function obj = RoiPoolingFreeform(varargin) 34 | obj.load(varargin) ; 35 | end 36 | end 37 | end -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxRegresssGirshick.m: -------------------------------------------------------------------------------- 1 | function regressedBox = BoxRegresssGirshick(box, regressionFactor) 2 | % regressedBox = BoxRegresss(box, regressionFactor) 3 | % 4 | % Apply the regressionFactor to the box to get better bounding box 5 | % Regression factors are Girshick-style 6 | % 7 | % box: N x 4 vector with BB coordinates 8 | % regressionFactor: N x 4 vector with regression factors 9 | % 10 | % regressedBox: N x 4 vector with updated BB coordinates 11 | % 12 | % Jasper Uijlings - 2015 13 | 14 | % Undo zero-mean regression targets (numbers empirically determined on Pascal VOC) 15 | regressionFactor = bsxfun(@times, regressionFactor, [0.1131 0.1277 0.2173 0.2173]); 16 | 17 | % Obtain middle and width 18 | middleR = (box(:,1) + box(:,3)) / 2; 19 | middleC = (box(:,2) + box(:,4)) / 2; 20 | [boxR, boxC] = BoxSize(box); 21 | 22 | % Get new box middles 23 | newMiddleR = middleR + regressionFactor(:,1) .* boxR; 24 | newMiddleC = middleC + regressionFactor(:,2) .* boxC; 25 | 26 | % Get new width 27 | newBoxR = boxR .* exp(regressionFactor(:,3)); 28 | newBoxC = boxC .* exp(regressionFactor(:,4)); 29 | 30 | % Get actual boxes 31 | regressedBox(:,4) = newMiddleC + (newBoxC-1) ./ 2; 32 | regressedBox(:,3) = newMiddleR + (newBoxR-1) ./ 2; 33 | regressedBox(:,2) = newMiddleC - (newBoxC-1) ./ 2; 34 | regressedBox(:,1) = newMiddleR - (newBoxR-1) ./ 2; 35 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/imageInsertText.m: -------------------------------------------------------------------------------- 1 | function[image] = imageInsertText(image, textCell, fontSize, fontColor) 2 | % [image] = imageInsertText(image, textCell, [fontSize], [fontColor]) 3 | % 4 | % Uses textInserter (from Vision Toolbox) to insert a cell of strings into 5 | % the image. Each entry forms one row. The font size is chosen such that 6 | % all text is visible. 7 | % 8 | % Copyright by Holger Caesar, 2014 9 | 10 | % Settings 11 | maxFontSize = 100; 12 | 13 | if iscell(textCell), 14 | labelString = strjoin(textCell, '\n'); 15 | elseif ischar(textCell), 16 | labelString = textCell; 17 | else 18 | error('Error: Invalid input type!'); 19 | end; 20 | 21 | % Find the maximum font size so that the text fits on 22 | % screen 23 | if ~exist('fontSize', 'var') || isempty(fontSize), 24 | maxWidth = max(cellfun(@numel, textCell)); 25 | fontSizeY = round(size(image, 1) / numel(textCell) * 0.8); 26 | fontSizeX = round(size(image, 2) / maxWidth * 1.7); 27 | fontSize = min([fontSizeY, fontSizeX, maxFontSize]); 28 | end; 29 | 30 | if ~exist('fontColor', 'var') || isempty(fontColor), 31 | fontColor = [255, 0, 0]; 32 | end; 33 | 34 | % Do not change the font to anything other than Lucida* 35 | textInserter = vision.TextInserter(labelString, 'Color', fontColor, ... 36 | 'Location', [1, 5], 'FontSize', fontSize); 37 | image = step(textInserter, image); -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxRegresssLog.m: -------------------------------------------------------------------------------- 1 | function regressedBox = BoxRegresssLog(box, regressionFactor) 2 | % regressedBox = BoxRegresss(box, regressionFactor) 3 | % 4 | % Apply the regressionFactor to the box to get better bounding box: 5 | % regressedBox(4) = boxMiddle + 1/2 boxWidth + regressionFactor(4) * 1/2 boxWidth 6 | % = boxMiddle + (1+regressionFactor(4)) * 1/2 boxWidth 7 | % 8 | % box: N x 4 vector with BB coordinates 9 | % regressionFactor: N x 4 vector with regression factors 10 | % 11 | % regressedBox: N x 4 vector with updated BB coordinates 12 | % 13 | % In log-space, like Girshick 14 | % 15 | % Jasper Uijlings - 2015 16 | 17 | % regressedBox(N) = boxMiddle + (1+regressionFactor(N)) * 1/2 boxWidth 18 | regressionFactor = 2 * (exp(regressionFactor / 6.537) - 0.5); 19 | 20 | % Obtain middle 21 | middleOdd = (box(:,1) + box(:,3)) / 2; 22 | middleEven = (box(:,2) + box(:,4)) / 2; 23 | 24 | % Apply factors with respect to middle. 25 | % regress coord = middle + distance to middle .* F 26 | regressedBox(:,4) = middleEven + (box(:,4) - middleEven) .* regressionFactor(:,4); 27 | regressedBox(:,3) = middleOdd + (box(:,3) - middleOdd) .* regressionFactor(:,3); 28 | regressedBox(:,2) = middleEven + (box(:,2) - middleEven) .* regressionFactor(:,2); 29 | regressedBox(:,1) = middleOdd + (box(:,1) - middleOdd) .* regressionFactor(:,1); 30 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/computeBlobOverlapSum.m: -------------------------------------------------------------------------------- 1 | function[overlaps] = computeBlobOverlapSum(propBlobsA, propBlobsB, imageSize) 2 | % [overlaps] = computeBlobOverlapSum(propBlobsA, propBlobsB, imageSize) 3 | % 4 | % Take Sel. Search regions and reconstruct which regions overlap by how many pixels. 5 | % (result is a matrix) 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | if numel(imageSize) == 3, 10 | imageSize = imageSize(1:2); 11 | end; 12 | propCountA = numel(propBlobsA); 13 | propCountB = numel(propBlobsB); 14 | overlaps = zeros(propCountA, propCountB); %Dense is fine 15 | 16 | % Precompute blob inds 17 | blobsIndsA = cell(propCountA, 1); 18 | for i = 1 : propCountA, 19 | blob = propBlobsA(i); 20 | blobsIndsA{i} = blobToImageInds(blob, imageSize); 21 | end; 22 | 23 | blobsIndsB = cell(propCountB, 1); 24 | for i = 1 : propCountB, 25 | blob = propBlobsB(i); 26 | blobsIndsB{i} = blobToImageInds(blob, imageSize); 27 | end; 28 | 29 | for i = 1 : propCountA, 30 | for j = 1 : propCountB, 31 | % Compute the intersection / union 32 | % (Faster than ismember()) 33 | overlaps(i, j) = sum(builtin('_ismemberhelper', blobsIndsA{i}, blobsIndsB{j})); 34 | % assert(overlap == sum(ismember(blobsIndsA{i}, blobsIndsB{j}))); 35 | % assert(sum(ismember(blobsIndsA{i}, blobsIndsB{j})) == sum(ismember(blobsIndsA{j}, blobsIndsB{i}))); 36 | end; 37 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/setup/downloadSelectiveSearch.m: -------------------------------------------------------------------------------- 1 | function downloadSelectiveSearch() 2 | % downloadSelectiveSearch() 3 | % 4 | % Downloads and unpacks the Selective Search region proposal method. 5 | % 6 | % Copyright by Holger Caesar, 2016 7 | 8 | % Settings 9 | zipName = 'SelectiveSearchCodeIJCV.zip'; 10 | url = 'http://koen.me/research/downloads/SelectiveSearchCodeIJCV.zip'; 11 | rootFolder = calvin_root(); 12 | codeFolder = fullfile(rootFolder, 'data', 'Code'); 13 | downloadFolder = fullfile(rootFolder, 'data', 'Downloads'); 14 | zipFile = fullfile(downloadFolder, zipName); 15 | checkFile = fullfile(codeFolder, 'SelectiveSearchCodeIJCV', 'Image2HierarchicalGrouping.m'); 16 | 17 | % Download dataset 18 | if ~exist(checkFile, 'file') 19 | % Create folder 20 | if ~exist(codeFolder, 'dir') 21 | mkdir(codeFolder); 22 | end 23 | if ~exist(downloadFolder, 'dir') 24 | mkdir(downloadFolder); 25 | end 26 | 27 | % Download zip file 28 | if ~exist(zipFile, 'file') 29 | fprintf('Downloading Selective Search (0.3MB)...\n'); 30 | websave(zipFile, url); 31 | end 32 | 33 | % Unzip it 34 | if ~exist(checkFile, 'file') 35 | fprintf('Unpacking Selective Search...\n'); 36 | unzip(zipFile, codeFolder); 37 | end 38 | end 39 | 40 | % Add to path 41 | addpath(genpath(fullfile(codeFolder, 'SelectiveSearchCodeIJCV'))); 42 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/e2s2/e2s2_prepareImage.m: -------------------------------------------------------------------------------- 1 | function[image, oriImSize] = e2s2_prepareImage(net, image, maxImageSize) 2 | % [image, oriImSize] = e2s2_prepareImage(net, image, maxImageSize) 3 | % 4 | % Resize the image and subtract the mean image. 5 | % 6 | % Copyright by Holger Caesar, 2015 7 | 8 | % Resize image 9 | oriImSize = size(image); 10 | resizeFactor = maxImageSize / max(oriImSize(1:2)); 11 | targetSize = ceil(oriImSize(1:2) .* resizeFactor); % ceil corresponds to Matlab's imresize behavior 12 | image = imresize(image, resizeFactor); 13 | assert(size(image, 1) == targetSize(1) && size(image, 2) == targetSize(2)); 14 | 15 | if numel(net.meta.normalization.averageImage) == 3, 16 | % Subtract fixed number from each channel 17 | image(:, :, 1) = image(:, :, 1) - net.meta.normalization.averageImage(1); 18 | image(:, :, 2) = image(:, :, 2) - net.meta.normalization.averageImage(2); 19 | image(:, :, 3) = image(:, :, 3) - net.meta.normalization.averageImage(3); 20 | else 21 | % Resize averageImage and subtract it from each image 22 | % Note: This cannot be done on the gpu as Matlab's gpu-compatible 23 | % imresize function can only resize by a constant factor and the image 24 | % might not be square. 25 | averageImage = net.meta.normalization.averageImage ./ 255; 26 | averageImage = imresize(averageImage, targetSize); 27 | image = image - averageImage; 28 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/indicesOfAInB.m: -------------------------------------------------------------------------------- 1 | function[inds] = indicesOfAInB(A, B) 2 | % [inds] = indicesOfAInB(A, B) 3 | % 4 | % Get the indices of a list B such that B(inds) == A. 5 | % This assumes that each element of A is in B (but not the other way 6 | % around). 7 | % Note: % B need not be unique, but A does! 8 | % Afterwards: B(indicesOfAInB(A, B)) == A 9 | % 10 | % Copyright by Holger Caesar, 2014 11 | 12 | % Make column vectors 13 | A = A(:)'; 14 | B = B(:)'; 15 | 16 | % Check for duplicates in A 17 | Aun = unique(A, 'stable'); 18 | assert(isequal(A, Aun), 'Error: This function only works for unique elements in A (but not in B)!'); 19 | 20 | % Intersect A and B 21 | [~, indsIntoB] = intersect(B, A); 22 | 23 | % Since inter is sorted, we need to sort A as well 24 | [~, AunSorting] = sort(A); 25 | 26 | % Additional assertions to understand what's going on 27 | % assert(isequal(A(Asorting), inter)); 28 | % assert(isequal(inter, B(indsIntoB))); 29 | 30 | % Check if A is really included in B 31 | checkIncluded = ismember(A, B); 32 | if ~all(checkIncluded), 33 | notIncludedList = strjoin(A(find(~checkIncluded, 5)), ', '); 34 | error('Error: Some elements of A are not included in B! Examples are %s', notIncludedList); 35 | end; 36 | 37 | % Apply mappings from B to inter and then from inter to A 38 | inds = indsIntoB(invertPermutation(AunSorting)); 39 | 40 | % Consistency check 41 | assert(isequal(A, B(inds))); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/setup/downloadSiftFlow.m: -------------------------------------------------------------------------------- 1 | function downloadSiftFlow() 2 | % downloadSiftFlow() 3 | % 4 | % Downloads and unpacks the SIFT Flow dataset. 5 | % 6 | % Copyright by Holger Caesar, 2016 7 | 8 | % Settings 9 | zipName = 'SiftFlowDataset.zip'; 10 | url = 'http://www.cs.unc.edu/~jtighe/Papers/ECCV10/siftflow/SiftFlowDataset.zip'; 11 | rootFolder = calvin_root(); 12 | datasetFolder = fullfile(rootFolder, 'data', 'Datasets', 'SiftFlow'); 13 | downloadFolder = fullfile(rootFolder, 'data', 'Downloads'); 14 | zipFile = fullfile(downloadFolder, zipName); 15 | semanticLabelFolder = fullfile(datasetFolder, 'SemanticLabels'); 16 | metaFolder = fullfile(datasetFolder, 'Meta'); 17 | 18 | % Download dataset 19 | if ~exist(metaFolder, 'file') 20 | % Create folder 21 | if ~exist(datasetFolder, 'dir') 22 | mkdir(datasetFolder); 23 | end 24 | if ~exist(downloadFolder, 'dir') 25 | mkdir(downloadFolder); 26 | end 27 | 28 | % Download zip file 29 | if ~exist(zipFile, 'file') 30 | fprintf('Downloading SIFT Flow dataset (140MB)...\n'); 31 | websave(zipFile, url); 32 | end 33 | 34 | % Unzip it 35 | if ~exist(semanticLabelFolder, 'dir') 36 | fprintf('Unpacking SIFT Flow dataset...\n'); 37 | unzip(zipFile, datasetFolder); 38 | end 39 | 40 | % Create meta folder 41 | if ~exist(metaFolder, 'dir') 42 | mkdir(metaFolder); 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/setupDataOpts.m: -------------------------------------------------------------------------------- 1 | function setupDataOpts(vocYear, testName, datasetDir) 2 | 3 | global DATAopts; 4 | 5 | % Setup VOC data 6 | devkitroot = [datasetDir, 'VOCdevkit', '/']; 7 | DATAopts.year = vocYear; 8 | DATAopts.dataset = sprintf('VOC%d', DATAopts.year); 9 | DATAopts.datadir = [devkitroot, DATAopts.dataset, '/']; 10 | DATAopts.resdir = [devkitroot, 'results', '/', DATAopts.dataset '/']; 11 | DATAopts.localdir = [devkitroot, 'local', '/', DATAopts.dataset, '/']; 12 | DATAopts.gStructPath = [DATAopts.resdir, 'GStructs', '/']; 13 | DATAopts.imgsetpath = [DATAopts.datadir, 'ImageSets', '/', 'Main', '/', '%s.txt']; 14 | DATAopts.imgpath = [DATAopts.datadir, 'JPEGImages', '/', '%s.jpg']; 15 | DATAopts.clsimgsetpath = [DATAopts.datadir, 'ImageSets', '/', 'Main', '/', '%s_%s.txt']; 16 | DATAopts.annopath = [DATAopts.datadir, 'Annotations', '/', '%s.xml']; 17 | DATAopts.annocachepath = [DATAopts.localdir, '%s_anno.mat']; 18 | DATAopts.classes={... 19 | 'aeroplane' 20 | 'bicycle' 21 | 'bird' 22 | 'boat' 23 | 'bottle' 24 | 'bus' 25 | 'car' 26 | 'cat' 27 | 'chair' 28 | 'cow' 29 | 'diningtable' 30 | 'dog' 31 | 'horse' 32 | 'motorbike' 33 | 'person' 34 | 'pottedplant' 35 | 'sheep' 36 | 'sofa' 37 | 'train' 38 | 'tvmonitor'}; 39 | DATAopts.nclasses = length(DATAopts.classes); 40 | DATAopts.testset = testName; 41 | DATAopts.minoverlap = 0.5; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/dagnn_calvin/replaceLayer.m: -------------------------------------------------------------------------------- 1 | function replaceLayer(obj, layerName, newLayerName, newBlock, addInputs, addOutputs, addParams, removeOldInputs) 2 | % replaceLayer(obj, layerName, newLayerName, newBlock, [addInputs], [addOutputs], [addParams], [removeOldInputs]) 3 | % 4 | % Takes a DAG and replaces an existing layer with a new one. 5 | % If not specified, the inputs, outputs and params are reused from the 6 | % previous layer. 7 | % 8 | % Copyright by Holger Caesar, 2015 9 | 10 | % Default settings 11 | if ~exist('removeOldInputs', 'var') || isempty(removeOldInputs), 12 | removeOldInputs = false; 13 | end; 14 | 15 | % Find the old layer 16 | layerIdx = obj.getLayerIndex(layerName); 17 | layer = obj.layers(layerIdx); 18 | 19 | % Keep old settings (or not) 20 | if removeOldInputs 21 | newInputs = {}; 22 | else 23 | newInputs = layer.inputs; 24 | end 25 | newOutputs = layer.outputs; 26 | newParams = layer.params; 27 | 28 | % Add new settings 29 | if exist('addInputs', 'var') && ~isempty(addInputs) 30 | newInputs = [newInputs, addInputs]; 31 | end 32 | if exist('addOutputs', 'var') && ~isempty(addOutputs) 33 | newOutputs = [newOutputs, addOutputs]; 34 | end 35 | if exist('addParams', 'var') && ~isempty(addParams) 36 | newParams = [newParams, addParams]; 37 | end 38 | 39 | % Remove the old layer (must come before add to avoid conflicts) 40 | obj.removeLayer(layerName); 41 | 42 | % Add the new layer 43 | obj.addLayer(newLayerName, newBlock, newInputs, newOutputs, newParams); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/regiontopixel/regionToPixel_forward.m: -------------------------------------------------------------------------------- 1 | function[scoresSP, mapSP] = regionToPixel_forward(scoresAll, overlapListAll) 2 | % [scoresSP, mapSP] = regionToPixel_forward(scoresAll, overlapListAll) 3 | % 4 | % Go from a region level to a pixel level. 5 | % (to be able to compute a loss there) 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | % Move to CPU 10 | gpuMode = isa(scoresAll, 'gpuArray'); 11 | if gpuMode 12 | scoresAll = gather(scoresAll); 13 | end 14 | 15 | % Check inputs 16 | assert(~any(isnan(scoresAll(:)) | isinf(scoresAll(:)))); 17 | 18 | % Reshape scores 19 | scoresAll = reshape(scoresAll, [size(scoresAll, 3), size(scoresAll, 4)]); 20 | 21 | % Init 22 | labelCount = size(scoresAll, 1); 23 | spCount = size(overlapListAll, 2); 24 | scoresSP = nan(labelCount, spCount, 'single'); % Note that zeros will be counted anyways! 25 | mapSP = nan(labelCount, spCount); 26 | 27 | % Compute maximum scores and map/mask for the backward pass 28 | for spIdx = 1 : spCount 29 | ancestors = find(overlapListAll(:, spIdx)); 30 | if ~isempty(ancestors) 31 | % For each label, compute the ancestor with the highest score 32 | [scoresSP(:, spIdx), curInds] = max(scoresAll(:, ancestors), [], 2); 33 | curBoxInds = ancestors(curInds); 34 | mapSP(:, spIdx) = curBoxInds; 35 | end 36 | end 37 | 38 | % Reshape the scores 39 | scoresSP = reshape(scoresSP, [1, 1, size(scoresSP)]); 40 | 41 | % Convert outputs back to GPU if necessary 42 | if gpuMode 43 | scoresSP = gpuArray(scoresSP); 44 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/accumulateGradients.m: -------------------------------------------------------------------------------- 1 | function state = accumulateGradients(obj, state, net, batchSize) 2 | % state = accumulateGradients(obj, state, net, batchSize) 3 | % 4 | % Perform a Stochastic Gradient Descent update step of the network weights 5 | % using momentum and weight decay. 6 | % 7 | % Copyright by Matconvnet (cnn_train_dag.m) 8 | % Modified by Holger Caesar, 2016 9 | 10 | for p = 1 : numel(net.params) 11 | switch net.params(p).trainMethod 12 | case 'average' % mainly for batch normalization 13 | thisLR = net.params(p).learningRate; 14 | net.params(p).value = ... 15 | (1 - thisLR) * net.params(p).value + ... 16 | (thisLR / batchSize / net.params(p).fanout) * net.params(p).der; 17 | assert(gather(~any(isnan(net.params(p).value(:))))); 18 | case 'gradient' 19 | thisDecay = obj.nnOpts.weightDecay * net.params(p).weightDecay; 20 | thisLR = state.learningRate * net.params(p).learningRate; 21 | state.momentum{p} = obj.nnOpts.momentum * state.momentum{p} ... 22 | - thisDecay * net.params(p).value ... 23 | - (1 / batchSize) * net.params(p).der; 24 | net.params(p).value = net.params(p).value + thisLR * state.momentum{p}; 25 | 26 | case 'otherwise' 27 | error('Unknown training method ''%s'' for parameter ''%s''.', ... 28 | net.params(p).trainMethod, ... 29 | net.params(p).name); 30 | end 31 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/groundTruthExtractRegions.m: -------------------------------------------------------------------------------- 1 | function[regionMap, labelList, regionPixelCounts] = groundTruthExtractRegions(gtMap) 2 | % [regionMap, labelList, regionPixelCounts] = groundTruthExtractRegions(gtMap) 3 | % 4 | % Takes a map of ground-truth labels and extracts different 8-connected 5 | % regions from it. 6 | % 7 | % Copyright by Holger Caesar, 2014 8 | 9 | % Get a list of the labels in the image and remove background (0) 10 | labels = double(unique(gtMap(:))); 11 | labels(labels == 0) = []; 12 | labelCount = numel(labels); 13 | 14 | % Extract all 8-connected regions of the same label in the image 15 | % regions = cell(labelCount, 1); 16 | labelList = zeros(labelCount, 1); 17 | regionPixelCounts = zeros(labelCount, 1); 18 | regionIdx = 1; 19 | regionMap = zeros(size(gtMap)); 20 | for labelIdx = 1 : labelCount, 21 | % Find the number of different regions with the current label 22 | regionStruct = bwconncomp(gtMap == labels(labelIdx)); 23 | regionsPixels = regionStruct.PixelIdxList'; 24 | regionCount = numel(regionsPixels); 25 | 26 | for labelRegionIdx = 1 : regionCount, 27 | regionMap(regionsPixels{labelRegionIdx}) = regionIdx; 28 | 29 | % Count the number of pixels in the region 30 | regionPixelCounts(regionIdx, 1) = numel(regionsPixels{labelRegionIdx}); 31 | 32 | % Assign the current label 33 | labelList(regionIdx, 1) = labels(labelIdx); 34 | 35 | % Increment counter 36 | regionIdx = regionIdx + 1; 37 | end; 38 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/roipool/roiPooling_freeform_forward.m: -------------------------------------------------------------------------------- 1 | function[rois, masks] = roiPooling_freeform_forward(rois, masks, blobMasks, combineFgBox) 2 | % [rois, masks] = roiPooling_freeform_forward(rois, masks, blobMasks, combineFgBox) 3 | % 4 | % Freeform pooling forward pass. 5 | % 6 | % This is an extension to the roiPool layer and MUST come after it in a 7 | % network. It applies a each blob's freeform mask to the activations and 8 | % masks of roi pooling. 9 | % 10 | % Depending on the options, it either keeps the entire box, just the 11 | % foreground or both. 12 | % 13 | % Copyright by Holger Caesar, 2015 14 | 15 | % Store a copy of the box features if we still need them 16 | if combineFgBox, 17 | roisBox = rois; 18 | masksBox = masks; 19 | end; 20 | 21 | % Perform freeform pooling and update mask for backpropagation 22 | assert(numel(blobMasks) == size(rois, 4)); 23 | blobMasksMat = cat(4, blobMasks{:}); 24 | blobMasksNanMat = double(~blobMasksMat); 25 | blobMasksNanMat(blobMasksNanMat(:) == 0) = nan; 26 | rois = bsxfun(@times, rois, blobMasksMat); 27 | masks = bsxfun(@times, masks, blobMasksNanMat); 28 | 29 | % Debug: To visualize each blob (requires an update) 30 | % figure(1); imagesc(blobMaskOri); figure(2); imagesc(blobMask); nonEmptyChannel = maxInd(squeeze(sum(sum(rois(:, :, :, blobIdx), 1), 2))); figure(3); imagesc(rois(:, :, nonEmptyChannel, blobIdx)); figure(4); imagesc(masks(:, :, nonEmptyChannel, blobIdx)) 31 | 32 | % Concatenate fg and box 33 | if combineFgBox, 34 | rois = cat(3, rois, roisBox); 35 | masks = cat(3, masks, masksBox); 36 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/labelMapToColorImage.m: -------------------------------------------------------------------------------- 1 | function[image] = labelMapToColorImage(labelMap, labelCount, cmap) 2 | % [image] = labelMapToColorImage(labelMap, [labelCount], [cmap]) 3 | % 4 | % Convert an mxn labelMap to a color image. 5 | % Each index is replaced by a certain color from a colormap. 6 | % If labelCount is specified, we use absolute colors, otherwise the colors 7 | % are sorted by label indices. 8 | % 9 | % Copyright by Holger Caesar, 2014 10 | 11 | % Default arguments 12 | if ~exist('labelCount', 'var'), 13 | labelCount = []; 14 | end; 15 | if ~exist('cmap', 'var'), 16 | cmap = @jet; 17 | end; 18 | 19 | % Get a unique list of all labels 20 | labelList = unique(labelMap(:)); 21 | labelList(labelList == 0) = []; 22 | labelListCount = numel(labelList); 23 | 24 | % Initialize result 25 | image = zeros(size(labelMap, 1), size(labelMap, 2), 3); 26 | if isempty(labelCount), 27 | % Take a variable color scheme 28 | colormap = cmap(labelListCount); 29 | else 30 | % Take a fixed color scheme relative to the number of labels in the 31 | % dataset 32 | colormap = cmap(labelCount); 33 | colormap = colormap(labelList, :); 34 | end; 35 | 36 | % Go through each label and replace its pixels by a color 37 | for labelListIdx = 1 : labelListCount, 38 | labelIdx = labelList(labelListIdx); 39 | indices = labelMap == labelIdx; 40 | 41 | curImage = cat(3, ... 42 | indices .* colormap(labelListIdx, 1), ... 43 | indices .* colormap(labelListIdx, 2), ... 44 | indices .* colormap(labelListIdx, 3)); 45 | image = image + curImage; 46 | end; -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015-2016, Holger Caesar and Jasper Uijlings. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those 25 | of the authors and should not be interpreted as representing official policies, 26 | either expressed or implied, of the FreeBSD Project. -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/imdb/ImdbCalvin.m: -------------------------------------------------------------------------------- 1 | classdef ImdbCalvin < handle 2 | %IMDBCALVIN 3 | % Base image database that holds information about the 4 | % dataset and various retrieval functions, such as getBatch(..). 5 | % 6 | % Copyright by Holger Caesar & Jasper Uijlings, 2015 7 | 8 | properties 9 | numClasses 10 | datasetMode % train, val or test 11 | epoch 12 | misc % structure for arbitrary other info 13 | 14 | data % data.train data.val data.test 15 | end 16 | 17 | methods (Abstract) 18 | % This is the main method which needs to be implemented. 19 | % It is used by CalvinNN.train() 20 | [inputs, numElements] = getBatch(obj, batchInds, net, nnOpts); 21 | end 22 | 23 | methods 24 | function setDatasetMode(obj, datasetMode) 25 | % 'train', 'val', or 'test' set 26 | if ~ismember(datasetMode, {'train', 'val', 'test'}), 27 | error('Unknown datasetMode'); 28 | end 29 | 30 | obj.datasetMode = datasetMode; 31 | end 32 | 33 | function allBatchInds = getAllBatchInds(obj) 34 | % Obtain the indices and ordering of all batches (for this epoch) 35 | switch obj.datasetMode 36 | case 'train' 37 | allBatchInds = randperm(size(obj.data.train, 1)); 38 | otherwise 39 | allBatchInds = 1:size(obj.data.(obj.datasetMode), 1); 40 | end 41 | end 42 | 43 | function initEpoch(obj, epoch) 44 | obj.epoch = epoch; 45 | end 46 | end 47 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/rpGetProposalsVars.m: -------------------------------------------------------------------------------- 1 | function[proposalsVars] = rpGetProposalsVars(proposalName, newProposalsVars) 2 | % [proposalsVars] = rpGetProposalsVars(proposalName, newProposalsVars) 3 | % 4 | % Define the parameters for a region proposal function. 5 | % Default parameters will be overwritten with those in "newProposalsVars". 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | % Set default arguments 10 | if strcmp(proposalName, 'Felzenszwalb2004'), 11 | defProposalsVars = {'k', 100, 'sigma', 0.8, 'colorTypes', {'Rgb'}}; 12 | elseif strcmp(proposalName, 'Uijlings2013'), 13 | defProposalsVars = {'ks', 100, 'sigma', 0.8, 'colorTypes', {'Rgb'}}; 14 | else 15 | defProposalsVars = {}; 16 | end; 17 | 18 | % Copy old proposalsVars 19 | proposalsVars = defProposalsVars; 20 | 21 | % Overwrite default arguments if necessary 22 | if ~exist('newProposalsVars', 'var') || isempty(newProposalsVars), 23 | % Do nothing 24 | else 25 | % Check number of arguments 26 | assert(mod(numel(newProposalsVars), 2) == 0); 27 | 28 | for attrPairIdx = 1 : numel(newProposalsVars) / 2, 29 | attrIdx = 1 + (attrPairIdx - 1) * 2; 30 | matchInds = find(strcmp(defProposalsVars, newProposalsVars{attrIdx})); 31 | 32 | if numel(matchInds) == 1, 33 | % Overwrite existing key 34 | proposalsVars{matchInds + 1} = newProposalsVars{attrIdx + 1}; 35 | elseif numel(matchInds) == 0, 36 | % Append new key 37 | proposalsVars = [proposalsVars, newProposalsVars(attrIdx : attrIdx + 1)]; %#ok 38 | else 39 | error('Error: Attributes need to be unique!'); 40 | end; 41 | end; 42 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxOverlap.m: -------------------------------------------------------------------------------- 1 | function [IoU, IoA, IoB] = BoxOverlap(boxesA, boxesB) 2 | % scores = BoxOverlap(boxesA, boxesB) 3 | % Get overlap scores between boxesA and boxesB. 4 | % boxesA and boxesB must be of the same length or one of the two 5 | % must be a single box. In the latter case this single box is compared to 6 | % all boxes of the other argument 7 | % 8 | % boxesA: Nx4 OR 1x4 matrix containing boxes 9 | % boxesB: Nx4 OR 1x4 matrix containing boxes 10 | % 11 | % IoU: Nx1 IoU scores per comparison 12 | % IoA: Nx1 IoA scores per comparison 13 | % IoB: Nx1 IoB scores per comparison 14 | % 15 | % Note: IoA is intersection divided by size of A 16 | 17 | % Get intersection of the boxes 18 | intersectBox = BoxIntersection(boxesA, boxesB); 19 | goodIdx = intersectBox(:,1) ~= -1; % find where boxes overlap 20 | 21 | [~, ~, intersectSize] = BoxSize(intersectBox(goodIdx,:)); 22 | 23 | 24 | 25 | % Get area of boxesA and boxesB 26 | % If statement deals with multiple/single boxes in argument 27 | if size(boxesA,1) == 1 28 | [~, ~, sizeBoxesA] = BoxSize(boxesA); 29 | else 30 | [~, ~, sizeBoxesA] = BoxSize(boxesA(goodIdx,:)); 31 | end 32 | 33 | if size(boxesB,1) == 1 34 | [~, ~, sizeBoxesB] = BoxSize(boxesB); 35 | else 36 | [~, ~, sizeBoxesB] = BoxSize(boxesB(goodIdx,:)); 37 | end 38 | 39 | % Union = sizeBoxA + sizeBoxB - intersection(A,B) 40 | IoU = zeros(size(intersectBox,1),1); 41 | IoU(goodIdx,:) = intersectSize ./ (sizeBoxesA + sizeBoxesB - intersectSize); 42 | 43 | IoA = zeros(size(intersectBox,1),1); 44 | IoA(goodIdx,:) = intersectSize ./ sizeBoxesA; 45 | 46 | IoB = zeros(size(intersectBox,1),1); 47 | IoB(goodIdx,:) = intersectSize ./ sizeBoxesB; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/AnnotationMC.m: -------------------------------------------------------------------------------- 1 | classdef AnnotationMC 2 | % ANNOTATIONMC 3 | % Annotation class used by all Datasets 4 | % Stores information about the labels etc. 5 | % 6 | % Copyright by Holger Caesar, 2014 7 | 8 | properties 9 | name; 10 | annotationFolder; 11 | metaFolder = 'Meta'; 12 | imageCount = []; 13 | labelIdx = []; 14 | 15 | labelExt = '.mat'; 16 | labelFormat = 'mat-labelMap'; 17 | 18 | labelImageExt = '.mat'; 19 | labelImageFormat = 'mat-labelList'; 20 | 21 | hasPixelLabels = true; 22 | hasPixelLabelsOnlyTst = false; 23 | hasImageLabels = false; 24 | 25 | labelFolder; 26 | labelImageFolder; 27 | labelCount = []; 28 | imageListFunc = @(varargin) getImageListAll(varargin{:}); 29 | namesFile = 'labelNames.mat'; 30 | active = false; 31 | hasStuffThingLabels = false; 32 | labelOneIsBg = false; 33 | 34 | cmap = @jet; 35 | end 36 | 37 | methods 38 | % Constructor 39 | function[obj] = AnnotationMC(name) 40 | % [obj] = AnnotationMC(name) 41 | % 42 | % Annotation constructor 43 | 44 | if ~exist('name', 'var'), 45 | name = 'Original'; 46 | end; 47 | 48 | obj.name = name; 49 | obj.annotationFolder = fullfile('Annotations', name); 50 | obj.labelFolder = fullfile(obj.annotationFolder, 'PixelLabels'); 51 | obj.labelImageFolder = fullfile(obj.annotationFolder, 'ImageLabels'); 52 | end 53 | end 54 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/computeBlobOverlapAny.m: -------------------------------------------------------------------------------- 1 | function[overlap] = computeBlobOverlapAny(propBlobsA, propBlobsB, imageSize) 2 | % [overlap] = computeBlobOverlapAny(propBlobsA, propBlobsB, imageSize) 3 | % 4 | % Take Sel. Search regions and reconstruct whether blobs overlap at all. 5 | % (result is a matrix). 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | if numel(imageSize) == 3, 10 | imageSize = imageSize(1:2); 11 | end; 12 | propCountA = numel(propBlobsA); 13 | propCountB = numel(propBlobsB); 14 | 15 | % Precompute blob inds 16 | blobsIndsA = cell(propCountA, 1); 17 | blobsIndsB = cell(propCountB, 1); 18 | for i = 1 : propCountA, 19 | blob = propBlobsA(i); 20 | blobsIndsA{i} = int32(blobToImageInds(blob, imageSize)); 21 | end; 22 | if isequal(propBlobsA, propBlobsB), 23 | blobsIndsB = blobsIndsA; 24 | else 25 | for i = 1 : propCountB, 26 | blob = propBlobsB(i); 27 | blobsIndsB{i} = int32(blobToImageInds(blob, imageSize)); 28 | end; 29 | end; 30 | 31 | overlap = computeBlobOverlapAnyPair(blobsIndsA, blobsIndsB); % Mex-version is 6 times faster for Barcelona dataset 32 | 33 | 34 | % function[overlap] = computeBlobOverlapAnyPair(blobsIndsA, blobsIndsB) %#ok 35 | % % [overlap] = computeBlobOverlapAnyPair(blobsIndsA, blobsIndsB) 36 | % % 37 | % % Compute if two blobs overlap at all (boolean). 38 | % % To be replaced by a mex function. 39 | % 40 | % % Initialize 41 | % propCountA = numel(blobsIndsA); 42 | % propCountB = numel(blobsIndsB); 43 | % overlap = zeros(propCountA, propCountB); %Dense is fine 44 | % 45 | % for i = 1 : propCountA, 46 | % for j = 1 : propCountB, 47 | % overlap(i, j) = any(builtin('_ismemberhelper', blobsIndsA{i}, blobsIndsB{j})); 48 | % end; 49 | % end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/evaluatePixAccHierarchy_preload.m: -------------------------------------------------------------------------------- 1 | function[pixAcc, meanClassPixAcc] = evaluatePixAccHierarchy_preload(imageList, probs, ~, overlapListCell, superPixelLabelHistosCell, varargin) 2 | % [pixAcc, meanClassPixAcc] = evaluatePixAccHierarchy_preload(imageList, probs, ~, varargin) 3 | % 4 | % Same as evaluatePixAccPaint, but much faster by taking into account the SS hierarchy. 5 | % To use this hierarchy, run reconstructSelSearchHierarchyFromFz(). 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | % Parse input 10 | p = inputParser; 11 | addParameter(p, 'printStatus', true); 12 | parse(p, varargin{:}); 13 | 14 | printStatus = p.Results.printStatus; 15 | 16 | % Init 17 | labelCount = size(probs{1}, 2); 18 | assert(labelCount > 1); 19 | pixCorrectHisto = zeros(labelCount, 1); 20 | pixTotalHisto = zeros(labelCount, 1); 21 | 22 | imageCount = numel(imageList); 23 | for imageIdx = 1 : imageCount, 24 | if printStatus, 25 | printProgress('Evaluating pixel accuracy for image', imageIdx, imageCount); 26 | end; 27 | 28 | % Skip images without ground-truth 29 | if isempty(probs{imageIdx}), 30 | continue; 31 | end; 32 | 33 | % Precompute maximum over labels 34 | [maxProbs, maxInds] = max(probs{imageIdx}, [], 2); 35 | 36 | % Compute maximum over regions (that contain a superpixel) and count pixels 37 | [pixCorrectHisto, pixTotalHisto] = evaluatePixAccHierarchy_loop(maxProbs, maxInds, full(overlapListCell{imageIdx}), superPixelLabelHistosCell{imageIdx}, pixCorrectHisto, pixTotalHisto); 38 | end; 39 | 40 | % Compute overall accuracies 41 | pixAcc = sum(pixCorrectHisto) / sum(pixTotalHisto); 42 | classPixAcc = pixCorrectHisto ./ pixTotalHisto; 43 | meanClassPixAcc = nanmean(classPixAcc); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/rpFelzenszwalb2004.m: -------------------------------------------------------------------------------- 1 | function[blobs] = rpFelzenszwalb2004(image, varargin) 2 | % [blobs] = rpFelzenszwalb2004(image, varargin) 3 | % 4 | % Create a list of region proposals (blobs) of an image using the method of Felzenszwalb and Huttenlocher 2004. 5 | % 6 | % For information on the returned blobs see blob(). 7 | % 8 | % Copyright by Holger Caesar, 2014 9 | 10 | % Parse input 11 | p = inputParser; 12 | addParameter(p, 'sigma', 0.8); % From IJCV paper Tighe 13 | addParameter(p, 'k', 200); % From IJCV paper Tighe 14 | addParameter(p, 'minSize', []); % By default this is set to be equal to k. (as done in Selective Search) 15 | addParameter(p, 'colorTypes', {'Rgb'}); 16 | parse(p, varargin{:}); 17 | 18 | sigma = p.Results.sigma; 19 | k = p.Results.k; 20 | minSize = p.Results.minSize; 21 | colorTypes = p.Results.colorTypes; 22 | 23 | % Set minsize to k if nothing is set 24 | if isempty(minSize), 25 | minSize = k; 26 | end; 27 | 28 | % Convert to 8 bit (necessary for mex code) 29 | image = im2uint8(image); 30 | 31 | % Convert to correct colorType 32 | assert(iscell(colorTypes)); 33 | assert(numel(colorTypes) == 1); 34 | if ~strcmp(colorTypes, 'Rgb') 35 | [~, image] = Image2ColourSpace(image, colorTypes{1}); 36 | end 37 | 38 | % Create segmentation (same version as in Selective Search code) 39 | regionMap = mexFelzenSegmentIndex(image, sigma, k, minSize); 40 | 41 | % Initialize 42 | blobCount = max(regionMap(:)); 43 | blobs = cell(blobCount, 1); 44 | 45 | for blobIdx = 1 : blobCount, 46 | % Create blob 47 | iMask = regionMap == blobIdx; 48 | 49 | % Convert pixel mask to blob 50 | blob = maskToBlob(iMask); 51 | 52 | % Store result 53 | blobs{blobIdx} = blob; 54 | end; 55 | 56 | % Convert to col struct 57 | blobs = [blobs{:}]; 58 | blobs = blobs(:); -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/setupImdbDetection.m: -------------------------------------------------------------------------------- 1 | function[imdb] = setupImdbDetection(trainName, testName, net) 2 | % [imdb] = setupImdbDetection(trainName, testName, net) 3 | 4 | global DATAopts; 5 | 6 | %%% Setup the Imdb 7 | % Get and test images 8 | trainIms = textread(sprintf(DATAopts.imgsetpath, trainName), '%s'); %#ok 9 | testIms = textread(sprintf(DATAopts.imgsetpath, testName), '%s'); %#ok 10 | 11 | % Make train, val, and test set. For Pascal, I illegally use part of the test images 12 | % as validation set. This is to match Girshick performance while still having 13 | % meaningful graphs for the validation set. 14 | % Note: allIms are just all images. datasetIdx determines how these are divided over 15 | % train, val, and test. 16 | allIms = cat(1, trainIms, testIms); 17 | datasetIdx = cell(3, 1); 18 | datasetIdx{1} = (1:length(trainIms))'; % Jasper: Use all training images. Only for comparison Pascal Girshick 19 | datasetIdx{2} = (length(trainIms)+1:length(trainIms)+501)'; % Use part of the test images for validation. Not entirely legal, but otherwise it will take much longer to get where we want. 20 | datasetIdx{3} = (length(trainIms)+1:length(allIms))'; 21 | 22 | imdb = ImdbDetectionFullSupervision(DATAopts.imgpath(1:end-6), ... % path 23 | DATAopts.imgpath(end-3:end), ... % image extension 24 | DATAopts.gStructPath, ... % gStruct path 25 | allIms, ... % all images 26 | datasetIdx, ... % division into train/val/test 27 | net.meta.normalization.averageImage); % average image used to pretrain network 28 | 29 | % Usually instance weighting gives better performance. But not Girshick style 30 | % imdbPascal.SetInstanceWeighting(true); 31 | 32 | % Store lists for use in eval 33 | imdb.misc.trainIms = trainIms; 34 | imdb.misc.testIms = testIms; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/DetectionToPascalVOCFiles.m: -------------------------------------------------------------------------------- 1 | function [recall, prec, ap, apUpperBound] = DetectionToPascalVOCFiles(set, class, boxes, boxIms, boxClfs, compName, doEval, overlapNms) 2 | % Filters overlapping boxes (near duplicates), creates official VOC 3 | % detection files. Evaluates results. 4 | 5 | global DATAopts; 6 | 7 | DATAopts.testset = set; 8 | 9 | if ~exist('doEval', 'var') 10 | doEval = 0; 11 | end 12 | 13 | className = DATAopts.classes{class}; 14 | 15 | % Sort scores/boxes/images 16 | [boxClfs, sI] = sort(boxClfs, 'descend'); 17 | boxIms = boxIms(sI); 18 | boxes = boxes(sI, :); 19 | 20 | % Filter boxes if wanted 21 | if exist('overlapNms', 'var') && overlapNms > 0 22 | [uIms, ~, uN] = unique(boxIms); 23 | keepIds = true(size(boxes,1), 1); 24 | fprintf('Filtering %d: ', length(uIms)); 25 | for i = 1 : length(uIms) 26 | if mod(i-1, 500) == 0 27 | fprintf('%d ', i); 28 | end 29 | currIds = find(uN == i); 30 | [~, goodBoxesI] = BoxNMS(boxes(currIds, :), overlapNms); 31 | keepIds(currIds) = goodBoxesI; 32 | end 33 | boxClfs = boxClfs(keepIds); 34 | boxIms = boxIms(keepIds); 35 | boxes = boxes(keepIds, :); 36 | fprintf('\n'); 37 | end 38 | 39 | % Save detection results using detection results 40 | savePath = fullfile(DATAopts.resdir, 'Main', ['%s_det_', set, '_%s.txt']); 41 | resultsName = sprintf(savePath, compName, className); 42 | fid = fopen(resultsName, 'w'); 43 | for j = 1 : length(boxIms) 44 | fprintf(fid,'%s %f %f %f %f %f\n', boxIms{j}, boxClfs(j), boxes(j, :)); 45 | end 46 | fclose(fid); 47 | fprintf('\n'); 48 | 49 | if doEval 50 | [recall, prec, ap] = VOCevaldet_modified(DATAopts, className, resultsName, false); 51 | apUpperBound = max(recall); 52 | else 53 | recall = 0; 54 | prec = 0; 55 | ap = 0; 56 | apUpperBound = 0; 57 | end -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxNMS.m: -------------------------------------------------------------------------------- 1 | function [boxesOut, idx] = BoxNMS(boxesIn, overlapNms) 2 | % function [boxesOut idx] = BoxNMS(boxesIn, overlapNms) 3 | % 4 | % Performs non-maximum suppression: all boxes which have an overlap higher than 5 | % maxScore with another box higher in the boxesIn list will be filtered 6 | % out. The boxes must be already ordered 7 | % 8 | % boxesIn: N x 4 array of boxes 9 | % overlapNms: Maximum overlap threshold 10 | % 11 | % boxesOut: M x 4 array of boxes, M <= N 12 | % idx: Logical array denoting kept boxes 13 | % 14 | % Jasper Uijlings - 2013 15 | 16 | 17 | if nargin < 2 18 | overlapNms = 0.3; 19 | end 20 | 21 | numBoxes = size(boxesIn,1); 22 | isGood = true(numBoxes, 1); 23 | 24 | [~, ~, boxSizes] = BoxSize(boxesIn); 25 | 26 | for i=1:numBoxes-1 27 | if isGood(i) % Remove near duplicates 28 | % Get target boxes (lower than current box) 29 | targetI = find(isGood); 30 | targetI = targetI(targetI > i); 31 | targetBoxes = boxesIn(targetI,:); 32 | 33 | % Get size of intersection 34 | intersectR = (1 + min(boxesIn(i,3), targetBoxes(:,3)) - max(boxesIn(i,1), targetBoxes(:,1))); 35 | intersectC = (1 + min(boxesIn(i,4), targetBoxes(:,4)) - max(boxesIn(i,2), targetBoxes(:,2))); 36 | 37 | doesIntersect = intersectR > 0 & intersectC > 0; 38 | intersectSizes = zeros(size(intersectR)); 39 | intersectSizes(doesIntersect) = intersectR(doesIntersect) .* intersectC(doesIntersect); 40 | 41 | % Calculate Intersection / Union. 42 | scores = intersectSizes ./ (boxSizes(targetI) + boxSizes(i) - intersectSizes); 43 | isGoodT = scores < overlapNms; 44 | 45 | isGood(targetI) = isGood(targetI) & isGoodT; 46 | end 47 | end 48 | 49 | idx = isGood; 50 | boxesOut = boxesIn(isGood,:); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/test.m: -------------------------------------------------------------------------------- 1 | function[stats] = test(obj) 2 | % [stats] = test(obj) 3 | % 4 | % Test function 5 | % - Does a single processing of an epoch for testing 6 | % - Uses the nnOpts.testFn function for the testing (inside process_epoch) 7 | % - Automatically changes softmaxloss to softmax, removes hinge loss. Other losses are not yet supported 8 | % 9 | % Copyright by Jasper Uijlings, 2015 10 | % Modified by Holger Caesar, 2016 11 | 12 | % Check that we only use one GPU 13 | numGpus = numel(obj.nnOpts.gpus); 14 | assert(numGpus <= 1); 15 | 16 | % Replace softmaxloss layer with softmax layer 17 | softMaxLossIdx = obj.net.getLayerIndex('softmaxloss'); 18 | if ~isnan(softMaxLossIdx) 19 | softmaxlossInput = obj.net.layers(softMaxLossIdx).inputs{1}; 20 | obj.net.removeLayer('softmaxloss'); 21 | obj.net.addLayer('softmax', dagnn.SoftMax(), softmaxlossInput, 'scores', {}); 22 | softmaxIdx = obj.net.layers(obj.net.getLayerIndex('softmax')).outputIndexes; 23 | assert(numel(softmaxIdx) == 1); 24 | end 25 | 26 | % Remove hinge loss layer 27 | hingeLossIdx = obj.net.getLayerIndex('hingeloss'); 28 | if ~isnan(hingeLossIdx) 29 | theInputs = obj.net.layers(hingeLossIdx).inputs; 30 | finalLayerOutputIdx = find(ismember(theInputs, {'label'}) == 0); 31 | assert(numel(finalLayerOutputIdx) == 1); 32 | finalLayerOutputName = obj.net.layers(hingeLossIdx).inputs{finalLayerOutputIdx}; 33 | obj.net.removeLayer('hingeloss'); 34 | obj.net.renameVar(finalLayerOutputName, 'scores'); 35 | end 36 | 37 | % Set datasetMode in imdb 38 | datasetMode = 'test'; 39 | obj.net.mode = datasetMode; % Disable dropout 40 | obj.imdb.setDatasetMode(datasetMode); 41 | state.epoch = 1; 42 | state.allBatchInds = obj.imdb.getAllBatchInds(); 43 | 44 | % Process the epoch 45 | obj.stats.(datasetMode) = obj.processEpoch(obj.net, state); 46 | 47 | % The stats are the desired results 48 | stats = obj.stats.(datasetMode); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/init.m: -------------------------------------------------------------------------------- 1 | function init(obj, varargin) 2 | % init(obj, varargin) 3 | % 4 | % Initialize the CalvinNN network with the default options. 5 | % Also starts up GPU pool. 6 | % 7 | % Copyright by Holger Caesar, 2016 8 | 9 | defnnOpts.expDir = fullfile('data', 'exp'); 10 | defnnOpts.continue = true; 11 | defnnOpts.batchSize = 2; 12 | defnnOpts.numSubBatches = 2; 13 | defnnOpts.gpus = []; 14 | defnnOpts.numEpochs = 16; 15 | defnnOpts.learningRate = [repmat(1e-3, [1, 12]), repmat(1e-4, [1, 4])]; 16 | defnnOpts.weightDecay = 0.0005; 17 | defnnOpts.momentum = 0.9; 18 | defnnOpts.derOutputs = {'objective', 1}; 19 | defnnOpts.lossFnObjective = 'softmaxlog'; % Default is softmax 20 | defnnOpts.extractStatsFn = @CalvinNN.extractStats; 21 | defnnOpts.testFn = @(imdb, nnOpts, net, inputs, batchInds) error('Error: Test function not implemented'); % function used at test time to evaluate performance 22 | defnnOpts.misc = struct(); % fields used by custom layers are stored here 23 | defnnOpts.plotEval = true; 24 | defnnOpts.plotAccuracy = true; 25 | 26 | % Network options 27 | defnnOpts.convertToTrain = true; 28 | 29 | % Fast R-CNN options 30 | defnnOpts.fastRcnn = true; 31 | defnnOpts.fastRcnnParams = true; % learning rates and weight decay 32 | defnnOpts.misc.roiPool.use = true; 33 | defnnOpts.misc.roiPool.freeform.use = false; 34 | defnnOpts.bboxRegress = true; 35 | 36 | % Merge input settings with default settings 37 | nnOpts = vl_argparse_old(defnnOpts, varargin, 'nonrecursive'); 38 | 39 | % Check settings 40 | assert(numel(nnOpts.learningRate) == 1 || numel(nnOpts.learningRate) == nnOpts.numEpochs); 41 | 42 | % Do not create directory in evaluation mode 43 | if ~exist(nnOpts.expDir, 'dir') && ~isempty(nnOpts.expDir), 44 | mkdir(nnOpts.expDir); 45 | end 46 | 47 | % Setup GPU 48 | numGpus = numel(nnOpts.gpus); 49 | assert(numGpus <= 1); 50 | if numGpus == 1, 51 | gpuDevice(nnOpts.gpus); 52 | end 53 | 54 | % Set new fields 55 | obj.nnOpts = nnOpts; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/loadNetwork.m: -------------------------------------------------------------------------------- 1 | function loadNetwork(obj, netIn) 2 | % loadNetwork(obj, netIn) 3 | % 4 | % Loads a network and converts it into DAG format. 5 | % 6 | % Network can be: 7 | % - DAG (netIn.net, [netIn.stats] or netIn.layers, netIn.vars, ...) 8 | % - SimpleNN (netIn.layers, [netIn.normalization] , ...) 9 | % - Path to any of the above 10 | 11 | % Load the network from file if necessary 12 | if ischar(netIn), 13 | netIn = load(netIn); 14 | end 15 | 16 | % Convert the network 17 | if isfield(netIn, 'net') 18 | if isa(netIn.net, 'dagnn.DagNN') 19 | % DagNN as class object 20 | obj.net = netIn.net; 21 | else 22 | % Any network as struct 23 | % Recurse on the net structure 24 | obj.loadNetwork(netIn.net); 25 | end 26 | 27 | % Load stats of a snapshot 28 | if isfield(netIn, 'stats') 29 | obj.stats = netIn.stats; 30 | end; 31 | elseif isfield(netIn, 'vars') 32 | % DagNN as struct: Only DAG formats have the 'vars' field 33 | obj.net = dagnn.DagNN.loadobj(netIn); 34 | 35 | elseif isfield(netIn, 'layers') && iscell(netIn.layers) 36 | % SimpleNN as struct 37 | % Convert SimpleNN to DagNN 38 | obj.net = dagnn.DagNN.fromSimpleNN(netIn); 39 | else 40 | error('Error: Network is neither in SimpleNN nor DAG format!'); 41 | end 42 | 43 | % Remove unused/incorrect meta fields from old network 44 | if isprop(obj.net, 'meta') 45 | if isfield(obj.net.meta, 'normalization') 46 | fields = {'keepAspect', 'border', 'imageSize', 'interpolation'}; 47 | for i = 1 : numel(fields) 48 | fieldName = fields{i}; 49 | if isfield(obj.net.meta.normalization, fieldName) 50 | obj.net.meta.normalization = rmfield(obj.net.meta.normalization, fieldName); 51 | end 52 | end 53 | end 54 | if isfield(obj.net.meta, 'classes') 55 | obj.net.meta = rmfield(obj.net.meta, 'classes'); 56 | end 57 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/setup/downloadModel.m: -------------------------------------------------------------------------------- 1 | function downloadModel(modelName) 2 | % downloadModel(modelName) 3 | % 4 | % Downloads and unzips a trained model. 5 | % 6 | % Copyright by Holger Caesar, 2016 7 | 8 | % Settings 9 | global glBaseFolder; 10 | baseUrl = 'http://groups.inf.ed.ac.uk/calvin/caesar16eccv'; 11 | downloadFolder = fullfile(glBaseFolder, 'Downloads'); 12 | if strcmp(modelName, 'frcn') 13 | exportName = 'FRCN_VOC2010_model'; 14 | exportFilesTarget = fullfile(glBaseFolder, 'Features', 'CNN-Models', 'FRCN', 'VOC2010', 'VOC2010-testRelease'); 15 | elseif strcmp(modelName, 'fcn') 16 | exportName = 'FCN_SiftFlow_model'; 17 | exportFilesTarget = fullfile(glBaseFolder, 'Features', 'CNN-Models', 'FCN', 'SiftFlow', 'fcn16s-testRelease'); 18 | elseif strcmp(modelName, 'e2s2_fast') 19 | exportName = 'E2S2_SiftFlow_model_fast'; 20 | exportFilesTarget = fullfile(glBaseFolder, 'Features', 'CNN-Models', 'E2S2', 'SiftFlow', 'Run1', 'SiftFlow_e2s2_run1_exp1'); 21 | elseif strcmp(modelName, 'e2s2_full') 22 | exportName = 'E2S2_SiftFlow_model_full'; 23 | exportFilesTarget = fullfile(glBaseFolder, 'Features', 'CNN-Models', 'E2S2', 'SiftFlow', 'Run1', 'SiftFlow_e2s2_run1_exp2'); 24 | else 25 | error('Error: Unknown model: %s', modelName); 26 | end 27 | 28 | % Create folder 29 | if ~exist(downloadFolder, 'dir') 30 | mkdir(downloadFolder); 31 | end 32 | 33 | % Download model 34 | url = fullfile(baseUrl, [exportName, '.zip']); 35 | downloadPath = fullfile(downloadFolder, [exportName, '.zip']); 36 | if exist(downloadPath, 'file') 37 | fprintf('Using existing model file: %s\n', downloadPath); 38 | else 39 | fprintf('Downloading model to: %s\n', downloadPath); 40 | websave(downloadPath, url); 41 | end 42 | 43 | % Unzip model 44 | if exist(exportFilesTarget, 'dir') 45 | fprintf('Using existing model in: %s\n', exportFilesTarget); 46 | else 47 | fprintf('Unzipping model to: %s\n', exportFilesTarget); 48 | unzip(downloadPath, exportFilesTarget); 49 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/dagnn_calvin/insertLayer.m: -------------------------------------------------------------------------------- 1 | function insertLayer(obj, leftLayerName, rightLayerName, newLayerName, newBlock, addInputs, addOutputs, newParams) 2 | % insertLayer(obj, leftLayerName, rightLayerName, newLayerName, newBlock, [addInputs], [addOutputs], [newParams]) 3 | % 4 | % Takes a DAG and inserts a new layer before an existing layer. 5 | % The outputs of the previous layer and inputs of the following layer are 6 | % adapted accordingly. 7 | % 8 | % Copyright by Holger Caesar, 2015 9 | 10 | % Find the old layers and their outputs/inputs 11 | leftLayerIdx = obj.getLayerIndex(leftLayerName); 12 | rightLayerIdx = obj.getLayerIndex(rightLayerName); 13 | leftLayer = obj.layers(leftLayerIdx); 14 | rightLayer = obj.layers(rightLayerIdx); 15 | leftOutputs = leftLayer.outputs; 16 | rightInputs = rightLayer.inputs; 17 | 18 | % Check whether left and right are actually connected 19 | assert(leftLayerIdx ~= rightLayerIdx); 20 | 21 | % Introduce new free variables for new layer outputs 22 | rightInputs = replaceVariables(obj, rightInputs); 23 | 24 | % Change the input of the right layer (to avoid cycles) 25 | obj.layers(rightLayerIdx).inputs = rightInputs; 26 | 27 | newInputs = leftOutputs; 28 | 29 | % % % Automatic filtering is problematic for different variable names, do 30 | % % % everything manually! 31 | % % % newOutputs = rightInputs; 32 | 33 | % Remove special inputs from rightInputs (i.e. labels) 34 | newOutputs = regexp(rightInputs, '^(x\d+)', 'match', 'once'); 35 | newOutputs = newOutputs(~cellfun(@isempty, newOutputs)); 36 | 37 | % Adapt inputs, outputs and params 38 | if exist('addInputs', 'var') && ~isempty(addInputs), 39 | newInputs = [newInputs, addInputs]; 40 | end; 41 | if exist('addOutputs', 'var') && ~isempty(addOutputs), 42 | newOutputs = [newOutputs, addOutputs]; 43 | end; 44 | if ~exist('newParams', 'var') || isempty(newParams), 45 | newParams = {}; 46 | end; 47 | 48 | % Add the new layer 49 | obj.addLayer(newLayerName, newBlock, newInputs, newOutputs, newParams); 50 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/evaluatePixAcc.m: -------------------------------------------------------------------------------- 1 | function[pixAcc, meanClassPixAcc, classPixAccs] = evaluatePixAcc(dataset, imageList, probs, segmentFolder, varargin) 2 | % [pixAcc, meanClassPixAcc, classPixAccs] = evaluatePixAcc(dataset, imageList, probs, segmentFolder, varargin) 3 | % 4 | % Evaluates pixel-level accuracies either directly or using Selective Search hierarchies. 5 | % 6 | % Copyright by Holger Caesar, 2015 7 | 8 | % Test if hierarchy information is available 9 | assert(~isempty(segmentFolder)); 10 | imageIdx = 1; 11 | imageName = imageList{imageIdx}; 12 | segmentPath = fullfile(segmentFolder, [imageName, '.mat']); 13 | matObj = matfile(segmentPath); 14 | info = whos(matObj, 'superPixelLabelHistos'); 15 | if numel(info) == 1, 16 | hasHierarchy = true; 17 | else 18 | hasHierarchy = false; 19 | end; 20 | usePreload = false; 21 | 22 | % Evaluate pixel accuracy 23 | if hasHierarchy, 24 | if usePreload, 25 | % Init 26 | imageCount = numel(imageList); %#ok 27 | spLabelHistosCell = cell(imageCount, 1); 28 | overlapListCell = cell(imageCount, 1); 29 | 30 | % Preload segmentation info 31 | for imageIdx = 1 : imageCount, 32 | imageName = imageList{imageIdx}; 33 | segmentPath = [segmentFolder, filesep, imageName, '.mat']; 34 | segmentStruct = load(segmentPath, 'superPixelLabelHistos', 'overlapList'); 35 | spLabelHistosCell{imageIdx} = segmentStruct.superPixelLabelHistos'; 36 | overlapListCell{imageIdx} = double(segmentStruct.overlapList); 37 | end; 38 | 39 | [pixAcc, meanClassPixAcc, classPixAccs] = evaluatePixAccHierarchy_preload(imageList, probs, segmentFolder, overlapListCell, spLabelHistosCell, varargin{:}); 40 | else 41 | [pixAcc, meanClassPixAcc, classPixAccs] = evaluatePixAccHierarchy(imageList, probs, segmentFolder, varargin{:}); 42 | end; 43 | else 44 | [pixAcc, meanClassPixAcc, classPixAccs] = evaluatePixAccPaint(dataset, imageList, probs, segmentFolder, varargin{:}); 45 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/computeBlobOverlapAnyPair.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * [...] = computeBlobOverlapAnyPair(...) 3 | * Used in computeBlobOverlapAny(). 4 | * 5 | * Computes whether any two blobs overlap at all. 6 | * 7 | * Copyright by Holger Caesar, 2015 8 | * 9 | */ 10 | #include "mex.h" 11 | #include "matrix.h" 12 | 13 | void mexFunction( 14 | int nlhs, mxArray *plhs[], 15 | int nrhs, const mxArray *prhs[] ) { 16 | 17 | // Get inputs 18 | const mxArray* blobsIndsA = prhs[0]; 19 | const mxArray* blobsIndsB = prhs[1]; 20 | int propCountA = mxGetM(blobsIndsA); 21 | int propCountB = mxGetM(blobsIndsB); 22 | 23 | // Create outputs 24 | plhs[0] = mxCreateNumericMatrix(propCountA, propCountB, mxDOUBLE_CLASS, mxREAL); 25 | double* overlaps = (double*) mxGetData(plhs[0]); 26 | 27 | for (int i = 0; i <= propCountA-1; i++) { 28 | mxArray* indsAMx = mxGetCell(blobsIndsA, i); 29 | int* indsA = (int*) mxGetData(indsAMx); 30 | int mA = mxGetM(indsAMx); 31 | 32 | for (int j = 0; j <= propCountB-1; j++) { 33 | mxArray* indsBMx = mxGetCell(blobsIndsB, j); 34 | int* indsB = (int*) mxGetData(indsBMx); 35 | int mB = mxGetM(indsBMx); 36 | 37 | 38 | // Early abort if the ranges are completely disjoint 39 | if (indsB[mB-1] < indsA[0] || indsA[mA-1] < indsB[0]) { 40 | //continue; % Uncomment this and test 41 | } else { 42 | int posA = 0; 43 | int posB = 0; 44 | 45 | for ( ; posB < mB && posA < mA; ) { 46 | if (indsA[posA] == indsB[posB]) { 47 | overlaps[i + propCountA * j] = 1; 48 | break; 49 | } else if (indsA[posA] < indsB[posB]) { 50 | posA++; 51 | } else { 52 | posB++; 53 | } 54 | } 55 | } 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /matconvnet-calvin/examples/e2s2/e2s2_storeSPGTOverlap.m: -------------------------------------------------------------------------------- 1 | function e2s2_storeSPGTOverlap(varargin) 2 | % e2s2_storeSPGTOverlap(varargin) 3 | % 4 | % Augment the superpixel structure with the overlap with the GT blob. 5 | % 6 | % Copyright by Holger Caesar, 2015 7 | 8 | % Parse input 9 | p = inputParser; 10 | addParameter(p, 'dataset', SiftFlowDatasetMC()); 11 | addParameter(p, 'projectName', 'WeaklySupervisedLearning'); 12 | addParameter(p, 'spName', 'Felzenszwalb2004-k100-sigma0.8-colorTypesRgb'); 13 | addParameter(p, 'gtName', 'GroundTruth'); 14 | parse(p, varargin{:}); 15 | 16 | dataset = p.Results.dataset; 17 | projectName = p.Results.projectName; 18 | spName = p.Results.spName; 19 | gtName = p.Results.gtName; 20 | 21 | % Create paths 22 | global glFeaturesFolder; 23 | segmentFolderSP = fullfile(glFeaturesFolder, projectName, dataset.name, 'segmentations', spName); 24 | segmentFolderGT = fullfile(glFeaturesFolder, projectName, dataset.name, 'segmentations', gtName); 25 | 26 | % Get image list 27 | [imageList, imageCount] = dataset.getImageList(true); 28 | 29 | for imageIdx = 1 : imageCount, 30 | printProgress(sprintf('Storing GT-SP (%s) overlap for image', spName), imageIdx, imageCount, 50); 31 | imageName = imageList{imageIdx}; 32 | imageSize = dataset.getImageSize(imageName); 33 | 34 | % Get SP blobs 35 | segmentPathSP = fullfile(segmentFolderSP, [imageName, '.mat']); 36 | assert(exist(segmentPathSP, 'file') ~= 0); 37 | segmentStructSP = load(segmentPathSP, 'propBlobs'); 38 | blobsSP = segmentStructSP.propBlobs; 39 | 40 | % Get GT blobs 41 | segmentPathGT = fullfile(segmentFolderGT, [imageName, '.mat']); 42 | assert(exist(segmentPathGT, 'file') ~= 0); 43 | segmentStructGT = load(segmentPathGT, 'propBlobs'); 44 | blobsGT = segmentStructGT.propBlobs; 45 | 46 | % Compute overlap 47 | overlapRatiosSPGT = computeBlobOverlapSum(blobsSP, blobsGT, imageSize); %#ok 48 | 49 | % Store to disk (append) 50 | assert(exist(segmentPathSP, 'file') ~= 0); 51 | save(segmentPathSP, 'overlapRatiosSPGT', '-append'); 52 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/segmentationLossImage_extractMaxScores.m: -------------------------------------------------------------------------------- 1 | function[scoresImageSoftmax, mask] = segmentationLossImage_extractMaxScores(obj, labelCount, sampleCount, imageCount, masksThingsCell) 2 | % [scoresImageSoftmax, mask] = segmentationLossImage_extractMaxScores(obj, labelCount, sampleCount, imageCount, [masksThingsCell]) 3 | % 4 | % Inner loop of the SegmentationLossImage layer. 5 | % 6 | % Copyright by Holger Caesar, 2016 7 | 8 | % Init 9 | scoresImageSoftmax = nan(1, 1, labelCount, sampleCount, 'like', obj.scoresMapSoftmax); 10 | mask = nan(sampleCount, 1, 'like', obj.scoresMapSoftmax); % contains the coordinates of the pixel with highest score per class 11 | 12 | % Process each image/crop separately % very slow (!!) 13 | for imageIdx = 1 : imageCount 14 | offset = (imageIdx-1) * labelCount; 15 | 16 | if ~isempty(masksThingsCell) && ~isempty(masksThingsCell{imageIdx}) 17 | maskThings = masksThingsCell{imageIdx}; 18 | if isa(obj.scoresMapSoftmax, 'gpuArray') 19 | maskThings = gpuArray(maskThings); 20 | end 21 | end 22 | 23 | for labelIdx = 1 : labelCount 24 | sampleIdx = offset + labelIdx; 25 | 26 | if obj.useScoreDiffs 27 | % Use pixel with highest score compared to all other labels 28 | s = obj.scoresMapSoftmax(:, :, labelIdx, imageIdx) - max(obj.scoresMapSoftmax(:, :, setdiff(1:labelCount, labelIdx), imageIdx), [], 3); 29 | else 30 | % Use pixel with overall highest score 31 | s = obj.scoresMapSoftmax(:, :, labelIdx, imageIdx); 32 | end 33 | 34 | % Remove things such that the highest score lies on stuff 35 | if ~isempty(masksThingsCell) && ~isempty(masksThingsCell{imageIdx}) 36 | s = bsxfun(@times, s, ~maskThings); 37 | end 38 | 39 | [~, ind] = max(s(:)); % always take first pix with max score 40 | mask(sampleIdx, 1) = ind; 41 | [y, x] = ind2sub(size(s), ind); 42 | scoresImageSoftmax(1, 1, :, sampleIdx) = obj.scoresMapSoftmax(y, x, :, imageIdx); 43 | end 44 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/blobProbsToOutputMap.m: -------------------------------------------------------------------------------- 1 | function[outputMap, outputMapProbs] = blobProbsToOutputMap(propBlobs, regionAss, regionProbs, imageSize, varargin) 2 | % [outputMap, outputMapProbs] = blobProbsToOutputMap(propBlobs, regionAss, regionProbs, imageSize, varargin) 3 | % 4 | % Convert blobs and their probs to an outputMap. 5 | % Blobs need not be sorted by score in advance. 6 | % fallBackLabel is 1 7 | % Note: This file cannot be used in Matlab coder. 8 | % 9 | % Copyright by Holger Caesar, 2015 10 | 11 | % Parse input 12 | p = inputParser; 13 | addParameter(p, 'fallBackLabel', 1); 14 | addParameter(p, 'checkProbs', true); 15 | parse(p, varargin{:}); 16 | 17 | fallBackLabel = p.Results.fallBackLabel; 18 | checkProbs = p.Results.checkProbs; 19 | 20 | % Check inputs 21 | if checkProbs, 22 | assert(min(regionProbs) >= 0, 'Error: regionProbs are not valid probabilities!'); 23 | end; 24 | 25 | % Initialize 26 | outputMapProbs = zeros(imageSize); 27 | outputMap = ones(imageSize) * fallBackLabel; 28 | 29 | % Sort blobs to reduce the number of blobs we have to look at 30 | [~, sortOrder] = sort(regionProbs, 'descend'); 31 | 32 | % Extract most likely label per pixel (not region proposal) 33 | imageNumel = prod(imageSize); 34 | pixelSetCount = 0; 35 | propCount = numel(propBlobs); 36 | for sortOrderIdx = 1 : propCount, 37 | propIdx = sortOrder(sortOrderIdx); 38 | blob = propBlobs(propIdx); 39 | 40 | % Get blob label and score 41 | blobLabel = regionAss(propIdx); 42 | blobScore = regionProbs(propIdx); 43 | 44 | % Get indices of blob in image (Fast alternative to sub2ind) 45 | outputInds = blobToImageInds(blob, imageSize); 46 | 47 | % Take only inds which are not set yet 48 | outputInds = outputInds(outputMapProbs(outputInds) == 0); 49 | 50 | % Update pixel probs and labels 51 | pixelSetCount = pixelSetCount + numel(outputInds); 52 | outputMapProbs(outputInds) = blobScore; 53 | outputMap(outputInds) = blobLabel; 54 | 55 | % Check if we're done 56 | if pixelSetCount >= imageNumel, 57 | break; 58 | end; 59 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/evaluatePixAccHierarchy.m: -------------------------------------------------------------------------------- 1 | function[pixAcc, meanClassPixAcc, classPixAccs] = evaluatePixAccHierarchy(imageList, probs, segmentationFolder, varargin) 2 | % [pixAcc, meanClassPixAcc, classPixAccs] = evaluatePixAccHierarchy(imageList, probs, segmentationFolder, varargin) 3 | % 4 | % Same as evaluatePixAccPaint, but much faster by taking into account the SS hierarchy. 5 | % To use this hierarchy, run reconstructSelSearchHierarchyFromFz(). 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | % Parse input 10 | p = inputParser; 11 | addParameter(p, 'printStatus', true); 12 | parse(p, varargin{:}); 13 | 14 | printStatus = p.Results.printStatus; 15 | 16 | % Init 17 | labelCount = size(probs{1}, 2); 18 | assert(labelCount > 1); 19 | pixCorrectHisto = zeros(labelCount, 1); 20 | pixTotalHisto = zeros(labelCount, 1); 21 | 22 | imageCount = numel(imageList); 23 | for imageIdx = 1 : imageCount, 24 | if printStatus, 25 | printProgress('Evaluating pixel accuracy for image', imageIdx, imageCount); 26 | end; 27 | 28 | % Skip images without ground-truth 29 | if isempty(probs{imageIdx}), 30 | continue; 31 | end; 32 | 33 | % Create segmentation path 34 | imageName = imageList{imageIdx}; 35 | segmentPath = [segmentationFolder, filesep, imageName, '.mat']; 36 | 37 | % Precompute maximum over labels 38 | [maxProbs, maxInds] = max(probs{imageIdx}, [], 2); 39 | 40 | % Load superpixel information 41 | segmentStruct = load(segmentPath, 'superPixelLabelHistos', 'overlapList'); 42 | superPixelLabelHistos = segmentStruct.superPixelLabelHistos'; 43 | overlapList = double(segmentStruct.overlapList); 44 | 45 | % Compute maximum over regions (that contain a superpixel) and count pixels 46 | [pixCorrectHisto, pixTotalHisto] = evaluatePixAccHierarchy_loop(maxProbs, maxInds, full(overlapList), superPixelLabelHistos, pixCorrectHisto, pixTotalHisto); 47 | end; 48 | 49 | % Compute overall accuracies 50 | pixAcc = sum(pixCorrectHisto) / sum(pixTotalHisto); 51 | classPixAccs = pixCorrectHisto ./ pixTotalHisto; 52 | meanClassPixAcc = nanmean(classPixAccs); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/setup/downloadVOC2010.m: -------------------------------------------------------------------------------- 1 | function downloadVOC2010() 2 | % downloadVOC2010() 3 | % 4 | % Downloads and unpacks the PASCAL VOC 2010 dataset. 5 | % 6 | % Copyright by Holger Caesar, 2016 7 | 8 | % Settings 9 | zipNameData = 'VOCtrainval_03-May-2010.tar'; 10 | zipNameDevkit = 'VOCdevkit_08-May-2010.tar'; 11 | urlData = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2010/VOCtrainval_03-May-2010.tar'; 12 | urlDevkit = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2010/VOCdevkit_08-May-2010.tar'; 13 | rootFolder = calvin_root(); 14 | datasetFolder = fullfile(rootFolder, 'data', 'Datasets', 'VOC2010'); 15 | downloadFolder = fullfile(rootFolder, 'data', 'Downloads'); 16 | zipFileData = fullfile(downloadFolder, zipNameData); 17 | zipFileDevkit = fullfile(downloadFolder, zipNameDevkit); 18 | dataFolder = fullfile(datasetFolder, 'VOCdevkit', 'VOC2010', 'JPEGImages'); 19 | devkitFolder = fullfile(datasetFolder, 'VOCdevkit', 'VOCcode'); 20 | 21 | % Download dataset 22 | if ~exist(devkitFolder, 'dir') 23 | % Create folder 24 | if ~exist(datasetFolder, 'dir') 25 | mkdir(datasetFolder); 26 | end 27 | if ~exist(downloadFolder, 'dir') 28 | mkdir(downloadFolder); 29 | end 30 | 31 | % Download data 32 | if ~exist(dataFolder, 'dir') 33 | % Download tar file 34 | if ~exist(zipFileData, 'file') 35 | fprintf('Downloading VOC 2010 dataset (1.3GB)...\n'); 36 | websave(zipFileData, urlData); 37 | end 38 | 39 | % Untar it 40 | fprintf('Unpacking VOC 2010 dataset...\n'); 41 | untar(zipFileData, datasetFolder); 42 | end 43 | 44 | % Download devkit 45 | if ~exist(devkitFolder, 'dir') 46 | % Download tar file 47 | if ~exist(zipFileDevkit, 'file') 48 | fprintf('Downloading VOC 2010 devkit (0.3MB)...\n'); 49 | websave(zipFileDevkit, urlDevkit); 50 | end 51 | 52 | % Untar it 53 | fprintf('Unpacking VOC 2010 devkit...\n'); 54 | untar(zipFileDevkit, datasetFolder); 55 | end 56 | end 57 | 58 | % Add to path 59 | addpath(devkitFolder); 60 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/CalvinNN.m: -------------------------------------------------------------------------------- 1 | classdef CalvinNN < handle 2 | % CalvinNN 3 | % 4 | % Default network training script in Matconvnet-Calvin. 5 | % This uses Matconvnet's Directed Acyclic Graph structure and is 6 | % heavily inspired by the cnn_train_dag example. 7 | % 8 | % Copyright by Holger Caesar, 2015 9 | 10 | properties 11 | net 12 | imdb 13 | nnOpts 14 | stats 15 | end 16 | 17 | methods 18 | function obj = CalvinNN(net, imdb, nnOpts) 19 | % obj = CalvinNN(net, imdb, [nnOpts]) 20 | 21 | % Default arguments 22 | if ~exist('nnOpts', 'var') 23 | nnOpts = struct(); 24 | end 25 | 26 | % Set fields 27 | obj.imdb = imdb; 28 | 29 | % Init options and GPUs 30 | obj.init(nnOpts); 31 | 32 | % Load network and convert to DAG format 33 | obj.loadNetwork(net); 34 | 35 | if obj.nnOpts.convertToTrain 36 | % Convert network from test to train (add loss layer, 37 | % dropout etc.) 38 | obj.convertNetwork(); 39 | end 40 | end 41 | 42 | % Declarations for methods that are in separate files 43 | convertNetwork(obj, net); 44 | convertNetworkToFastRcnn(obj, varargin); 45 | init(obj, varargin); 46 | plotStats(obj, epochs, stats, plotAccuracy); 47 | saveState(obj, fileName); 48 | stats = test(obj); 49 | train(obj); 50 | end 51 | 52 | methods (Access = protected) 53 | % Declarations for methods that are in separate files 54 | stats = accumulateStats(obj, stats_); 55 | state = accumulateGradients(obj, state, net, batchSize); 56 | stats = processEpoch(obj, net, state); 57 | end 58 | 59 | methods (Static) 60 | stats = extractStats(net, inputs); 61 | epoch = findLastCheckpoint(modelDir); 62 | [net, stats] = loadState(obj, fileName); 63 | end 64 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/setup/exportModel.m: -------------------------------------------------------------------------------- 1 | function exportModel(modelName) 2 | % exportModel(modelName) 3 | % 4 | % Zips a trained model so that it can be uploaded to our homepage. 5 | % 6 | % Copyright by Holger Caesar, 2016 7 | 8 | % Settings 9 | global glBaseFolder; 10 | if strcmp(modelName, 'frcn') 11 | exportName = 'FRCN_VOC2010_model'; 12 | exportFiles = {'net-epoch-16.mat', 'net-train.pdf', 'log.txt', 'resultsEpochFinalTest.mat'}; 13 | exportFilesPrefix = fullfile(glBaseFolder, 'Features', 'CNN-Models', 'FRCN', 'VOC2010', 'VOC2010-testRelease'); 14 | elseif strcmp(modelName, 'fcn') 15 | exportName = 'FCN_SiftFlow_model'; 16 | exportFiles = {'labelings-test-epoch50', 'net-epoch-50.mat', 'net-opts.mat', 'net-train.pdf', 'log.txt', 'stats-test-epoch50.mat'}; 17 | exportFilesPrefix = fullfile(glBaseFolder, 'Features', 'CNN-Models', 'FCN', 'SiftFlow', 'fcn16s-testRelease'); 18 | elseif strcmp(modelName, 'e2s2_fast') 19 | exportName = 'E2S2_SiftFlow_model_fast'; 20 | exportFiles = {'labelings-test-epoch10', 'net-epoch-10.mat', 'net-opts.mat', 'net-train.pdf', 'log.txt', 'stats-test-epoch10.mat'}; 21 | exportFilesPrefix = fullfile(glBaseFolder, 'Features', 'CNN-Models', 'E2S2', 'SiftFlow', 'Run1', 'SiftFlow_e2s2_run1_exp1'); 22 | elseif strcmp(modelName, 'e2s2_full') 23 | exportName = 'E2S2_SiftFlow_model_full'; 24 | exportFiles = {'labelings-test-epoch25', 'net-epoch-25.mat', 'net-opts.mat', 'net-train.pdf', 'log.txt', 'stats-test-epoch25.mat'}; 25 | exportFilesPrefix = fullfile(glBaseFolder, 'Features', 'CNN-Models', 'E2S2', 'SiftFlow', 'Run1', 'SiftFlow_e2s2_run1_exp2'); 26 | else 27 | error('Error: Unknown model: %s', modelName); 28 | end 29 | 30 | % Define paths 31 | exportFolder = fullfile(glBaseFolder, 'Export'); 32 | if ~exist(exportFolder, 'dir') 33 | mkdir(exportFolder); 34 | end 35 | 36 | % Create zip file 37 | exportFilePath = fullfile(exportFolder, [exportName, '.zip']); 38 | if exist(exportFilePath, 'file') 39 | fprintf('Using existing zip file: %s\n', exportFilePath); 40 | else 41 | fprintf('Creating zip file: %s\n', exportFilePath); 42 | zip(exportFilePath, strcat(exportFilesPrefix, filesep, exportFiles)); 43 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/regiontopixel/regionToPixelSoft_forward.m: -------------------------------------------------------------------------------- 1 | function[scoresSP, weightsSP] = regionToPixelSoft_forward(scoresAll, overlapListAll, decay) 2 | % [scoresSP, weightsSP] = regionToPixelSoft_forward(scoresAll, overlapListAll, decay) 3 | % 4 | % Go from a region level to a pixel level. 5 | % (to be able to compute a loss there) 6 | % 7 | % Outputs: 8 | % - scoresSP: Per-class scores for each superpixel. 9 | % - weightsSP: Per-class weight of each region for each superpixel. 10 | % 11 | % Copyright by Holger Caesar, 2015 12 | 13 | % Move to CPU 14 | gpuMode = isa(scoresAll, 'gpuArray'); 15 | if gpuMode 16 | scoresAll = gather(scoresAll); 17 | end 18 | 19 | % Check inputs 20 | assert(~any(isnan(scoresAll(:)) | isinf(scoresAll(:)))); 21 | 22 | % Reshape scores 23 | scoresAll = reshape(scoresAll, [size(scoresAll, 3), size(scoresAll, 4)]); 24 | 25 | % Init 26 | labelCount = size(scoresAll, 1); 27 | rpCount = size(overlapListAll, 1); 28 | spCount = size(overlapListAll, 2); 29 | scoresSP = nan(labelCount, spCount, 'single'); % Note that zeros will be counted anyways! 30 | weightsSP = zeros(labelCount, spCount, rpCount); 31 | 32 | % Compute maximum scores and map/mask for the backward pass 33 | for spIdx = 1 : spCount 34 | ancestors = find(overlapListAll(:, spIdx)); 35 | if ~isempty(ancestors) 36 | for labelIdx = 1 : labelCount 37 | % For each label, compute the ancestor with the highest score 38 | ancestorScores = scoresAll(labelIdx, ancestors); 39 | [~, sortOrder] = sort(ancestorScores, 'descend'); 40 | [~, rank] = sort(sortOrder, 'ascend'); 41 | masses = decay .^ (rank - 1); 42 | masses = masses ./ sum(masses); 43 | outScore = sum(masses .* ancestorScores); 44 | scoresSP(labelIdx, spIdx) = outScore; 45 | weightsSP(labelIdx, spIdx, ancestors) = masses; 46 | 47 | % weightsSP(labelIdx, spIdx, outCoord) = 1; before more like that 48 | end 49 | end 50 | end 51 | 52 | % Reshape the scores 53 | scoresSP = reshape(scoresSP, [1, 1, size(scoresSP)]); 54 | 55 | % Convert outputs back to GPU if necessary 56 | if gpuMode 57 | scoresSP = gpuArray(scoresSP); 58 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/setup/setupFastRcnnRegions.m: -------------------------------------------------------------------------------- 1 | function setupFastRcnnRegions(varargin) 2 | % setupFastRcnnRegions(varargin) 3 | % 4 | % Extract Selective Search proposals and ground-truth for each image in the 5 | % PASCAL VOC 20xx dataset. Note that this takes about 4s/im or about 11h 6 | % for VOC 2010. 7 | % 8 | % Copyright by Holger Caesar, 2016 9 | 10 | %%% Settings 11 | % Dataset 12 | vocYear = 2010; 13 | trainName = 'train'; 14 | testName = 'val'; 15 | vocName = sprintf('VOC%d', vocYear); 16 | global glDatasetFolder; 17 | datasetDir = [fullfile(glDatasetFolder, vocName), '/']; 18 | setupDataOpts(vocYear, testName, datasetDir); 19 | global DATAopts; % Database specific paths 20 | assert(~isempty(DATAopts), 'Error: Dataset not initialized properly!'); 21 | 22 | % Get image lists 23 | trainIms = textread(sprintf(DATAopts.imgsetpath, trainName), '%s'); %#ok 24 | testIms = textread(sprintf(DATAopts.imgsetpath, testName), '%s'); %#ok 25 | 26 | % Check if regions were already extracted by looking at the last file 27 | lastFile = [DATAopts.gStructPath, testIms{end}, '.mat']; 28 | if ~exist(lastFile, 'file') 29 | % Create output folder 30 | if ~exist(DATAopts.gStructPath, 'dir') 31 | mkdir(DATAopts.gStructPath); 32 | end 33 | 34 | for idxImg = 1:size(trainIms,1) 35 | fprintf('Processing train img: %d/%d\n', idxImg, size(trainIms, 1)); 36 | boxPath = [DATAopts.gStructPath, trainIms{idxImg}, '.mat']; 37 | if exist(boxPath, 'file') 38 | fprintf('Skipping existing file: %s\n', boxPath); 39 | continue; 40 | end 41 | boxStruct = GetGTAndSSBoxes(trainIms{idxImg}); %#ok 42 | save(boxPath, '-struct', 'boxStruct'); 43 | end 44 | 45 | for idxImg = 1:size(testIms,1) 46 | fprintf('Processing test img: %d/%d\n', idxImg, size(testIms, 1)); 47 | boxPath = [DATAopts.gStructPath, testIms{idxImg}, '.mat']; 48 | if exist(boxPath, 'file') 49 | fprintf('Skipping existing file: %s\n', boxPath); 50 | continue; 51 | end 52 | boxStruct = GetGTAndSSBoxes(testIms{idxImg}); %#ok 53 | save(boxPath, '-struct', 'boxStruct'); 54 | end 55 | end -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/calvinNNDetection.m: -------------------------------------------------------------------------------- 1 | % function calvinNNDetection() 2 | % 3 | % Copyright by Holger Caesar, 2016 4 | 5 | % Global variables 6 | global glDatasetFolder glFeaturesFolder; 7 | assert(~isempty(glDatasetFolder) && ~isempty(glFeaturesFolder)); 8 | 9 | %%% Settings 10 | % Dataset 11 | vocYear = 2010; 12 | trainName = 'train'; 13 | testName = 'val'; 14 | 15 | % Specify paths 16 | vocName = sprintf('VOC%d', vocYear); 17 | datasetDir = [fullfile(glDatasetFolder, vocName), '/']; 18 | outputFolder = fullfile(glFeaturesFolder, 'CNN-Models', 'FRCN', vocName, sprintf('%s-testRelease', vocName)); 19 | netPath = fullfile(glFeaturesFolder, 'CNN-Models', 'matconvnet', 'imagenet-vgg-verydeep-16.mat'); 20 | logFilePath = fullfile(outputFolder, 'log.txt'); 21 | 22 | % Fix randomness 23 | randSeed = 42; 24 | rng(randSeed); 25 | 26 | % Setup dataset specific options and check validity 27 | setupDataOpts(vocYear, testName, datasetDir); 28 | global DATAopts; 29 | assert(~isempty(DATAopts), 'Error: Dataset not initialized properly!'); 30 | 31 | % Task-specific 32 | nnOpts.testFn = @testDetection; 33 | nnOpts.misc.overlapNms = 0.3; 34 | nnOpts.derOutputs = {'objective', 1, 'regressObjective', 1}; 35 | 36 | % General 37 | nnOpts.batchSize = 2; 38 | nnOpts.numSubBatches = nnOpts.batchSize; % 1 image per sub-batch 39 | nnOpts.weightDecay = 5e-4; 40 | nnOpts.momentum = 0.9; 41 | nnOpts.numEpochs = 16; 42 | nnOpts.learningRate = [repmat(1e-3, 12, 1); repmat(1e-4, 4, 1)]; 43 | nnOpts.misc.netPath = netPath; 44 | nnOpts.expDir = outputFolder; 45 | nnOpts.gpus = 1; % for automatic selection use: SelectIdleGpu(); 46 | 47 | % Create outputFolder 48 | if ~exist(outputFolder, 'dir') 49 | mkdir(outputFolder); 50 | end 51 | 52 | % Start logging 53 | diary(logFilePath); 54 | 55 | %%% Setup 56 | % Start from pretrained network 57 | net = load(nnOpts.misc.netPath); 58 | 59 | % Setup imdb 60 | imdb = setupImdbDetection(trainName, testName, net); 61 | 62 | % Create calvinNN CNN class 63 | % By default, network is transformed into fast-rcnn with bbox regression 64 | calvinn = CalvinNN(net, imdb, nnOpts); 65 | 66 | %%% Train 67 | calvinn.train(); 68 | 69 | %%% Test 70 | stats = calvinn.test(); 71 | 72 | %%% Eval 73 | evalDetection(testName, imdb, stats, nnOpts); 74 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/misc/Cell2Matrix.m: -------------------------------------------------------------------------------- 1 | function [m, c] = Cell2Matrix(cell, dim) 2 | % m = Cell2Matrix(cell) 3 | % 4 | % Function concatenates all elements of a cell. 5 | % 6 | % cell: 1D cell structure. Elements in a cell should be 1D or 2D 7 | % matrices 8 | % dim: Dimension over which features are concatenated. 9 | % Default: 1. 10 | % 11 | % m: Concatenated elements of the cell. 12 | % c: Indices of row/col vectors denoting the cell index where 13 | % feature came from 14 | 15 | if nargin == 1 16 | dim = 1; 17 | end 18 | 19 | % Concatenate all features in row direction. 20 | if dim == 1 21 | % Calculate size for memory allocation 22 | mSize = 0; 23 | for i=1:length(cell) 24 | mSize = mSize + size(cell{i}, 1); 25 | end 26 | 27 | % Get number of columns for m 28 | nrC = 0; 29 | for i=1:length(cell) 30 | if size(cell{i}, 1) > 0 31 | nrC = size(cell{i}, 2); 32 | break; 33 | end 34 | end 35 | if nrC == 0 % All cells are empty 36 | m = []; 37 | c = []; 38 | return 39 | end 40 | 41 | % Concatenate everything. 42 | if isnumeric(class(cell{1})) 43 | m = zeros(mSize, nrC, class(cell{1})); 44 | else 45 | m = zeros(mSize, nrC); 46 | end 47 | c = zeros(mSize, 1); 48 | idx = 1; 49 | for i=1:length(cell) 50 | fSize = size(cell{i},1); 51 | if fSize ~= 0 52 | m(idx:idx+fSize-1, :) = cell{i}; 53 | c(idx:idx+fSize-1, 1) = i; 54 | end 55 | idx = idx + fSize; 56 | end 57 | end 58 | 59 | % concatenate all features in col direction. 60 | if dim == 2 61 | % Calculate size for memory allocation 62 | mSize = 0; 63 | for i=1:length(cell) 64 | mSize = mSize + size(cell{i}, 2); 65 | end 66 | 67 | % Concatenate everything. 68 | m = zeros(size(cell{1}, 1), mSize); 69 | c = zeros(1, mSize); 70 | idx = 1; 71 | for i=1:length(cell) 72 | fSize = size(cell{i},2); 73 | if fSize ~= 0 74 | m(:, idx:idx+fSize-1) = cell{i}; 75 | c(1, idx:idx+fSize-1) = i; 76 | end 77 | idx = idx + fSize; 78 | end 79 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/train.m: -------------------------------------------------------------------------------- 1 | function train(obj) 2 | % train(obj) 3 | % 4 | % Main training script used for training and validation. 5 | % Only <= 1 GPUs supported. 6 | % 7 | % Copyright by Holger Caesar, 2016 8 | 9 | modelPath = @(ep) fullfile(obj.nnOpts.expDir, sprintf('net-epoch-%d.mat', ep)); 10 | modelFigPath = fullfile(obj.nnOpts.expDir, 'net-train.pdf'); 11 | numGpus = numel(obj.nnOpts.gpus); 12 | assert(numGpus <= 1); 13 | 14 | % Load previous training snapshot 15 | lastCheckPoint = CalvinNN.findLastCheckpoint(obj.nnOpts.expDir); 16 | if isnan(lastCheckPoint) 17 | % No checkpoint found 18 | start = 0; 19 | 20 | % Save untrained net 21 | obj.saveState(modelPath(start)); 22 | else 23 | % Load existing checkpoint and continue 24 | start = obj.nnOpts.continue * lastCheckPoint; 25 | fprintf('Resuming by loading epoch %d\n', start); 26 | if start >= 1 27 | [obj.net, obj.stats] = CalvinNN.loadState(modelPath(start)); 28 | end 29 | end 30 | 31 | for epoch = start + 1 : obj.nnOpts.numEpochs 32 | 33 | % Set epoch and it's learning rate 34 | state.epoch = epoch; 35 | state.learningRate = obj.nnOpts.learningRate(min(epoch, numel(obj.nnOpts.learningRate))); 36 | 37 | % Set the current epoch in imdb 38 | obj.imdb.initEpoch(epoch); 39 | 40 | % Do training and validation 41 | datasetModes = {'train', 'val'}; 42 | for datasetModeIdx = 1 : numel(datasetModes) 43 | datasetMode = datasetModes{datasetModeIdx}; 44 | 45 | % Set train/val mode (disable Dropout etc.) 46 | obj.imdb.setDatasetMode(datasetMode); 47 | if strcmp(datasetMode, 'train'), 48 | obj.net.mode = 'train'; 49 | else % val 50 | obj.net.mode = 'test'; 51 | end; 52 | state.allBatchInds = obj.imdb.getAllBatchInds(); 53 | 54 | obj.stats.(datasetMode)(epoch) = obj.processEpoch(obj.net, state); 55 | end 56 | 57 | % Save current snapshot 58 | obj.saveState(modelPath(epoch)); 59 | 60 | % Plot statistics 61 | if obj.nnOpts.plotEval 62 | plotAccuracy = isfield(obj.stats.val, 'accuracy') && obj.nnOpts.plotAccuracy; 63 | obj.plotStats(1:epoch, obj.stats, plotAccuracy); 64 | 65 | drawnow; 66 | print(1, modelFigPath, '-dpdf'); %#ok 67 | end 68 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/isEqualFuncs.m: -------------------------------------------------------------------------------- 1 | function[result] = isEqualFuncs(a, b, ignoreFields) 2 | % [result] = isEqualFuncs(a, b, [ignoreFields]) 3 | % 4 | % Same as isequal(a, b), but: 5 | % - function handles are matched as strings. 6 | % - the order of fields in a struct does not matter. 7 | % 8 | % Copyright by Holger Caesar, 2014 9 | 10 | if ~exist('ignoreFields', 'var'), 11 | ignoreFields = {}; 12 | end; 13 | 14 | result = true; 15 | 16 | % Check size 17 | if any(size(a) ~= size(b)), 18 | result = false; 19 | return; 20 | end; 21 | 22 | if (isstruct(a) && isstruct(b)) ... 23 | || (isobject(a) && isobject(b)), 24 | % Structs and class objects 25 | 26 | % Check if field names of a and b are the same 27 | fieldsA = fieldnames(a); 28 | fieldsB = fieldnames(b); 29 | 30 | % Remove ignoreFields 31 | fieldsA = setdiff(fieldsA, ignoreFields); 32 | fieldsB = setdiff(fieldsB, ignoreFields); 33 | 34 | % Sort fields 35 | fieldsA = sort(fieldsA); 36 | fieldsB = sort(fieldsB); 37 | 38 | if ~isequal(fieldsA, fieldsB), 39 | result = false; 40 | differElements = setdiff(union(fieldsA, fieldsB), intersect(fieldsA, fieldsB)); 41 | fprintf('Warning: a and b do not have the same fields: %s\n', strjoin(differElements, ', ')); 42 | return; 43 | end; 44 | 45 | fieldCount = numel(fieldsA); 46 | for fieldIdx = 1 : fieldCount, 47 | fieldName = fieldsA{fieldIdx}; 48 | 49 | % Ignore specified fields 50 | if ismember(fieldName, ignoreFields), 51 | continue; 52 | end; 53 | 54 | if ~isEqualFuncs({a.(fieldName)}, {b.(fieldName)}, ignoreFields), 55 | result = false; 56 | fprintf('Warning: Field %s differs!\n', fieldName); 57 | return; 58 | end; 59 | end; 60 | elseif isa(a, 'function_handle') && isa(b, 'function_handle'), 61 | % Functions 62 | result = isequal(func2str(a), func2str(b)); 63 | elseif iscell(a) && iscell(b), 64 | % Cells 65 | for i = 1 : numel(a), 66 | if ~isEqualFuncs(a{i}, b{i}, ignoreFields), 67 | result = false; 68 | fprintf('Warning: Cell element %d differs!\n', i); 69 | return; 70 | end; 71 | end; 72 | else 73 | % Everything else 74 | result = isequal(a, b); 75 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/scoreBlobIoUs.m: -------------------------------------------------------------------------------- 1 | function[ious] = scoreBlobIoUs(blobsA, blobsB) 2 | % [ious] = scoreBlobIoUs(blobsA, blobsB) 3 | % 4 | % Score the intersection over union for each combination of blobs in a and b 5 | % 6 | % Copyright by Holger Caesar, 2014 7 | 8 | % % Matlab Coder (no significant speedups) 9 | % assert(isstruct(blobsA)); 10 | % assert(size(blobsA, 2) == 1); 11 | % assert(isa(blobsA(1).rect, 'double')); 12 | % assert(isa(blobsA(1).mask, 'logical')); 13 | % assert(isa(blobsA(1).size, 'double')); 14 | % assert(size(blobsA(1).rect, 2) == 4); 15 | % assert(size(blobsA(1).mask, 1) >= 1); 16 | % assert(size(blobsA(1).mask, 2) >= 1); 17 | % 18 | % assert(isstruct(blobsB)); 19 | % assert(size(blobsB, 2) == 1); 20 | % assert(isa(blobsB(1).rect, 'double')); 21 | % assert(isa(blobsB(1).mask, 'logical')); 22 | % assert(isa(blobsB(1).size, 'double')); 23 | % assert(size(blobsB(1).rect, 2) == 4); 24 | % assert(size(blobsB(1).mask, 1) >= 1); 25 | % assert(size(blobsB(1).mask, 2) >= 1); 26 | 27 | % Init 28 | nA = numel(blobsA); 29 | nB = numel(blobsB); 30 | ious = zeros(nA, nB); 31 | 32 | for aIdx = 1 : nA, 33 | for bIdx = 1 : nB, 34 | 35 | blobA = blobsA(aIdx); 36 | blobB = blobsB(bIdx); 37 | 38 | aRect = blobA.rect; 39 | bRect = blobB.rect; 40 | 41 | if isBoxIntersect(aRect, bRect), 42 | % If both boxes intersect, compute the IoU 43 | intersectionRect = [max(aRect(1:2), bRect(1:2)), min(aRect(3:4), bRect(3:4))]; 44 | 45 | aMaskCutRect = [intersectionRect(1:2) - aRect(1:2) + 1, intersectionRect(3:4) - aRect(1:2) + 1]; 46 | bMaskCutRect = [intersectionRect(1:2) - bRect(1:2) + 1, intersectionRect(3:4) - bRect(1:2) + 1]; 47 | 48 | aMaskCut = blobA.mask(aMaskCutRect(1):aMaskCutRect(3), aMaskCutRect(2):aMaskCutRect(4)); 49 | bMaskCut = blobB.mask(bMaskCutRect(1):bMaskCutRect(3), bMaskCutRect(2):bMaskCutRect(4)); 50 | 51 | intersection = sum(aMaskCut(:) & bMaskCut(:)); 52 | union = blobA.size + blobB.size - intersection; 53 | iou = intersection / union; 54 | else 55 | % If there is no intersection, set IoU to 0 56 | iou = 0; 57 | end; 58 | 59 | % Save iou 60 | ious(aIdx, bIdx) = iou; 61 | end; 62 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/storeLabelListGT.m: -------------------------------------------------------------------------------- 1 | function storeLabelListGT(varargin) 2 | % storeLabelListGT(varargin) 3 | % 4 | % Create the labelListGT entry for each ground-truth segmentation file of the dataset. 5 | % 6 | % Copyright by Holger Caesar, 2015 7 | 8 | % Parse input 9 | p = inputParser; 10 | addParameter(p, 'projectName', 'WeaklySupervisedLearning'); 11 | addParameter(p, 'dataset', SiftFlowDatasetMC()); 12 | addParameter(p, 'proposalNameGT', 'GroundTruth'); 13 | addParameter(p, 'subset', 'all'); 14 | parse(p, varargin{:}); 15 | 16 | projectName = p.Results.projectName; 17 | dataset = p.Results.dataset; 18 | proposalNameGT = p.Results.proposalNameGT; 19 | subset = p.Results.subset; 20 | 21 | % Create paths 22 | global glFeaturesFolder; 23 | segmentFolder = fullfile(glFeaturesFolder, projectName, dataset.name, 'segmentations', proposalNameGT); 24 | 25 | % Get list of images in dataset 26 | if strcmp(subset, 'all'), 27 | [imageList, imageCount] = dataset.getImageList(); 28 | elseif strcmp(subset, 'train'), 29 | [imageList, imageCount] = dataset.getImageListTrn(); 30 | elseif strcmp(subset, 'test'), 31 | [imageList, imageCount] = dataset.getImageListTst(); 32 | else 33 | error('Error: Unknown subset: %s', subset); 34 | end; 35 | 36 | for imageIdx = 1 : imageCount, 37 | printProgress('Processing image', imageIdx, imageCount, 50); 38 | 39 | % Get segmentation 40 | imageName = imageList{imageIdx}; 41 | segmentPath = [segmentFolder, filesep, imageName, '.mat']; 42 | segmentStruct = load(segmentPath); 43 | if isfield(segmentStruct, 'labelListGT'), 44 | fprintf('Skipping existing file %s...\n', segmentPath); 45 | continue; 46 | end; 47 | propBlobs = segmentStruct.propBlobs; 48 | blobCount = numel(propBlobs); 49 | 50 | % Get labelMap 51 | labelMap = dataset.getImLabelMap(imageName); 52 | labelListGT = nan(blobCount, 1); 53 | 54 | for blobIdx = 1 : blobCount, 55 | % Get labels for that blob 56 | blob = propBlobs(blobIdx); 57 | blobInds = blobToImageInds(blob, size(labelMap)); 58 | blobLabel = unique(labelMap(blobInds)); 59 | assert(numel(blobLabel) == 1 && blobLabel > 0); 60 | labelListGT(blobIdx) = blobLabel; 61 | end; 62 | 63 | % Sanity check 64 | assert(~any(isnan(labelListGT))); 65 | 66 | % Append to file 67 | save(segmentPath, 'labelListGT', '-append'); 68 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/+dagnn/SegmentationAccuracyFlexible.m: -------------------------------------------------------------------------------- 1 | classdef SegmentationAccuracyFlexible < dagnn.Loss 2 | % Copyright by Matconvnet 3 | % Modified by Holger Caesar, 2016 4 | % - Changed number of classes from 21 to obj.labelCount 5 | 6 | properties 7 | labelCount = []; 8 | end 9 | 10 | properties (Transient) 11 | pixelAccuracy = 0 12 | meanAccuracy = 0 13 | meanIntersectionUnion = 0 14 | confusion = 0 15 | end 16 | 17 | methods 18 | function outputs = forward(obj, inputs, params) %#ok 19 | 20 | % Check settings 21 | assert(~isempty(obj.labelCount)); 22 | 23 | [~, predictions] = max(inputs{1}, [], 3); 24 | predictions = gather(predictions); 25 | labels = gather(inputs{2}); 26 | 27 | % compute statistics only on accumulated pixels 28 | ok = labels > 0; 29 | numPixels = sum(ok(:)); 30 | obj.confusion = obj.confusion + accumarray([labels(ok), predictions(ok)], 1, [obj.labelCount, obj.labelCount]); 31 | 32 | % Compute accuracies 33 | [obj.pixelAccuracy, obj.meanAccuracy, obj.meanIntersectionUnion] = confMatToAccuracies(obj.confusion); 34 | 35 | obj.average = [obj.pixelAccuracy; obj.meanAccuracy; obj.meanIntersectionUnion]; 36 | obj.numAveraged = obj.numAveraged + numPixels; 37 | outputs{1} = obj.average; 38 | end 39 | 40 | function [derInputs, derParams] = backward(obj, inputs, params, derOutputs) %#ok 41 | derInputs{1} = []; 42 | derInputs{2} = []; 43 | derParams = {}; 44 | end 45 | 46 | function reset(obj) 47 | obj.confusion = 0; 48 | obj.pixelAccuracy = 0; 49 | obj.meanAccuracy = 0; 50 | obj.meanIntersectionUnion = 0; 51 | obj.average = [0; 0; 0]; 52 | obj.numAveraged = 0; 53 | end 54 | 55 | function str = toString(obj) 56 | str = sprintf('acc:%.2f, mAcc:%.2f, mIU:%.2f', ... 57 | obj.pixelAccuracy, obj.meanAccuracy, obj.meanIntersectionUnion); 58 | end 59 | 60 | function obj = SegmentationAccuracyFlexible(varargin) 61 | obj.load(varargin); 62 | end 63 | end 64 | end -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/calvinNNClassification.m: -------------------------------------------------------------------------------- 1 | % function calvinNNClassification() 2 | % 3 | % Copyright by Holger Caesar, 2016 4 | 5 | % Global variables 6 | global glDatasetFolder glFeaturesFolder; 7 | assert(~isempty(glDatasetFolder) && ~isempty(glFeaturesFolder)); 8 | 9 | %%% Settings 10 | % Dataset 11 | vocYear = 2010; 12 | trainName = 'train'; 13 | testName = 'val'; 14 | 15 | % Specify paths 16 | vocName = sprintf('VOC%d', vocYear); 17 | datasetDir = [fullfile(glDatasetFolder, vocName), '/']; 18 | outputFolder = fullfile(glFeaturesFolder, 'CNN-Models', 'CLS', vocName, sprintf('%s-testRelease', vocName)); 19 | netPath = fullfile(glFeaturesFolder, 'CNN-Models', 'matconvnet', 'imagenet-vgg-verydeep-16.mat'); 20 | logFilePath = fullfile(outputFolder, 'log.txt'); 21 | 22 | % Fix randomness 23 | randSeed = 42; 24 | rng(randSeed); 25 | 26 | % Setup dataset specific options and check validity 27 | setupDataOpts(vocYear, testName, datasetDir); 28 | global DATAopts; 29 | assert(~isempty(DATAopts), 'Error: Dataset not initialized properly!'); 30 | 31 | % Task-specific 32 | nnOpts.testFn = @testClassification; 33 | nnOpts.lossFnObjective = 'hinge'; 34 | nnOpts.derOutputs = {'objective', single(1)}; 35 | 36 | % Disable Fast R-CNN (default is on) 37 | nnOpts.fastRcnn = false; 38 | nnOpts.fastRcnnParams = false; % learning rates and weight decay 39 | nnOpts.misc.roiPool.use = false; 40 | nnOpts.misc.roiPool.freeform.use = false; 41 | nnOpts.bboxRegress = false; 42 | 43 | % General 44 | nnOpts.batchSize = 64; 45 | nnOpts.numSubBatches = 1; % 64 images per sub-batch 46 | nnOpts.weightDecay = 5e-4; 47 | nnOpts.momentum = 0.9; 48 | nnOpts.numEpochs = 16; 49 | nnOpts.learningRate = [repmat(1e-3, 12, 1); repmat(1e-4, 4, 1)]; 50 | nnOpts.misc.netPath = netPath; 51 | nnOpts.gpus = 1; % for automatic selection use: SelectIdleGpu(); 52 | nnOpts.expDir = outputFolder; 53 | 54 | % Create outputFolder 55 | if ~exist(outputFolder, 'dir') 56 | mkdir(outputFolder); 57 | end 58 | 59 | % Start logging 60 | diary(logFilePath); 61 | 62 | %%% Setup 63 | % Start from pretrained network 64 | net = load(nnOpts.misc.netPath); 65 | 66 | % Setup imdb 67 | imdb = setupImdbClassification(trainName, testName, net); 68 | imdb.targetImSize = [224, 224]; 69 | 70 | % Create calvinNN CNN class 71 | calvinn = CalvinNN(net, imdb, nnOpts); 72 | 73 | %%% Train 74 | calvinn.train(); 75 | 76 | %%% Test 77 | stats = calvinn.test(); 78 | 79 | %%% Eval 80 | evalClassification(imdb, stats, nnOpts); 81 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/setupImdbClassification.m: -------------------------------------------------------------------------------- 1 | function[imdb] = setupImdbClassification(trainName, testName, net) 2 | % [imdb] = setupImdbClassification(trainName, testName, net) 3 | 4 | global DATAopts; 5 | 6 | % Setup the Imdb 7 | % Get images and labels 8 | set = trainName; 9 | [trainIms, ~] = textread(sprintf(DATAopts.imgsetpath,set),'%s %d'); %#ok 10 | trainLabs = zeros(length(trainIms), DATAopts.nclasses); 11 | 12 | % Get all testlabels 13 | if ~strcmp(set,'test') || DATAopts.year == 2007 14 | for classIdx = 1 : DATAopts.nclasses 15 | class = DATAopts.classes{classIdx}; 16 | [~, trainLabs(:,classIdx)] = textread(sprintf(DATAopts.clsimgsetpath, class, set), '%s %d'); %#ok 17 | end 18 | 19 | trainLabs(trainLabs == 0) = -1; 20 | end 21 | 22 | set = testName; 23 | [testIms, ~] = textread(sprintf(DATAopts.imgsetpath,set),'%s %d'); %#ok 24 | testLabs = zeros(length(testIms), DATAopts.nclasses); 25 | 26 | % Get all testlabels 27 | if ~strcmp(set,'test') || DATAopts.year == 2007 28 | for classIdx = 1 : DATAopts.nclasses 29 | class = DATAopts.classes{classIdx}; 30 | [~, testLabs(:, classIdx)] = textread(sprintf(DATAopts.clsimgsetpath, class, set), '%s %d'); %#ok 31 | end 32 | 33 | testLabs(testLabs == -1) = 0; 34 | end 35 | 36 | % Make train, val, and test set. 37 | numValIms = 500; 38 | allIms = cat(1, trainIms, testIms); 39 | allLabs = cat(1, trainLabs, testLabs); 40 | datasetIdx{1} = (1 : length(trainIms) - numValIms)'; % Last numValIms are val set 41 | datasetIdx{2} = (length(trainIms) - numValIms + 1 : length(trainIms))'; 42 | datasetIdx{3} = (length(trainIms) + 1 : length(allIms))'; 43 | 44 | % Setup the classification imdb 45 | imdb = ImdbClassification(DATAopts.imgpath(1:end-6), ... % path 46 | DATAopts.imgpath(end-3:end), ... % image extension 47 | allIms, ... % all images 48 | allLabs, ... % all labels 49 | datasetIdx, ... % division into train/val/test 50 | net.meta.normalization.averageImage, ... % average image used to pretrain network 51 | 20); % num classes 52 | 53 | % Store lists for use in eval 54 | imdb.misc.trainName = trainName; 55 | imdb.misc.testName = testName; 56 | imdb.misc.trainIms = trainIms; 57 | imdb.misc.trainLabs = trainLabs; 58 | imdb.misc.testIms = testIms; 59 | imdb.misc.testLabs = testLabs; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/fcn/fcnInitializeModel16sGeneric.m: -------------------------------------------------------------------------------- 1 | function net = fcnInitializeModel16sGeneric(labelCount, net) 2 | %FCNINITIALIZEMODEL16S Initialize the FCN-16S model from FCN-32 3 | 4 | %% Remove the last layer 5 | net.removeLayer('deconv32') ; 6 | 7 | %% Add the first Deconv layer 8 | filters = single(bilinear_u(4, 1, labelCount)) ; 9 | net.addLayer('deconv2', ... 10 | dagnn.ConvTranspose('size', size(filters), ... 11 | 'upsample', 2, ... 12 | 'crop', [1 1 1 1], ... 13 | 'hasBias', false), ... 14 | 'x38', 'x39', 'deconv1f') ; 15 | f = net.getParamIndex('deconv1f') ; 16 | net.params(f).value = filters ; 17 | net.params(f).learningRate = 0 ; 18 | net.params(f).weightDecay = 1 ; 19 | 20 | %% Add skip layers on top of pool4 21 | net.addLayer('skip4', ... 22 | dagnn.Conv('size', [1 1 512 labelCount], 'pad', 0), ... 23 | 'x24', 'x40', {'skip4f','skip4b'}); 24 | 25 | f = net.getParamIndex('skip4f') ; 26 | net.params(f).value = zeros(1, 1, 512, labelCount, 'single') ; 27 | net.params(f).learningRate = 0.1 ; 28 | net.params(f).weightDecay = 1 ; 29 | 30 | f = net.getParamIndex('skip4b') ; 31 | net.params(f).value = zeros(1, 1, labelCount, 'single') ; 32 | net.params(f).learningRate = 2 ; 33 | net.params(f).weightDecay = 1 ; 34 | 35 | %% Add the summation layer 36 | net.addLayer('sum1', dagnn.Sum(), {'x39', 'x40'}, 'x41') ; 37 | 38 | %% Add deconvolutional layer implementing bilinear interpolation 39 | filters = single(bilinear_u(32, labelCount, labelCount)) ; 40 | net.addLayer('deconv16', ... 41 | dagnn.ConvTranspose('size', size(filters), ... 42 | 'upsample', 16, ... 43 | 'crop', 8, ... 44 | 'numGroups', labelCount, ... 45 | 'hasBias', false), ... 46 | 'x41', 'prediction', 'deconv16f') ; 47 | 48 | f = net.getParamIndex('deconv16f') ; 49 | net.params(f).value = filters ; 50 | net.params(f).learningRate = 0 ; 51 | net.params(f).weightDecay = 1 ; 52 | 53 | % Make sure that the output of the bilinear interpolator is not discared for 54 | % visualization purposes 55 | net.vars(net.getVarIndex('prediction')).precious = 1 ; 56 | 57 | % empirical test 58 | if 0 59 | figure(100) ; clf ; 60 | n = numel(net.vars) ; 61 | for i=1:n 62 | vl_tightsubplot(n,i) ; 63 | showRF(net, 'input', net.vars(i).name) ; 64 | title(sprintf('%s', net.vars(i).name)) ; 65 | drawnow ; 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/fcn/fcnInitializeModel8sGeneric.m: -------------------------------------------------------------------------------- 1 | function net = fcnInitializeModel8sGeneric(labelCount, net) 2 | %FCNINITIALIZEMODEL8S Initialize the FCN-16S model from FCN-32 3 | 4 | %% Remove the last layer 5 | net.removeLayer('deconv16') ; 6 | 7 | %% Add the first deconv layer 8 | filters = single(bilinear_u(4, 1, labelCount)) ; 9 | net.addLayer('deconv2bis', ... 10 | dagnn.ConvTranspose('size', size(filters), ... 11 | 'upsample', 2, ... 12 | 'crop', 1, ... 13 | 'hasBias', false), ... 14 | 'x41', 'x42', 'deconv2bisf') ; 15 | f = net.getParamIndex('deconv2bisf') ; 16 | net.params(f).value = filters ; 17 | net.params(f).learningRate = 0 ; 18 | net.params(f).weightDecay = 1 ; 19 | 20 | %% Add a convolutional layer that take as input the pool3 layer 21 | net.addLayer('skip3', ... 22 | dagnn.Conv('size', [1 1 256 labelCount]), ... 23 | 'x17', 'x43', {'skip3f','skip3b'}); 24 | 25 | f = net.getParamIndex('skip3f') ; 26 | net.params(f).value = zeros(1, 1, 256, labelCount, 'single') ; 27 | net.params(f).learningRate = 0.01 ; 28 | net.params(f).weightDecay = 1 ; 29 | 30 | f = net.getParamIndex('skip3b') ; 31 | net.params(f).value = zeros(1, 1, labelCount, 'single') ; 32 | net.params(f).learningRate = 2 ; 33 | net.params(f).weightDecay = 1 ; 34 | 35 | %% Add the sumwise layer 36 | net.addLayer('sum2', dagnn.Sum(), {'x42', 'x43'}, 'x44') ; 37 | 38 | %% Add deconvolutional layer implementing bilinear interpolation 39 | filters = single(bilinear_u(16, labelCount, labelCount)) ; 40 | net.addLayer('deconv8', ... 41 | dagnn.ConvTranspose('size', size(filters), ... 42 | 'upsample', 8, ... 43 | 'crop', 4, ... 44 | 'numGroups', labelCount, ... 45 | 'hasBias', false), ... 46 | 'x44', 'prediction', 'deconv8f') ; 47 | 48 | f = net.getParamIndex('deconv8f') ; 49 | net.params(f).value = filters ; 50 | net.params(f).learningRate = 0 ; 51 | net.params(f).weightDecay = 1 ; 52 | 53 | % Make the output of the bilinear interpolator is not discared for 54 | % visualization purposes 55 | net.vars(net.getVarIndex('prediction')).precious = 1 ; 56 | 57 | % empirical test 58 | if 0 59 | figure(100) ; clf ; 60 | n = numel(net.vars) ; 61 | for i=1:n 62 | vl_tightsubplot(n,i) ; 63 | showRF(net, 'input', net.vars(i).name) ; 64 | title(sprintf('%s', net.vars(i).name)) ; 65 | drawnow ; 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/setup/setupE2S2Regions.m: -------------------------------------------------------------------------------- 1 | function setupE2S2Regions(varargin) 2 | % setupE2S2Regions(varargin) 3 | % 4 | % Bundles all pre-processing steps to extract region proposals and labels 5 | % (our method requires Selective Search and Felzenszwalb 6 | % oversegmentations). This can take about an hour for a medium-sized 7 | % dataset (3K images). 8 | % 9 | % Copyright by Holger Caesar, 2016 10 | 11 | % Parse input 12 | p = inputParser; 13 | addParameter(p, 'dataset', SiftFlowDatasetMC()); 14 | addParameter(p, 'colorSpaces', {'Rgb', 'Hsv', 'Lab'}); 15 | parse(p, varargin{:}); 16 | 17 | dataset = p.Results.dataset; 18 | colorSpaces = p.Results.colorSpaces; 19 | 20 | % Settings 21 | global glFeaturesFolder; 22 | segmentationsFolder = fullfile(glFeaturesFolder, 'WeaklySupervisedLearning', dataset.name, 'segmentations'); 23 | 24 | if ~exist(segmentationsFolder, 'dir') 25 | % Gound-truth 26 | rpExtract('dataset', dataset, 'proposalName', 'GroundTruth'); 27 | 28 | % Store GT labels for quick access 29 | storeLabelListGT('dataset', dataset); 30 | 31 | % Extract downsized copy of ground truth regions masks 32 | e2s2_storeBlobMasks('dataset', dataset, 'proposalName', 'GroundTruth'); 33 | 34 | % Perform the same operations on each color space 35 | for colorSpaceIdx = 1 : numel(colorSpaces) 36 | colorSpace = colorSpaces{colorSpaceIdx}; 37 | proposalNameSP = sprintf('Felzenszwalb2004-k100-sigma0.8-colorTypes%s', colorSpace); 38 | proposalNameRP = sprintf('Uijlings2013-ks100-sigma0.8-colorTypes%s', colorSpace); 39 | 40 | % Selective Search 41 | rpExtract('dataset', dataset, 'proposalName', 'Uijlings2013', 'proposalsVars', {'colorTypes', {colorSpace}}); 42 | 43 | % Felzenszwalb 44 | rpExtract('dataset', dataset, 'proposalName', 'Felzenszwalb2004', 'proposalsVars', {'colorTypes', {colorSpace}}); 45 | 46 | % Reconstruct region proposal hierarchy 47 | reconstructSelSearchHierarchyFromFz('dataset', dataset, 'segmentationNameSS', proposalNameRP, 'segmentationNameFz', proposalNameSP); 48 | 49 | % Extract downsized copy of each regions mask 50 | e2s2_storeBlobMasks('dataset', dataset, 'proposalName', proposalNameRP); 51 | 52 | % Compute the overlaps between ground-truth and superpixels (relevant 53 | % for evaluation) 54 | e2s2_storeSPGTOverlap('dataset', dataset, 'spName', proposalNameSP); 55 | end 56 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/rpUijlings2013.m: -------------------------------------------------------------------------------- 1 | function[blobs] = rpUijlings2013(image, varargin) 2 | % [blobs] = rpUijlings2013(image, varargin) 3 | % 4 | % Create a list of region proposals (blobs) of an image using the method of Uijlings2013. 5 | % 6 | % For information on the returned blobs see blob(). 7 | % 8 | % Copyright by Holger Caesar, 2014 9 | 10 | % Parse input 11 | p = inputParser; 12 | % Default settings suggested by Jasper 13 | addParameter(p, 'ks', [50, 100]); 14 | addParameter(p, 'sigma', 0.8); 15 | addParameter(p, 'colorTypes', {'Hsv', 'Lab'}); 16 | addParameter(p, 'simFunctionHandles', {@SSSimColourTextureSizeFillOrig, @SSSimTextureSizeFill}); 17 | parse(p, varargin{:}); 18 | 19 | ks = p.Results.ks; 20 | sigma = p.Results.sigma; 21 | colorTypes = p.Results.colorTypes; 22 | simFunctionHandles = p.Results.simFunctionHandles; 23 | 24 | % Init SS 25 | hierarchyCount = numel(ks) * numel(colorTypes) * numel(simFunctionHandles); 26 | blobs = cell(hierarchyCount, 1); 27 | priorities = cell(hierarchyCount, 1); 28 | hBlobsInd = 1; 29 | 30 | % Perform Selective Search on all parameter combinations 31 | for ksInd = 1 : numel(ks), 32 | for colorTypeInd = 1 : numel(colorTypes), 33 | for simHndsI = 1 : numel(simFunctionHandles), 34 | % Compute hierarchical grouping with current settings 35 | [~, blobIndIm, blobBoxesInIm, hierarchy, priority] = Image2HierarchicalGrouping(image, sigma, ks(ksInd), ks(ksInd), colorTypes{colorTypeInd}, simFunctionHandles(simHndsI)); 36 | 37 | % Convert boxes to blobs (when there is only one blob, we can't 38 | % create a hierarchy, i.e. for k=400) 39 | if isempty(hierarchy), 40 | blobs{hBlobsInd} = RecreateBlobHierarchyIndIm(blobIndIm, blobBoxesInIm, []); 41 | else 42 | blobs{hBlobsInd} = RecreateBlobHierarchyIndIm(blobIndIm, blobBoxesInIm, hierarchy{1}); 43 | end; 44 | 45 | % Store priority 46 | priorities{hBlobsInd} = priority; 47 | hBlobsInd = hBlobsInd + 1; 48 | end; 49 | end; 50 | end; 51 | 52 | % Put all parameter results in a single cell array 53 | blobs = flattenCellArray(blobs); 54 | priorities = flattenCellArray(priorities); 55 | 56 | % Do pseudo random sorting as in paper 57 | priorities = priorities .* rand(size(priorities)); 58 | [~, sortIds] = sort(priorities, 'ascend'); 59 | blobs = blobs(sortIds, :); 60 | 61 | % Convert to col struct 62 | blobs = [blobs{:}]; 63 | blobs = blobs(:); -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxRegressionTarget.m: -------------------------------------------------------------------------------- 1 | function regressionTarget = BoxRegressionTarget(gtBox, trainBox) 2 | % regressionTarget = BoxRegressionTarget(gtBox, trainBox) 3 | % 4 | % Obtain regression factors in terms of four scalars with respect to the centre box 5 | % Jasper: This is a simplified and more intuitive way of doing BB regression than 6 | % what is proposed by Girshick. Regression factors of 0 means no scaling. 7 | % Note that gtBox = BoxRegresss(trainBox, regressTarget) 8 | % gtBox(1) = trainBoxCenter - 1/2 trainBoxWidth + F * 1/2 trainBoxWidth 9 | % 10 | % 11 | % gtBox: N x 4 OR 1 x 4 vector with target BB coordinates 12 | % trainBox: N x 4 OR 1 x 4 vector with train BB coordinates 13 | % 14 | % regressionTarget: N x 4 vector with regression factors for each trainBox 15 | % 16 | % Jasper Uijlings - 2015 17 | 18 | % Get middles 19 | middleOdd = (trainBox(:,1) + trainBox(:,3)) / 2; 20 | middleEven = (trainBox(:,2) + trainBox(:,4)) / 2; 21 | 22 | % Get scaling factors 23 | regressionTarget(:,4) = (gtBox(:,4) - middleEven) ./ (trainBox(:,4) - middleEven); 24 | regressionTarget(:,3) = (gtBox(:,3) - middleOdd) ./ (trainBox(:,3) - middleOdd); 25 | regressionTarget(:,2) = (gtBox(:,2) - middleEven) ./ (trainBox(:,2) - middleEven); 26 | regressionTarget(:,1) = (gtBox(:,1) - middleOdd) ./ (trainBox(:,1) - middleOdd); 27 | 28 | % We want zero-mean regression targets. 29 | regressionTarget = regressionTarget - 1; 30 | 31 | 32 | %%% Debugging function 33 | % function test 34 | % 35 | % figure(1); clf; 36 | % 37 | % gtBox = [0 300 100 500]; 38 | % 39 | % drawBoxes(gtBox, 1, 'r'); 40 | % 41 | % trainBoxes = [0 300 100 500; 42 | % 0 300 100 500]; 43 | % 44 | % displacement = 10; 45 | % displacementDim = 2; 46 | % trainBoxes(1,displacementDim) = trainBoxes(1,displacementDim) + displacement; 47 | % trainBoxes(2,displacementDim) = trainBoxes(2,displacementDim) - displacement 48 | % 49 | % rT = BoxRegressionTarget(gtBox, trainBoxes) 50 | % BoxRegresss(trainBoxes, rT) 51 | % 52 | % 53 | % drawBoxes(trainBoxes(1,:), 1, 'g-'); 54 | % drawBoxes(trainBoxes(2,:), 1, 'b-'); 55 | % 56 | % 57 | % function drawBoxes(boxes, figN, style) 58 | % 59 | % figure(figN); 60 | % hold on; 61 | % 62 | % if nargin ~= 3 63 | % style = 'r-'; 64 | % end 65 | % 66 | % for i=1:size(boxes,1) 67 | % box = boxes(i,:); 68 | % plot(box([1 1]), box([2 4]), style); 69 | % plot(box([3 3]), box([2 4]), style); 70 | % plot(box([1 3]), box([2 2]), style); 71 | % plot(box([1 3]), box([4 4]), style); 72 | % end 73 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxRegressionTargetLog.m: -------------------------------------------------------------------------------- 1 | function regressionTarget = BoxRegressionTargetLog(gtBox, trainBox) 2 | % regressionTarget = BoxRegressionTarget(gtBox, trainBox) 3 | % 4 | % Obtain regression factors in terms of four scalars with respect to the centre box 5 | % Jasper: This is a simplified and more intuitive way of doing BB regression than 6 | % what is proposed by Girshick. Regression factors of 0 means no scaling. 7 | % Note that gtBox = BoxRegresss(trainBox, regressTarget) 8 | % gtBox(1) = trainBoxCenter - 1/2 trainBoxWidth + F * 1/2 trainBoxWidth 9 | % 10 | % 11 | % gtBox: N x 4 OR 1 x 4 vector with target BB coordinates 12 | % trainBox: N x 4 OR 1 x 4 vector with train BB coordinates 13 | % 14 | % regressionTarget: N x 4 vector with regression factors for each trainBox 15 | % 16 | % In log-space, like Girshick 17 | % 18 | % Jasper Uijlings - 2015 19 | 20 | % Get middles 21 | middleOdd = (trainBox(:,1) + trainBox(:,3)) / 2; 22 | middleEven = (trainBox(:,2) + trainBox(:,4)) / 2; 23 | 24 | % Get scaling factors 25 | regressionTarget(:,4) = (gtBox(:,4) - middleEven) ./ (trainBox(:,4) - middleEven); 26 | regressionTarget(:,3) = (gtBox(:,3) - middleOdd) ./ (trainBox(:,3) - middleOdd); 27 | regressionTarget(:,2) = (gtBox(:,2) - middleEven) ./ (trainBox(:,2) - middleEven); 28 | regressionTarget(:,1) = (gtBox(:,1) - middleOdd) ./ (trainBox(:,1) - middleOdd); 29 | 30 | % We want zero-mean regression targets. 31 | regressionTarget = log(regressionTarget / 2 + 0.5) * 6.537; 32 | 33 | 34 | %%% Debugging function 35 | % function test 36 | % 37 | % figure(1); clf; 38 | % 39 | % gtBox = [0 300 100 500]; 40 | % 41 | % drawBoxes(gtBox, 1, 'r'); 42 | % 43 | % trainBoxes = [0 300 100 500; 44 | % 0 300 100 500]; 45 | % 46 | % displacement = 10; 47 | % displacementDim = 2; 48 | % trainBoxes(1,displacementDim) = trainBoxes(1,displacementDim) + displacement; 49 | % trainBoxes(2,displacementDim) = trainBoxes(2,displacementDim) - displacement 50 | % 51 | % rT = BoxRegressionTarget(gtBox, trainBoxes) 52 | % BoxRegresss(trainBoxes, rT) 53 | % 54 | % 55 | % drawBoxes(trainBoxes(1,:), 1, 'g-'); 56 | % drawBoxes(trainBoxes(2,:), 1, 'b-'); 57 | % 58 | % 59 | % function drawBoxes(boxes, figN, style) 60 | % 61 | % figure(figN); 62 | % hold on; 63 | % 64 | % if nargin ~= 3 65 | % style = 'r-'; 66 | % end 67 | % 68 | % for i=1:size(boxes,1) 69 | % box = boxes(i,:); 70 | % plot(box([1 1]), box([2 4]), style); 71 | % plot(box([3 3]), box([2 4]), style); 72 | % plot(box([1 3]), box([2 2]), style); 73 | % plot(box([1 3]), box([4 4]), style); 74 | % end 75 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/BoxRegressionTargetGirshick.m: -------------------------------------------------------------------------------- 1 | function regressionTarget = BoxRegressionTargetGirshick(gtBox, trainBox) 2 | % regressionTarget = BoxRegressionTarget(gtBox, trainBox) 3 | % 4 | % Obtain regression factors as specified by Girshick. The first two coordinates 5 | % represent the offset wrt the middle of the box. The last two coordinates 6 | % represent the scaling of the box in log-space. 7 | % 8 | % gtBox: N x 4 OR 1 x 4 vector with target BB coordinates 9 | % trainBox: N x 4 OR 1 x 4 vector with train BB coordinates 10 | % 11 | % regressionTarget: N x 4 vector with regression factors for each trainBox 12 | % 13 | % Jasper Uijlings - 2015 14 | 15 | % Extract widths and middles (in Row and Col direction) 16 | [trainR, trainC] = BoxSize(trainBox); 17 | [gtR, gtC] = BoxSize(gtBox); 18 | middleR = (trainBox(:,1) + trainBox(:,3)) / 2; 19 | middleC = (trainBox(:,2) + trainBox(:,4)) / 2; 20 | middleRGt = (gtBox(:,1) + gtBox(:,3)) / 2; 21 | middleCGt = (gtBox(:,2) + gtBox(:,4)) / 2; 22 | 23 | % Get scaling factors 24 | regressionTarget(:,4) = log(gtC ./ trainC); 25 | regressionTarget(:,3) = log(gtR ./ trainR); 26 | 27 | % Get offset 28 | regressionTarget(:,2) = (middleCGt - middleC) ./ trainC; 29 | regressionTarget(:,1) = (middleRGt - middleR) ./ trainR; 30 | 31 | % We want zero-mean regression targets. Empirically determined on Pascal VOC 2007 32 | % regressionTarget = regressionTarget - 1; 33 | regressionTarget = bsxfun(@rdivide, regressionTarget, [0.1131 0.1277 0.2173 0.2173]); 34 | 35 | 36 | %%% Debugging function 37 | % function test 38 | % 39 | % figure(1); clf; 40 | % 41 | % gtBox = [0 300 100 500]; 42 | % 43 | % drawBoxes(gtBox, 1, 'r'); 44 | % 45 | % trainBoxes = [0 300 100 500; 46 | % 0 300 100 500]; 47 | % 48 | % displacement = 10; 49 | % displacementDim = 2; 50 | % trainBoxes(1,displacementDim) = trainBoxes(1,displacementDim) + displacement; 51 | % trainBoxes(2,displacementDim) = trainBoxes(2,displacementDim) - displacement 52 | % 53 | % rT = BoxRegressionTarget(gtBox, trainBoxes) 54 | % BoxRegresss(trainBoxes, rT) 55 | % 56 | % 57 | % drawBoxes(trainBoxes(1,:), 1, 'g-'); 58 | % drawBoxes(trainBoxes(2,:), 1, 'b-'); 59 | % 60 | % 61 | % function drawBoxes(boxes, figN, style) 62 | % 63 | % figure(figN); 64 | % hold on; 65 | % 66 | % if nargin ~= 3 67 | % style = 'r-'; 68 | % end 69 | % 70 | % for i=1:size(boxes,1) 71 | % box = boxes(i,:); 72 | % plot(box([1 1]), box([2 4]), style); 73 | % plot(box([3 3]), box([2 4]), style); 74 | % plot(box([1 3]), box([2 2]), style); 75 | % plot(box([1 3]), box([4 4]), style); 76 | % end 77 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/convertNetwork.m: -------------------------------------------------------------------------------- 1 | function convertNetwork(obj) 2 | % convertNetwork(obj) 3 | % 4 | % Converts a network from test to train, inserting lossed, dropout and 5 | % adopting the classification layer. 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | fprintf('Converting test network to train network (dropout, loss, etc.)...\n'); 10 | 11 | % Add dropout layers after relu6 and relu7 12 | fprintf('Adding dropout to network...\n'); 13 | dropout6Layer = dagnn.DropOut(); 14 | dropout7Layer = dagnn.DropOut(); 15 | insertLayer(obj.net, 'relu6', 'fc7', 'dropout6', dropout6Layer); 16 | insertLayer(obj.net, 'relu7', 'fc8', 'dropout7', dropout7Layer); 17 | 18 | % Rename variable prob to x%d+ style to make insertLayer work 19 | % (Required for beta18 or later matconvnet default networks) 20 | predVarIdx = obj.net.getVarIndex('prediction'); 21 | if ~isnan(predVarIdx) 22 | freeVarName = getFreeVariable(obj.net); 23 | obj.net.renameVar('prediction', freeVarName); 24 | end 25 | 26 | % Replace softmax with correct loss for training (default: softmax) 27 | switch obj.nnOpts.lossFnObjective 28 | case 'softmaxlog' 29 | softmaxlossBlock = dagnn.LossWeighted('loss', 'softmaxlog'); 30 | replaceLayer(obj.net, 'prob', 'softmaxloss', softmaxlossBlock, 'label'); 31 | objectiveName = obj.net.layers(obj.net.getLayerIndex('softmaxloss')).outputs; 32 | obj.net.renameVar(objectiveName, 'objective'); 33 | case 'hinge' 34 | hingeLossBlock = dagnn.Loss('loss', 'hinge'); 35 | replaceLayer(obj.net, 'prob', 'hingeloss', hingeLossBlock, 'label'); 36 | objectiveName = obj.net.layers(obj.net.getLayerIndex('hingeloss')).outputs; 37 | obj.net.renameVar(objectiveName, 'objective'); 38 | otherwise 39 | error('Unknown loss specified'); 40 | end 41 | 42 | % Adapt number of classes in softmaxloss layer from 1000 to numClasses 43 | fc8Idx = obj.net.getLayerIndex('fc8'); 44 | obj.net.layers(fc8Idx).block.size(4) = obj.imdb.numClasses; 45 | newParams = obj.net.layers(fc8Idx).block.initParams(); 46 | obj.net.params(obj.net.layers(fc8Idx).paramIndexes(1)).value = newParams{1} / std(newParams{1}(:)) * 0.01; % Girshick initialization 47 | obj.net.params(obj.net.layers(fc8Idx).paramIndexes(2)).value = newParams{2}'; 48 | 49 | % Rename input 50 | if ~isnan(obj.net.getVarIndex('x0')) 51 | % Input variable x0 is renamed to input 52 | obj.net.renameVar('x0', 'input'); 53 | else 54 | % Input variable already has the correct name 55 | assert(~isnan(obj.net.getVarIndex('input'))); 56 | end 57 | 58 | % Modify for Fast Rcnn (ROI pooling, bbox regression etc.) 59 | if obj.nnOpts.fastRcnn 60 | obj.convertNetworkToFastRcnn(); 61 | end -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/GetGTAndSSBoxes.m: -------------------------------------------------------------------------------- 1 | function boxStruct = GetGTAndSSBoxes(imName) 2 | % Using the imName, get the GT bounding boxes and the Selective Search bounding boxes 3 | % to create Girshick-style RCNN features afterwards 4 | % 5 | % The boxStruct contains the fields: 6 | % boxes: N x 4 (single) 7 | % class: N x 1 (uint8) 8 | % gt: N x 1 (logical) 9 | % is_difficult:N x 1 (logival) 10 | % overlap: N x 200 (double) 11 | 12 | global DATAopts; 13 | 14 | record = VOCreadxml(sprintf(DATAopts.annopath, imName)); 15 | 16 | class_to_id = containers.Map(DATAopts.classes, 1:length(DATAopts.classes)); 17 | 18 | im = imread(sprintf(DATAopts.imgpath, imName)); 19 | im = im2double(im); 20 | selectiveSearchBoxes = selective_search_boxes(im, true, 500); 21 | 22 | boxStruct = attach_proposals(record.annotation, selectiveSearchBoxes, class_to_id); 23 | 24 | 25 | % ------------------------------------------------------------------------ 26 | function rec = attach_proposals(voc_rec, boxes, class_to_id) 27 | % ------------------------------------------------------------------------ 28 | 29 | % change selective search order from [y1 x1 y2 x2] to [x1 y1 x2 y2] 30 | boxes = boxes(:, [2 1 4 3]); 31 | 32 | % gt: [2108x1 double] 33 | % overlap: [2108x20 single] 34 | % dataset: 'voc_2007_trainval' 35 | % boxes: [2108x4 single] 36 | % feat: [2108x9216 single] 37 | % class: [2108x1 uint8] 38 | if isfield(voc_rec, 'object') 39 | num_gt_boxes = length(voc_rec.object); 40 | gt_boxes = zeros(num_gt_boxes, 4); 41 | gt_classes = zeros(num_gt_boxes, 1); 42 | for i=1:length(voc_rec.object) 43 | gt_boxes(i,:) = [str2double(voc_rec.object(i).bndbox.xmin) ... 44 | str2double(voc_rec.object(i).bndbox.ymin) ... 45 | str2double(voc_rec.object(i).bndbox.xmax) ... 46 | str2double(voc_rec.object(i).bndbox.ymax)]; 47 | gt_classesT = class_to_id.values({voc_rec.object(i).name}); 48 | gt_classes(i) = gt_classesT{1}; 49 | end 50 | all_boxes = cat(1, gt_boxes, boxes); 51 | else 52 | gt_boxes = []; 53 | all_boxes = boxes; 54 | gt_classes = []; 55 | num_gt_boxes = 0; 56 | end 57 | num_boxes = size(boxes, 1); 58 | 59 | rec.gt = cat(1, true(num_gt_boxes, 1), false(num_boxes, 1)); 60 | rec.overlap = zeros(num_gt_boxes+num_boxes, class_to_id.Count, 'single'); 61 | % Get overlap wrt ground truth boxes 62 | for i = 1:num_gt_boxes 63 | rec.overlap(:, gt_classes(i)) = ... 64 | max(rec.overlap(:, gt_classes(i)), BoxOverlap(all_boxes, gt_boxes(i, :))); 65 | end 66 | rec.boxes = single(all_boxes); 67 | rec.feat = []; 68 | rec.class = uint16(cat(1, gt_classes, zeros(num_boxes, 1))); 69 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/evaluateMeanIOU.m: -------------------------------------------------------------------------------- 1 | function[meanIOU] = evaluateMeanIOU(imageList, probs, segmentFolder, varargin) 2 | % [meanIOU] = evaluateMeanIOU(imageList, probs, segmentFolder, varargin) 3 | % 4 | % Computes mean-class Intersection over Union with the ground-truth. 5 | % Takes into account the SS hierarchy. 6 | % To use this hierarchy, run reconstructSelSearchHierarchyFromFz(). 7 | % 8 | % Note: Void pixels are not taken into consideration! 9 | % 10 | % Copyright by Holger Caesar, 2015 11 | 12 | % Parse input 13 | p = inputParser; 14 | addParameter(p, 'printStatus', true); 15 | addParameter(p, 'backgroundBelowThresh', []); 16 | parse(p, varargin{:}); 17 | 18 | printStatus = p.Results.printStatus; 19 | backgroundBelowThresh = p.Results.backgroundBelowThresh; 20 | 21 | % Init 22 | labelCount = size(probs{1}, 2); 23 | assert(labelCount > 1); 24 | pixConfMatrix = zeros(labelCount, labelCount); % gt x output 25 | 26 | imageCount = numel(imageList); 27 | for imageIdx = 1 : imageCount, 28 | if printStatus, 29 | printProgress('Evaluating meanIOU for image', imageIdx, imageCount); 30 | end; 31 | 32 | % Skip images without ground-truth 33 | if isempty(probs{imageIdx}), 34 | continue; 35 | end; 36 | 37 | % Create segmentation path 38 | imageName = imageList{imageIdx}; 39 | segmentPath = [segmentFolder, filesep, imageName, '.mat']; 40 | 41 | % Precompute maximum over labels 42 | if ~isempty(backgroundBelowThresh), 43 | % Set all regions with low prob to be background 44 | [maxProbs, maxInds] = max(double(probs{imageIdx}(:, 2:end)), [], 2); 45 | maxInds = maxInds + 1; 46 | curThresh = backgroundBelowThresh; 47 | sel = maxProbs < curThresh; 48 | maxProbs(sel) = curThresh-eps; 49 | maxInds(sel) = 1; 50 | else 51 | [maxProbs, maxInds] = max(double(probs{imageIdx}), [], 2); 52 | end; 53 | 54 | %%% Max over all ancestors 55 | % Load superpixel information 56 | segmentStruct = load(segmentPath, 'superPixelLabelHistos', 'overlapList'); 57 | spLabelHistos = segmentStruct.superPixelLabelHistos'; 58 | overlapList = segmentStruct.overlapList; 59 | assert(~isempty(spLabelHistos)); 60 | 61 | % Compute confusion matrix 62 | pixConfMatrix = evaluatePixConfMatrix_loop_mex(maxProbs, maxInds, full(double(overlapList)), spLabelHistos, pixConfMatrix); 63 | end; 64 | 65 | % Compute IOU per class 66 | labelIOUs = zeros(labelCount, 1); 67 | for labelIdx = 1 : labelCount, 68 | truePos = pixConfMatrix(labelIdx, labelIdx); 69 | gtPos = sum(pixConfMatrix(labelIdx, :)); 70 | outPos = sum(pixConfMatrix(:, labelIdx)); 71 | labelIOUs(labelIdx) = truePos / (gtPos + outPos - truePos); 72 | end; 73 | 74 | % Compute mean (over classes) intersection-over-union 75 | meanIOU = nanmean(labelIOUs); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/regiontopixel/regionToPixel_backward.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "mex.h" 3 | #include "../compatibility/isScalar.h" 4 | 5 | /* 6 | * [dzdx] = regionToPixel_backward(boxCount, spMap, dzdy) 7 | * 8 | * Go from a pixel level back to region level. 9 | * This uses the mask saved in the forward pass. 10 | * 11 | * Copyright by Holger Caesar, 2015 12 | */ 13 | 14 | void mexFunction(int nlhs, mxArray *out[], int nrhs, const mxArray *input[]) 15 | { 16 | if (nlhs == 0) { 17 | return; 18 | } else if (nlhs != 1 || nrhs != 3) { 19 | mexErrMsgTxt("Error. Usage: [dzdx] = regionToPixel_backward(boxCount, spMap, dzdy)"); 20 | return; 21 | } 22 | 23 | // Get pointers 24 | const mxArray* boxCountMx = input[0]; 25 | const mxArray* spMapMx = input[1]; 26 | const mxArray* dzdyMx = input[2]; 27 | 28 | // Check inputs 29 | if (!mxIsDouble(boxCountMx) || !isScalar(boxCountMx)) { 30 | mexErrMsgTxt("Error: boxCount must be a scalar double!"); 31 | } 32 | if (!mxIsDouble(spMapMx) || mxGetNumberOfDimensions(spMapMx) != 2) { 33 | mexErrMsgTxt("Error: spMap must be double with format labelCount x spCount!"); 34 | } 35 | int labelCount = mxGetM(spMapMx); 36 | int spCount = mxGetN(spMapMx); 37 | const mwSize* dzdyDims = mxGetDimensions(dzdyMx); 38 | if (!mxIsSingle(dzdyMx) || dzdyDims[0] != 1 || dzdyDims[1] != 1 || dzdyDims[2] != labelCount || 39 | (!(mxGetNumberOfDimensions(dzdyMx) == 4 && dzdyDims[3] == spCount) 40 | && !(mxGetNumberOfDimensions(dzdyMx) == 3))) { 41 | mexErrMsgTxt("Error: dzdy must be single with format 1 x 1 x labelCount x spCount!"); 42 | } 43 | 44 | // Get arrays 45 | int boxCount = (int) mxGetScalar(boxCountMx); 46 | double* spMap = (double*) mxGetData(spMapMx); 47 | float* dzdy = (float*) mxGetData(dzdyMx); 48 | 49 | // Create output and initialize it to all zeros (in mxCreateNumericArray) 50 | mwSize dzdxSize[4]; 51 | dzdxSize[0] = 1; 52 | dzdxSize[1] = 1; 53 | dzdxSize[2] = labelCount; 54 | dzdxSize[3] = boxCount; 55 | out[0] = mxCreateNumericArray(4, dzdxSize, mxSINGLE_CLASS, mxREAL); 56 | float* dzdx = (float*) mxGetData(out[0]); 57 | 58 | for (int spIdx = 0; spIdx < spCount; spIdx++) { 59 | for (int labelIdx = 0; labelIdx < labelCount; labelIdx++) { 60 | // We can safely ignore the first two dimensions of these 61 | // matrices as they are always 1 62 | int spMapIdx = labelIdx + spIdx * labelCount; 63 | double boxIdxD = spMap[spMapIdx]; 64 | int boxIdx = (int) boxIdxD - 1; // Convert from Matlab to C indexing 65 | if (!mxIsNaN(boxIdxD)) { 66 | int dzdxIdx = labelIdx + boxIdx * labelCount; 67 | dzdx[dzdxIdx] = dzdx[dzdxIdx] + dzdy[spMapIdx]; 68 | } 69 | } 70 | } 71 | } -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/regiontopixel/regionToPixelSoft_backward.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "mex.h" 3 | #include "../compatibility/isScalar.h" 4 | 5 | /* 6 | * [dzdx] = regionToPixelSoft_backward(weightsSP, dzdy) 7 | * 8 | * Go from a pixel level back to region level. 9 | * This uses the mask saved in the forward pass. 10 | * 11 | * Copyright by Holger Caesar, 2015 12 | */ 13 | 14 | void mexFunction(int nlhs, mxArray *out[], int nrhs, const mxArray *input[]) 15 | { 16 | if (nlhs == 0) { 17 | return; 18 | } else if (nlhs != 1 || nrhs != 2) { 19 | mexErrMsgTxt("Error. Usage: [dzdx] = regionToPixelSoft_backward(weightsSP, dzdy)"); 20 | return; 21 | } 22 | 23 | // Get pointers 24 | const mxArray* weightsSPMx = input[0]; 25 | const mxArray* dzdyMx = input[1]; 26 | 27 | // Check inputs 28 | if (!mxIsDouble(weightsSPMx) || mxGetNumberOfDimensions(weightsSPMx) != 3) { 29 | mexErrMsgTxt("Error: weightsSP must be double with format labelCount x spCount x rpCount!"); 30 | } 31 | const mwSize* dims = mxGetDimensions(weightsSPMx); 32 | int labelCount = dims[0]; 33 | int spCount = dims[1]; 34 | int rpCount = dims[2]; 35 | const mwSize* dzdyDims = mxGetDimensions(dzdyMx); 36 | if (!mxIsSingle(dzdyMx) || dzdyDims[0] != 1 || dzdyDims[1] != 1 || dzdyDims[2] != labelCount || 37 | (!(mxGetNumberOfDimensions(dzdyMx) == 4 && dzdyDims[3] == spCount) 38 | && !(mxGetNumberOfDimensions(dzdyMx) == 3))) { 39 | mexErrMsgTxt("Error: dzdy must be single with format 1 x 1 x labelCount x spCount!"); 40 | } 41 | 42 | // Get arrays 43 | double* weightsSP = (double*) mxGetData(weightsSPMx); 44 | float* dzdy = (float*) mxGetData(dzdyMx); 45 | 46 | // Create output and initialize it to all zeros (in mxCreateNumericArray) 47 | mwSize dzdxSize[4]; 48 | dzdxSize[0] = 1; 49 | dzdxSize[1] = 1; 50 | dzdxSize[2] = labelCount; 51 | dzdxSize[3] = rpCount; 52 | out[0] = mxCreateNumericArray(4, dzdxSize, mxSINGLE_CLASS, mxREAL); 53 | float* dzdx = (float*) mxGetData(out[0]); 54 | 55 | for (int spIdx = 0; spIdx < spCount; spIdx++) { 56 | for (int labelIdx = 0; labelIdx < labelCount; labelIdx++) { 57 | for (int rpIdx = 0; rpIdx < rpCount; rpIdx++) { 58 | // 59 | 60 | // We can safely ignore the first two dimensions of these 61 | // matrices as they are always 1 62 | int weightsSPIdx = labelIdx + spIdx * labelCount; 63 | double boxIdxD = weightsSP[weightsSPIdx]; 64 | int boxIdx = (int) boxIdxD - 1; // Convert from Matlab to C indexing 65 | if (!mxIsNaN(boxIdxD)) { 66 | int dzdxIdx = labelIdx + boxIdx * labelCount; 67 | dzdx[dzdxIdx] = dzdx[dzdxIdx] + dzdy[weightsSPIdx]; 68 | } 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/@CalvinNN/plotStats.m: -------------------------------------------------------------------------------- 1 | function plotStats(obj, epochs, stats, plotAccuracy) %#ok 2 | % plotStats(obj, epochs, stats, plotAccuracy) 3 | % 4 | % Plot the results of different metrics on train and validation set. 5 | % 6 | % Copyright by Holger Caesar, 2016 7 | 8 | % Define color map 9 | cmap = parula(3); 10 | cmap(3, :) = [255, 140, 0] / 255; 11 | 12 | if true 13 | figure(1); 14 | clf; 15 | if plotAccuracy 16 | hsp1 = subplot(2, 1, 1); 17 | end 18 | hold on; 19 | leg = {}; 20 | datasetModes = {'train', 'val'}; 21 | for datasetModeIdx = 1 : numel(datasetModes) 22 | datasetMode = datasetModes{datasetModeIdx}; 23 | field = 'objective'; 24 | fieldValues = [stats.(datasetMode).(field)]; 25 | if datasetModeIdx == 1 26 | marker = '-'; 27 | else 28 | marker = '--'; 29 | end 30 | 31 | % For i.e. objective with 1 value 32 | leg{end + 1} = sprintf('%s (%s)', field, datasetMode); %#ok 33 | values = fieldValues(1, :)'; 34 | plot(epochs, values, 'Color', cmap(1, :), 'LineStyle', marker); 35 | end 36 | hleg1 = legend(leg, 'Location', 'NorthEastOutSide'); 37 | 38 | xlabel('epoch'); 39 | ylabel('objective'); 40 | grid on; 41 | end 42 | 43 | if plotAccuracy 44 | hsp2 = subplot(2, 1, 2); 45 | hold on; 46 | 47 | leg = {}; 48 | datasetModes = {'train', 'val'}; 49 | for datasetModeIdx = 1 : numel(datasetModes) 50 | datasetMode = datasetModes{datasetModeIdx}; 51 | field = 'accuracy'; 52 | fieldValues = [stats.(datasetMode).(field)]; 53 | if datasetModeIdx == 1 54 | marker = '-'; 55 | else 56 | marker = '--'; 57 | end 58 | 59 | leg{end + 1} = sprintf('Pix. Acc. (%s)', datasetMode); %#ok 60 | values = fieldValues(1, :)'; 61 | plot(epochs, values, 'Color', cmap(1, :), 'LineStyle', marker); 62 | 63 | leg{end + 1} = sprintf('Class. Acc. (%s)', datasetMode); %#ok 64 | values = fieldValues(2, :)'; 65 | plot(epochs, values, 'Color', cmap(2, :), 'LineStyle', marker); 66 | 67 | leg{end + 1} = sprintf('Mean IU (%s)', datasetMode); %#ok 68 | values = fieldValues(3, :)'; 69 | plot(epochs, values, 'Color', cmap(3, :), 'LineStyle', marker); 70 | end 71 | 72 | % Define legend 73 | hleg2 = legend(leg, 'Location', 'NorthEastOutSide'); 74 | 75 | % Align both plots 76 | if false 77 | plotPos1 = get(hsp1, 'Position'); 78 | plotPos2 = get(hsp2, 'Position'); 79 | set(hsp2, 'Position', [plotPos1(1), plotPos2(2), plotPos1(3:4)]); 80 | legPos1 = get(hleg1, 'Position'); 81 | legPos2 = get(hleg2, 'Position'); 82 | set(hleg2, 'Position', [legPos1(1), legPos2(2), legPos1(3:4)]); 83 | end 84 | 85 | xlabel('epoch'); 86 | ylabel('accuracy'); 87 | grid on; 88 | end -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/boxes/selective_search_boxes.m: -------------------------------------------------------------------------------- 1 | function [boxes, blobIndIm, blobIndBoxes, hierarchy] = selective_search_boxes(im, fast_mode, im_width) 2 | % Girshick Wrapper 3 | % 4 | % Based on the demo.m file included in the Selective Search 5 | % IJCV code. 6 | % 7 | % Requires selective search code. 8 | 9 | if ~exist('fast_mode', 'var') || isempty(fast_mode) 10 | fast_mode = true; 11 | end 12 | 13 | originalImSize = size(im); 14 | 15 | if ~exist('im_width', 'var') || isempty(im_width) 16 | im_width = []; 17 | scale = 1; 18 | else 19 | scale = size(im, 2) / im_width; 20 | end 21 | 22 | if scale ~= 1 23 | im = imresize(im, [NaN im_width]); 24 | end 25 | 26 | % Parameters. Note that this controls the number of hierarchical 27 | % segmentations which are combined. 28 | colorTypes = {'Hsv', 'Lab', 'RGI', 'H', 'Intensity'}; 29 | 30 | % Here you specify which similarity functions to use in merging 31 | simFunctionHandles = {@SSSimColourTextureSizeFillOrig, ... 32 | @SSSimTextureSizeFill, ... 33 | @SSSimBoxFillOrig, ... 34 | @SSSimSize}; 35 | 36 | % Thresholds for the Felzenszwalb and Huttenlocher segmentation algorithm. 37 | % Note that by default, we set minSize = k, and sigma = 0.8. 38 | % controls size of segments of initial segmentation. 39 | ks = [50 100 150 300]; 40 | sigma = 0.8; 41 | 42 | % After segmentation, filter out boxes which have a width/height smaller 43 | % than minBoxWidth (default = 20 pixels). 44 | minBoxWidth = 20; 45 | 46 | % Comment the following three lines for the 'quality' version 47 | if fast_mode 48 | colorTypes = colorTypes(1:2); % 'Fast' uses HSV and Lab 49 | simFunctionHandles = simFunctionHandles(1:2); % Two different merging strategies 50 | ks = ks(1:2); 51 | end 52 | 53 | idx = 1; 54 | for j = 1:length(ks) 55 | k = ks(j); % Segmentation threshold k 56 | minSize = k; % We set minSize = k 57 | for n = 1:length(colorTypes) 58 | colorType = colorTypes{n}; 59 | [boxesT{idx} blobIndIm{idx} blobIndBoxes{idx} hierarchy{idx} priorityT{idx}] = ... 60 | Image2HierarchicalGrouping(im, sigma, k, minSize, colorType, simFunctionHandles); 61 | idx = idx + 1; 62 | end 63 | end 64 | boxes = cat(1, boxesT{:}); % Concatenate boxes from all hierarchies 65 | priority = cat(1, priorityT{:}); % Concatenate priorities 66 | 67 | % Do pseudo random sorting as in paper 68 | priority = priority .* rand(size(priority)); 69 | [priority sortIds] = sort(priority, 'ascend'); 70 | boxes = boxes(sortIds,:); 71 | 72 | boxes = BoxRemoveDuplicates(boxes); 73 | 74 | if scale ~= 1 75 | boxes = floor((boxes - 1) * scale + 1); 76 | for iii = 1:length(blobIndBoxes) 77 | blobIndBoxes{iii} = floor((blobIndBoxes{iii} - 1) * scale + 1); 78 | blobIndIm{iii} = imresize(blobIndIm{iii}, [originalImSize(1) originalImSize(2)], 'nearest'); 79 | end 80 | end 81 | 82 | % Filter width of boxes 83 | [nr, nc] = BoxSize(boxes); 84 | idsGood = (nr >= minBoxWidth) & (nc >= minBoxWidth); 85 | boxes = boxes(idsGood,:); 86 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/labelpresence/labelPresence_backward.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "mex.h" 3 | #include "../compatibility/isScalar.h" 4 | 5 | /* 6 | * [dzdx] = labelPresence_backward(spCount, labelMap, dzdy) 7 | * 8 | * Note that this script works essentially the same as regionToPixel_backward. 9 | * It maps from one label to many superpixels. 10 | * This uses the mask saved in the forward pass. 11 | * 12 | * Copyright by Holger Caesar, 2015 13 | */ 14 | 15 | void mexFunction(int nlhs, mxArray *out[], int nrhs, const mxArray *input[]) { 16 | if (nlhs == 0) { 17 | return; 18 | } else if (nlhs != 1 || nrhs != 3) { 19 | mexErrMsgTxt("Error. Usage: [dzdx] = labelPresence_backward(spCount, labelMap, dzdy)"); 20 | return; 21 | } 22 | 23 | // Get pointers 24 | const mxArray* spCountMx = input[0]; 25 | const mxArray* labelMapMx = input[1]; 26 | const mxArray* dzdyMx = input[2]; 27 | 28 | // Check inputs 29 | if (!mxIsDouble(spCountMx) || !isScalar(spCountMx)) { 30 | mexErrMsgTxt("Error: spCount must be a scalar double!"); 31 | } 32 | if (!mxIsDouble(labelMapMx) || mxGetNumberOfDimensions(labelMapMx) != 2) { 33 | mexErrMsgTxt("Error: labelMap must be double with format labelCount x labelListCount!"); 34 | } 35 | int labelCount = mxGetM(labelMapMx); 36 | int labelListCount = mxGetN(labelMapMx); 37 | const mwSize* dzdyDims = mxGetDimensions(dzdyMx); 38 | if (!mxIsSingle(dzdyMx) || dzdyDims[0] != 1 || dzdyDims[1] != 1 || dzdyDims[2] != labelCount || 39 | (!(mxGetNumberOfDimensions(dzdyMx) == 4 && dzdyDims[3] == labelListCount) 40 | && !(mxGetNumberOfDimensions(dzdyMx) == 3))) { 41 | mexErrMsgTxt("Error: dzdy must be single with format 1 x 1 x labelCount x labelListCount!"); 42 | } 43 | 44 | // Get arrays 45 | int spCount = (int) mxGetScalar(spCountMx); 46 | double* labelMap = (double*) mxGetData(labelMapMx); 47 | float* dzdy = (float*) mxGetData(dzdyMx); 48 | 49 | // Create output and initialize it to all zeros (in mxCreateNumericArray) 50 | mwSize dzdxSize[4]; 51 | dzdxSize[0] = 1; 52 | dzdxSize[1] = 1; 53 | dzdxSize[2] = labelCount; 54 | dzdxSize[3] = spCount; 55 | out[0] = mxCreateNumericArray(4, dzdxSize, mxSINGLE_CLASS, mxREAL); 56 | float* dzdx = (float*) mxGetData(out[0]); 57 | 58 | for (int labelListIdx = 0; labelListIdx < labelListCount; labelListIdx++) { 59 | for (int labelIdx = 0; labelIdx < labelCount; labelIdx++) { 60 | // We can safely ignore the first two dimensions of these 61 | // matrices as they are always 1 62 | int labelMapIdx = labelIdx + labelListIdx * labelCount; 63 | double boxIdxD = labelMap[labelMapIdx]; 64 | if (!mxIsNaN(boxIdxD)) { 65 | int boxIdx = (int) boxIdxD - 1; // Convert from Matlab to C indexing 66 | int dzdxIdx = labelIdx + boxIdx * labelCount; 67 | dzdx[dzdxIdx] = dzdx[dzdxIdx] + dzdy[labelMapIdx]; 68 | } 69 | } 70 | } 71 | } -------------------------------------------------------------------------------- /matconvnet-calvin/examples/fcn/fcnTestGeneric.m: -------------------------------------------------------------------------------- 1 | function[stats] = fcnTestGeneric(varargin) 2 | % [stats] = fcnTestGeneric(varargin) 3 | % 4 | % Copyright by Holger Caesar, 2016 5 | 6 | % Initial settings 7 | p = inputParser; 8 | addParameter(p, 'dataset', SiftFlowDatasetMC()); 9 | addParameter(p, 'modelType', 'fcn16s'); 10 | addParameter(p, 'gpus', 1); 11 | addParameter(p, 'randSeed', 42); 12 | addParameter(p, 'expNameAppend', 'testRelease'); 13 | addParameter(p, 'epoch', 50); 14 | addParameter(p, 'subset', 'test'); % train / test 15 | addParameter(p, 'expSubFolder', ''); 16 | addParameter(p, 'findMapping', false); % Find the best possible label mapping from ILSVRC to target dataset 17 | addParameter(p, 'storeOutputMaps', true); 18 | addParameter(p, 'extractFeatsVarName', ''); % If used we don't measure performance 19 | addParameter(p, 'plotFreq', 15); 20 | parse(p, varargin{:}); 21 | 22 | dataset = p.Results.dataset; 23 | modelType = p.Results.modelType; 24 | gpus = p.Results.gpus; 25 | randSeed = p.Results.randSeed; 26 | expNameAppend = p.Results.expNameAppend; 27 | epoch = p.Results.epoch; 28 | subset = p.Results.subset; 29 | expSubFolder = p.Results.expSubFolder; 30 | findMapping = p.Results.findMapping; 31 | storeOutputMaps = p.Results.storeOutputMaps; 32 | extractFeatsVarName = p.Results.extractFeatsVarName; 33 | plotFreq = p.Results.plotFreq; 34 | callArgs = p.Results; %#ok 35 | 36 | % Experiment and data paths 37 | global glFeaturesFolder; 38 | expName = [modelType, prependNotEmpty(expNameAppend, '-')]; 39 | expDir = fullfile(glFeaturesFolder, 'CNN-Models', 'FCN', dataset.name, expSubFolder, expName); 40 | netPath = fullfile(expDir, sprintf('net-epoch-%d.mat', epoch)); 41 | netOptsPath = fullfile(expDir, 'net-opts.mat'); 42 | 43 | % Fix randomness 44 | rng(randSeed); 45 | 46 | % Check if directory exists 47 | if ~exist(expDir, 'dir') 48 | error('Error: Experiment directory does not exist %s\n', expDir); 49 | end 50 | 51 | % Load imdb and nnOpts from file 52 | if exist(netOptsPath, 'file') 53 | netOptsStruct = load(netOptsPath, 'callArgs', 'nnOpts', 'imdbFcn'); 54 | nnOpts = netOptsStruct.nnOpts; 55 | imdbFcn = netOptsStruct.imdbFcn; 56 | assert(isequal(dataset.name, imdbFcn.dataset.name)); 57 | else 58 | error('Error: Cannot find netOpts path: %s', netOptsPath); 59 | end 60 | 61 | % If there is no validation set specified, use val as test set 62 | if ~isfield(imdbFcn.data, 'test') 63 | imdbFcn.data.test = imdbFcn.data.val; 64 | imdbFcn.data = rmfield(imdbFcn.data, 'val'); 65 | end 66 | 67 | imdbFcn.numClasses = imdbFcn.dataset.labelCount; 68 | 69 | % Overwrite some settings 70 | nnOpts.gpus = gpus; 71 | nnOpts.convertToTrain = false; 72 | nnOpts.expDir = expDir; 73 | 74 | % Create network 75 | nnClass = FCNNN(netPath, imdbFcn, nnOpts); 76 | 77 | if isempty(extractFeatsVarName) 78 | % Test the network performance 79 | stats = nnClass.testOnSet('subset', subset, 'findMapping', findMapping, 'storeOutputMaps', storeOutputMaps, 'plotFreq', plotFreq); 80 | else 81 | % Extract the specified features 82 | nnClass.extractFeatures('outputVarName', extractFeatsVarName); 83 | end 84 | -------------------------------------------------------------------------------- /matconvnet-calvin/examples/e2s2/e2s2_storeBlobMasks.m: -------------------------------------------------------------------------------- 1 | function e2s2_storeBlobMasks(varargin) 2 | % e2s2_storeBlobMasks(varargin) 3 | % 4 | % Augment the region proposal structure with a downscaled version of each 5 | % blobs mask. 6 | % 7 | % Copyright by Holger Caesar, 2015 8 | 9 | % Parse input 10 | p = inputParser; 11 | addParameter(p, 'dataset', SiftFlowDatasetMC()); 12 | addParameter(p, 'projectName', 'WeaklySupervisedLearning'); 13 | addParameter(p, 'proposalName', 'Uijlings2013-ks100-sigma0.8-colorTypesRgb'); 14 | addParameter(p, 'roiPoolFreeformDilateSize', []); % Don't change this! 15 | addParameter(p, 'roiPoolFreeformThresh', 0); % Don't change this! 16 | addParameter(p, 'roiPoolSize', [7, 7]); %6x6 for AlexNet, 7x7 for VGG16 17 | addParameter(p, 'skipExisting', true); 18 | parse(p, varargin{:}); 19 | 20 | dataset = p.Results.dataset; 21 | projectName = p.Results.projectName; 22 | proposalName = p.Results.proposalName; 23 | roiPoolFreeformDilateSize = p.Results.roiPoolFreeformDilateSize; 24 | roiPoolFreeformThresh = p.Results.roiPoolFreeformThresh; 25 | roiPoolSize = p.Results.roiPoolSize; 26 | skipExisting = p.Results.skipExisting; 27 | 28 | % Create paths 29 | global glFeaturesFolder; 30 | segmentFolder = fullfile(glFeaturesFolder, projectName, dataset.name, 'segmentations', proposalName); 31 | 32 | % Get blob mask variable name 33 | masksName = sprintf('blobMasks%dx%d', roiPoolSize(1), roiPoolSize(2)); 34 | 35 | % Get image list 36 | [imageList, imageCount] = dataset.getImageList(); 37 | 38 | for imageIdx = 1 : imageCount, 39 | printProgress(sprintf('Storing blob mask (%s) for image', proposalName), imageIdx, imageCount, 50); 40 | 41 | % Get segmentation 42 | imageName = imageList{imageIdx}; 43 | segmentPath = fullfile(segmentFolder, [imageName, '.mat']); 44 | assert(exist(segmentPath, 'file') ~= 0); 45 | segmentStruct = load(segmentPath, 'propBlobs'); 46 | propBlobs = segmentStruct.propBlobs; 47 | blobCount = numel(propBlobs); 48 | if matFileHasField(segmentPath, masksName) && skipExisting, 49 | fprintf('Skipping existing blob masks in file: %s\n', segmentPath); 50 | continue; 51 | end; 52 | 53 | % Compute blob masks 54 | blobMasks = cell(blobCount, 1); 55 | for blobIdx = 1 : blobCount, 56 | % Resize and keep an element if the majority of pixels belong 57 | % to that class 58 | blobMaskOri = propBlobs(blobIdx).mask; 59 | 60 | if ~isempty(roiPoolFreeformDilateSize), 61 | filterSize = ceil(size(blobMaskOri) * roiPool.roiPoolFreeformDilateSize); 62 | filter = strel('rectangle', filterSize); 63 | blobMaskOri = imdilate(blobMaskOri, filter); 64 | end; 65 | 66 | blobMaskOri = double(blobMaskOri); 67 | blobMask = imresize(blobMaskOri, roiPoolSize, 'Method', 'bilinear', 'Antialiasing', false); 68 | blobMasks{blobIdx} = blobMask > roiPoolFreeformThresh; 69 | end; 70 | 71 | % Append the blobMasks to the existing segmentStruct 72 | eval(sprintf('%s = blobMasks;', masksName)); 73 | save(segmentPath, masksName, '-append'); 74 | end; -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/testDetection.m: -------------------------------------------------------------------------------- 1 | function [results] = testDetection(imdb, nnOpts, net, inputs, ~) 2 | % [results] = testDetection(imdb, nnOpts, net, inputs, ~) 3 | % 4 | % Get predicted boxes and scores per class 5 | % Only gets top nnOpts.maxNumBoxesPerImTest boxes (default: 5000) 6 | % Only gets boxes with score higher than nnOpts.minDetectionScore (default: 0.01) 7 | % NMS threshold: nnOpts.nmsTTest (default: 0.3) 8 | 9 | % Variables which should probably be in imdb.nnOpts or something 10 | % Jasper: Probably need to do something more robust here 11 | % 12 | % Copyright by Jasper Uijlings, 2015 13 | 14 | if isfield(nnOpts, 'maxNumBoxesPerImTest') 15 | maxNumBoxesPerImTest = nnOpts.maxNumBoxesPerImTest; 16 | else 17 | maxNumBoxesPerImTest = 5000; 18 | end 19 | 20 | if isfield(nnOpts, 'nmsTTest') 21 | nmsTTest = imdb.nmsTTest; 22 | else 23 | nmsTTest = 0.3; % non-maximum threshold 24 | end 25 | 26 | if isfield(nnOpts, 'minDetectionScore') 27 | minDetectionScore = nnOpts.minDetectionScore; 28 | else 29 | minDetectionScore = 0.01; 30 | end 31 | 32 | % Get scores 33 | vI = net.getVarIndex('scores'); 34 | scoresStruct = net.vars(vI); 35 | scores = permute(scoresStruct.value, [4 3 2 1]); 36 | 37 | % Get boxes 38 | inputNames = inputs(1:2:end); 39 | [~, boxI] = ismember('boxes', inputNames); 40 | boxI = boxI * 2; % Index of actual argument 41 | boxes = inputs{boxI}'; 42 | 43 | % Get regression targets for boxes 44 | if imdb.boxRegress 45 | vI = net.getVarIndex('regressionScore'); 46 | regressStruct = net.vars(vI); 47 | regressFactors = permute(regressStruct.value, [4 3 2 1]); 48 | else 49 | regressFactors = zeros(size(boxes,1), size(boxes,2) * imdb.numClasses); 50 | end 51 | 52 | % Get top boxes for each category. Perform NMS. Thresholds defined at top of function 53 | currMaxBoxes = min(maxNumBoxesPerImTest, size(boxes, 1)); 54 | for cI = size(scores,2) : -1 : 1 55 | % Get top scores and boxes 56 | [currScoresT, sI] = sort(scores(:,cI), 'descend'); 57 | currScoresT = currScoresT(1:currMaxBoxes); 58 | sI = sI(1:currMaxBoxes); 59 | currBoxes = boxes(sI,:); 60 | 61 | % Do regression 62 | regressFRange = (cI*4)-3:cI*4; 63 | currRegressF = gather(regressFactors(sI,regressFRange)); 64 | currBoxesReg = BoxRegresssGirshick(currBoxes, currRegressF); 65 | 66 | % Get scores (w boxes) above certain threshold 67 | goodI = currScoresT > minDetectionScore; 68 | currScoresT = currScoresT(goodI, :); 69 | currBoxes = currBoxes(goodI, :); 70 | currBoxesReg = currBoxesReg(goodI, :); 71 | 72 | % Perform NMS 73 | [~, goodBoxesI] = BoxNMS(currBoxes, nmsTTest); 74 | currBoxes = currBoxes(goodBoxesI, :); 75 | currScores = currScoresT(goodBoxesI ,:); 76 | 77 | results.boxes{cI} = gather(currBoxes); 78 | results.scores{cI} = gather(currScores); 79 | 80 | if imdb.boxRegress 81 | [~, goodBoxesI] = BoxNMS(currBoxesReg, nmsTTest); 82 | currBoxesReg = currBoxesReg(goodBoxesI, :); 83 | currScoresRegressed = currScoresT(goodBoxesI, :); 84 | results.boxesRegressed{cI} = gather(currBoxesReg); 85 | results.scoresRegressed{cI} = gather(currScoresRegressed); 86 | end 87 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/+dagnn/LabelPresence.m: -------------------------------------------------------------------------------- 1 | classdef LabelPresence < dagnn.Layer 2 | % Convert pixel label scores to presence/absence scores for each class per batch. 3 | % (to be able to compute an image-level loss there) 4 | % 5 | % inputs are: scoresSP, labelImage 6 | % outputs are: scoresImage 7 | % 8 | % Copyright by Holger Caesar, 2015 9 | 10 | properties (Transient) 11 | mask 12 | end 13 | 14 | methods 15 | function outputs = forward(obj, inputs, params) %#ok 16 | 17 | % Get inputs 18 | assert(numel(inputs) == 2); 19 | scoresSP = inputs{1}; 20 | labelImage = inputs{2}; 21 | 22 | % Move to CPU 23 | gpuMode = isa(scoresSP, 'gpuArray'); 24 | if gpuMode 25 | scoresSP = gather(scoresSP); 26 | end 27 | 28 | % Init 29 | labelList = unique(labelImage); 30 | labelListCount = numel(labelList); 31 | labelCount = size(scoresSP, 3); 32 | scoresImage = nan(1, 1, labelCount, labelListCount, 'single'); % score of the label, and all other labels 33 | obj.mask = nan(labelCount, labelListCount); % contains the label of each superpixel 34 | 35 | % For each label, get the scores of the highest scoring pixel 36 | for labelListIdx = 1 : labelListCount, 37 | labelIdx = labelList(labelListIdx); 38 | [~, spIdx] = max(scoresSP(:, :, labelIdx, :), [], 4); 39 | scoresImage(:, :, :, labelListIdx) = scoresSP(:, :, :, spIdx); 40 | obj.mask(:, labelListIdx) = spIdx; 41 | end 42 | 43 | % Convert outputs back to GPU if necessary 44 | if gpuMode 45 | scoresImage = gpuArray(scoresImage); 46 | end 47 | 48 | % Store outputs 49 | outputs = cell(1, 1); 50 | outputs{1} = scoresImage; 51 | end 52 | 53 | function [derInputs, derParams] = backward(obj, inputs, params, derOutputs) %#ok 54 | % 55 | % This uses the mask saved in the forward pass. 56 | 57 | % Get inputs 58 | assert(numel(derOutputs) == 1); 59 | spCount = size(inputs{1}, 4); 60 | dzdy = derOutputs{1}; 61 | 62 | % Move inputs from GPU if necessary 63 | gpuMode = isa(dzdy, 'gpuArray'); 64 | if gpuMode 65 | dzdy = gather(dzdy); 66 | end 67 | 68 | % Map Image gradients to RP+GT gradients 69 | dzdx = labelPresence_backward(spCount, obj.mask, dzdy); 70 | 71 | % Move outputs to GPU if necessary 72 | if gpuMode 73 | dzdx = gpuArray(dzdx); 74 | end 75 | 76 | % Store gradients 77 | derInputs{1} = dzdx; 78 | derInputs{2} = []; 79 | derParams = {}; 80 | end 81 | 82 | function obj = LabelPresence(varargin) 83 | obj.load(varargin); 84 | end 85 | end 86 | end -------------------------------------------------------------------------------- /matconvnet-calvin/examples/frcn/evalDetection.m: -------------------------------------------------------------------------------- 1 | function evalDetection(testName, imdb, stats, nnOpts) 2 | 3 | global DATAopts; 4 | 5 | % Get test images 6 | testIms = imdb.misc.testIms; 7 | 8 | % get image sizes 9 | testCount = length(testIms); 10 | for i = testCount : -1 : 1 11 | im = imread(sprintf(DATAopts.imgpath, testIms{i})); 12 | imSizes(i, :) = size(im); 13 | end 14 | 15 | for cI = 1 : 20 16 | % 17 | currBoxes = cell(testCount, 1); 18 | currScores = cell(testCount, 1); 19 | for i = 1 : testCount 20 | currBoxes{i} = stats.results(i).boxes{cI + 1}; 21 | currScores{i} = stats.results(i).scores{cI + 1}; 22 | end 23 | 24 | [currBoxes, fileIdx] = Cell2Matrix(gather(currBoxes)); 25 | [currScores, fileIdx2] = Cell2Matrix(gather(currScores)); 26 | 27 | assert(isequal(fileIdx, fileIdx2)); % Should be equal 28 | 29 | currFilenames = testIms(fileIdx); 30 | 31 | [~, sI] = sort(currScores, 'descend'); 32 | currScores = currScores(sI); 33 | currBoxes = currBoxes(sI,:); 34 | currFilenames = currFilenames(sI); 35 | 36 | % ShowImageRects(currBoxes(1:32, [2 1 4 3]), 4, 4, currFilenames(1:32), currScores(1:32)); 37 | 38 | % 39 | [recall{cI}, prec{cI}, ap(cI,1), upperBound{cI}] = ... 40 | DetectionToPascalVOCFiles(testName, cI, currBoxes, currFilenames, currScores, ... 41 | 'Matconvnet-Calvin', 1, nnOpts.misc.overlapNms); 42 | ap(cI) 43 | end 44 | 45 | ap 46 | mean(ap) 47 | 48 | if isfield(stats.results(1), 'boxesRegressed') 49 | for cI = 1 : 20 50 | % 51 | currBoxes = cell(testCount, 1); 52 | currScores = cell(testCount, 1); 53 | 54 | for i=1:testCount 55 | % Get regressed boxes and refit them to the image 56 | currBoxes{i} = stats.results(i).boxesRegressed{cI+1}; 57 | currBoxes{i}(:,1) = max(currBoxes{i}(:, 1), 1); 58 | currBoxes{i}(:,2) = max(currBoxes{i}(:, 2), 1); 59 | currBoxes{i}(:,3) = min(currBoxes{i}(:, 3), imSizes(i,2)); 60 | currBoxes{i}(:,4) = min(currBoxes{i}(:, 4), imSizes(i,1)); 61 | 62 | currScores{i} = stats.results(i).scoresRegressed{cI+1}; 63 | end 64 | 65 | [currBoxes, fileIdx] = Cell2Matrix(gather(currBoxes)); 66 | [currScores, fileIdx2] = Cell2Matrix(gather(currScores)); 67 | 68 | assert(isequal(fileIdx, fileIdx2)); % Should be equal 69 | 70 | currFilenames = testIms(fileIdx); 71 | 72 | [~, sI] = sort(currScores, 'descend'); 73 | currScores = currScores(sI); 74 | currBoxes = currBoxes(sI, :); 75 | currFilenames = currFilenames(sI); 76 | 77 | % ShowImageRects(currBoxes(1:32, [2 1 4 3]), 4, 4, currFilenames(1:32), currScores(1:32)); 78 | 79 | % 80 | [recall{cI}, prec{cI}, apRegressed(cI,1), upperBound{cI}] = ... 81 | DetectionToPascalVOCFiles(testName, cI, currBoxes, currFilenames, currScores, ... 82 | 'Matconvnet-Calvin', 1, nnOpts.misc.overlapNms); 83 | apRegressed(cI) 84 | end 85 | 86 | apRegressed 87 | mean(apRegressed) 88 | else 89 | apRegressed = 0; 90 | end 91 | 92 | % Save results to disk 93 | save([nnOpts.expDir, '/', 'resultsEpochFinalTest.mat'], 'nnOpts', 'stats', 'ap', 'apRegressed'); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/misc/evaluatePixAccPaint.m: -------------------------------------------------------------------------------- 1 | function[pixAcc, meanClassPixAcc, classPixAccs] = evaluatePixAccPaint(dataset, imageList, probs, segmentFolder, varargin) 2 | % [pixAcc, meanClassPixAcc, classPixAccs] = evaluatePixAccPaint(dataset, imageList, probs, segmentFolder, varargin) 3 | % 4 | % Evaluate the current iteration of the SVM/CRF optimization in terms of pixel 5 | % accuracy. 6 | % 7 | % Note: Maxing the scores is better for classPixAcc, whereas summing them 8 | % is better fo pixAcc. 9 | % 10 | % Copyright by Holger Caesar, 2015 11 | 12 | % Parse input 13 | p = inputParser; 14 | addParameter(p, 'printStatus', true); 15 | parse(p, varargin{:}); 16 | 17 | printStatus = p.Results.printStatus; 18 | 19 | % Check inputs 20 | assert(~isempty(dataset)); 21 | assert(~isempty(probs)); 22 | assert(numel(imageList) == numel(probs)); 23 | 24 | % Get parameters 25 | imageCount = numel(imageList); 26 | labelCount = dataset.labelCount; 27 | 28 | % Initialize 29 | pixCorrectHisto = zeros(labelCount, 1); 30 | pixTotalHisto = zeros(labelCount, 1); 31 | 32 | % Compute accuracy to ground-truth 33 | for imageIdx = 1 : imageCount, 34 | if printStatus, 35 | printProgress('Evaluating pixel accuracy for image', imageIdx, imageCount); 36 | end; 37 | 38 | % Get GT labels 39 | imageName = imageList{imageIdx}; 40 | labelMap = dataset.getImLabelMap(imageName); 41 | 42 | % Only create an outputMap if regions have been assigned 43 | if ~isempty(probs{imageIdx}), 44 | % Get segmentation 45 | segmentPath = fullfile(segmentFolder, [imageName, '.mat']); 46 | segmentStruct = load(segmentPath, 'propBlobs'); 47 | propBlobs = segmentStruct.propBlobs(:); 48 | propCount = numel(propBlobs); 49 | assert(propCount == size(probs{imageIdx}, 1)); 50 | 51 | % Get assignments and probs 52 | regionProbs = probs{imageIdx}; 53 | [~, regionAss] = max(regionProbs, [], 2); 54 | 55 | % Sort blobs (assignments, probs) by probs 56 | probsInds = sub2ind([propCount, labelCount], (1:propCount)', regionAss); 57 | regionProbs = regionProbs(probsInds); 58 | [regionProbs, sortOrder] = sort(regionProbs, 'descend'); 59 | regionAss = regionAss(sortOrder); 60 | propBlobs = propBlobs(sortOrder); 61 | 62 | % Extract most likely label per pixel (not region proposal) 63 | outputMap = blobProbsToOutputMap(propBlobs, regionAss, regionProbs, size(labelMap)); 64 | else 65 | if printStatus, 66 | % As this warning is quite common, we want to surpress it if specified 67 | fprintf('Warning: Creating dummy output map, as no regions have been assigned to this image!\n'); 68 | end; 69 | outputMap = nan(size(labelMap)); 70 | end; 71 | 72 | % Go through all ground-truth labels 73 | labelMapUn = unique(labelMap(:)); 74 | labelMapUn(labelMapUn == 0) = []; 75 | for labelMapUnIdx = 1 : numel(labelMapUn), 76 | label = labelMapUn(labelMapUnIdx); 77 | 78 | selection = labelMap(:) == label; 79 | pixCorrectHisto(label) = pixCorrectHisto(label) + sum(outputMap(selection) == label); 80 | pixTotalHisto(label) = pixTotalHisto(label) + sum(selection); 81 | end; 82 | end; 83 | 84 | % Compute total accuracy 85 | pixAcc = sum(pixCorrectHisto) / sum(pixTotalHisto); 86 | classPixAccs = pixCorrectHisto ./ pixTotalHisto; 87 | meanClassPixAcc = nanmean(classPixAccs); -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/+dagnn/GeneralSigmoid.m: -------------------------------------------------------------------------------- 1 | classdef GeneralSigmoid < dagnn.ElementWise 2 | % GeneralSigmoid 3 | % 4 | % Implements a general sigmoid layer (i.e. sigmoid of a 1d linear function): 5 | % S(x) = 1 / (1 + exp(- (ax + b) )) 6 | % 7 | % Note: This class is currently not used anywhere. 8 | % 9 | % For the derivatives check: 10 | % http://www.wolframalpha.com/input/?i=d%2Fdx+sigmoid%28ax%2Bb%29 11 | % http://www.wolframalpha.com/input/?i=d%2Fda+sigmoid%28ax%2Bb%29 12 | % http://www.wolframalpha.com/input/?i=d%2Fdb+sigmoid%28ax%2Bb%29 13 | % 14 | % Copyright by Holger Caesar, 2016 15 | 16 | properties 17 | numClasses = 0; 18 | end 19 | 20 | methods 21 | function obj = GeneralSigmoid(varargin) 22 | obj.load(varargin); 23 | end 24 | 25 | function outputs = forward(obj, inputs, params) 26 | % Get inputs 27 | assert(numel(inputs) == 1); 28 | assert(numel(params) == 2); 29 | x = inputs{1}; 30 | a = params{1}; 31 | b = params{2}; 32 | assert(size(x, 3) == size(a, 1) && ... 33 | size(x, 3) == size(b, 1)); 34 | 35 | % Reshape parameters 36 | a = reshape(a, 1, 1, [], 1); 37 | b = reshape(b, 1, 1, [], 1); 38 | 39 | y = obj.sigmoid(x, a, b); 40 | outputs{1} = y; 41 | end 42 | 43 | function [derInputs, derParams] = backward(obj, inputs, params, derOutputs) 44 | 45 | % Get inputs 46 | assert(numel(derOutputs) == 1); 47 | x = inputs{1}; 48 | a = params{1}; 49 | b = params{2}; 50 | dzdy = derOutputs{1}; 51 | assert(all(size(x) == size(dzdy))); 52 | 53 | % Reshape parameters 54 | a = reshape(a, 1, 1, [], 1); 55 | b = reshape(b, 1, 1, [], 1); 56 | 57 | % Compute outputs and gradients 58 | % Note: this can be further optimized by summing first and then 59 | % multiplying. 60 | y = obj.sigmoid(x, a, b); 61 | dzdb = dzdy .* (y .* (1 - y)); % dzdb = dzdy * dydb 62 | dzda = bsxfun(@times, dzdb, x); % dzda = dzdy * dyda 63 | dzdx = bsxfun(@times, dzdb, a); % dzdx = dzdy * dydx 64 | 65 | % Sum over regions (not for dzdx!) 66 | dzda = reshape((sum(dzda, 4)), [], 1); 67 | dzdb = reshape((sum(dzdb, 4)), [], 1); 68 | 69 | % Store outputs 70 | assert(all(size(x) == size(dzdx))); 71 | derInputs{1} = dzdx; 72 | derParams{1} = dzda; 73 | derParams{2} = dzdb; 74 | end 75 | 76 | function params = initParams(obj) 77 | % Note that compared to the Caesar BMVC 2015 paper, we use the proper 78 | % sigmoid function with the "-" sign. Hence the "a" parameter 79 | % should be positive! 80 | params{1} = repmat(single(1), [obj.numClasses, 1]); 81 | params{2} = repmat(single(0), [obj.numClasses, 1]); %zeros([obj.numClasses, 1], 'single'); 82 | end 83 | end 84 | 85 | methods (Access = protected) 86 | function[y] = sigmoid(obj, x, a, b) %#ok 87 | 88 | linear = bsxfun(@plus, bsxfun(@times, x, a), b); 89 | y = 1 ./ (1 + exp(- (linear))); 90 | end 91 | end 92 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/+dagnn/SegmentationLossSemiSupervised.m: -------------------------------------------------------------------------------- 1 | classdef SegmentationLossSemiSupervised < dagnn.Loss 2 | % SegmentationLossSemiSupervised 3 | % 4 | % Similar to dagnn.SegmentationLoss, but also sets pixel weights. 5 | % 6 | % Inputs: prediction, labels, labelsImage, [classWeights], 7 | % isWeaklySupervised, [masksThingsCell] 8 | % Outputs: loss 9 | % 10 | % Note: 11 | % - classWeights can be empty, which means they are ignored. 12 | % - labels or labelsImage can be empty. 13 | % 14 | % Copyright by Holger Caesar, 2016 15 | 16 | properties 17 | layerFS 18 | layerWS 19 | end 20 | 21 | methods 22 | function outputs = forward(obj, inputs, params) %#ok 23 | assert(numel(inputs) == 6); 24 | scoresMap = inputs{1}; 25 | labels = inputs{2}; 26 | labelsImage = inputs{3}; 27 | classWeights = inputs{4}; 28 | isWeaklySupervised = inputs{5}; 29 | masksThingsCell = inputs{6}; 30 | assert(~isempty(isWeaklySupervised)); 31 | 32 | if isWeaklySupervised 33 | outputs = obj.layerWS.forward({scoresMap, labelsImage, classWeights, masksThingsCell}, {}); 34 | else 35 | outputs = obj.layerFS.forward({scoresMap, labels, classWeights}, {}); 36 | end 37 | 38 | % Combine loss statistics 39 | imageCount = size(scoresMap, 4); 40 | n = obj.numAveraged; 41 | m = n + imageCount; 42 | obj.average = (n * obj.average + double(gather(outputs{1}))) / m; 43 | obj.numAveraged = m; 44 | end 45 | 46 | function [derInputs, derParams] = backward(obj, inputs, ~, derOutputs) 47 | scoresMap = inputs{1}; 48 | labels = inputs{2}; 49 | labelsImage = inputs{3}; 50 | classWeights = inputs{4}; 51 | isWeaklySupervised = inputs{5}; 52 | 53 | if isWeaklySupervised 54 | derInputs = obj.layerWS.backward({scoresMap, labelsImage, classWeights}, {}, derOutputs); 55 | else 56 | derInputs = obj.layerFS.backward({scoresMap, labels, classWeights}, {}, derOutputs); 57 | end 58 | derInputs{5} = []; 59 | derInputs{6} = []; 60 | derParams = {}; 61 | end 62 | 63 | function obj = SegmentationLossSemiSupervised(varargin) 64 | obj.load(varargin); 65 | end 66 | 67 | function forwardAdvanced(obj, layer) 68 | % Modification: Overrides standard forward pass to avoid giving up when any of 69 | % the inputs is empty. 70 | 71 | in = layer.inputIndexes; 72 | out = layer.outputIndexes; 73 | par = layer.paramIndexes; 74 | net = obj.net; 75 | inputs = {net.vars(in).value}; 76 | 77 | % clear inputs if not needed anymore 78 | for v = in 79 | net.numPendingVarRefs(v) = net.numPendingVarRefs(v) - 1; 80 | if net.numPendingVarRefs(v) == 0 81 | if ~net.vars(v).precious && ~net.computingDerivative && net.conserveMemory 82 | net.vars(v).value = []; 83 | end 84 | end 85 | end 86 | 87 | % call the simplified interface 88 | outputs = obj.forward(inputs, {net.params(par).value}); 89 | for oi = 1:numel(out) 90 | net.vars(out(oi)).value = outputs{oi}; 91 | end 92 | end 93 | end 94 | end -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/vl_nnloss_regress.m: -------------------------------------------------------------------------------- 1 | function Y = vl_nnloss_regress(X, t, dzdy, varargin) 2 | % vl_nnloss_regress(X, t, dzdy, varargin) 3 | % 4 | % Performs loss incurred by the predicted values X with respect 5 | % to the target values t. 6 | % 7 | % It is assumed that each vector in X is aligned w.r.t. its last dimension: 8 | % 2-dimensional vectors are row vectors 9 | % 3-dimensional vectors are aligned in the z-axis 10 | % 11 | % Possible loss: {'L1', 'L2', and 'Smooth'}, where Smooth is L2 up until the 12 | % but the maximum derivative with respect to the loss is capped to +/- 13 | % opts.smoothMaxDiff. For opts.smoothMaxDiff = 1, smooth is L2 when |X| < 1 and L1 14 | % otherwise. This form of smooth was used by Girshick to be less insensitive to outliers 15 | % while having more sensible gradient updates close to the target. The version here 16 | % is more flexible. 17 | % 18 | % Jasper - 2015 19 | 20 | % Set standard parameters 21 | opts.instanceWeights = []; 22 | opts.loss = 'L2'; 23 | opts.smoothMaxDiff = 1; 24 | opts = vl_argparse(opts, varargin); 25 | 26 | if isempty(opts.instanceWeights) 27 | instanceWeights = ones(size(X, ndims(X)), 1, 'single'); 28 | else 29 | instanceWeights = opts.instanceWeights; 30 | end 31 | 32 | % Display warning once 33 | warning('NotTested:regressloss', ... 34 | 'No loss has been thoroughly tested yet'); 35 | warning('off', 'NotTested:regressloss'); 36 | 37 | assert(isequal(size(X), size(t))); 38 | 39 | % Just compute loss 40 | if nargin == 2 || isempty(dzdy) 41 | switch lower(opts.loss) 42 | case 'l2' 43 | diff = X - t; 44 | Y = diff .* diff / 2; 45 | case 'l1' 46 | Y = abs(X - t); 47 | case 'smooth' 48 | diff = X - t; 49 | 50 | % Loss is L2 for part below opts.smoothMaxDiff and L1 for above. 51 | diffT = diff; 52 | maskTooHigh = (diff > opts.smoothMaxDiff); 53 | maskTooLow = (diff < -opts.smoothMaxDiff); 54 | diffT(maskTooHigh) = opts.smoothMaxDiff; 55 | diffT(maskTooLow) = -opts.smoothMaxDiff; 56 | squaredPart = (diffT .* diffT / 2); 57 | 58 | nonSquaredPart = zeros(size(squaredPart), 'like', X); 59 | nonSquaredPart(maskTooHigh) = diff(maskTooHigh) - opts.smoothMaxDiff; 60 | nonSquaredPart(maskTooLow) = -diff(maskTooLow) - opts.smoothMaxDiff; 61 | nonSquaredPart = nonSquaredPart * opts.smoothMaxDiff; % Derivative of loss is opts.smoothMaxDiff 62 | 63 | Y = squaredPart + nonSquaredPart; 64 | otherwise 65 | error('Incorrect loss: %s', opts.loss); 66 | end 67 | % Sum loss in all dimensions but the last 68 | for i=1:ndims(Y)-1 69 | Y = sum(Y, i); 70 | end 71 | 72 | % Weight loss using instanceWeights 73 | Y = instanceWeights(:)' * Y(:); 74 | else 75 | % Get derivatives w.r.t. loss function 76 | switch lower(opts.loss) 77 | case 'l2' 78 | Y = dzdy * (X-t); 79 | case 'l1' 80 | Y = dzdy * sign(X-t); 81 | case 'smooth' 82 | diff = X-t; 83 | diff(diff > opts.smoothMaxDiff) = opts.smoothMaxDiff; 84 | diff(diff < -opts.smoothMaxDiff) = -opts.smoothMaxDiff; 85 | Y = dzdy * (diff); 86 | otherwise 87 | error('Incorrect loss: %s', opts.loss); 88 | end 89 | 90 | % Get instanceWeights in the right shape (aligned according to last dimension) 91 | iwSize = ones(1, ndims(Y)); 92 | iwSize(end) = length(instanceWeights); 93 | instanceWeights = reshape(instanceWeights, iwSize); 94 | 95 | % Perform instance weighting 96 | Y = bsxfun(@times, Y, instanceWeights); 97 | 98 | end 99 | -------------------------------------------------------------------------------- /matconvnet-calvin/matlab/+dagnn/LossWeighted.m: -------------------------------------------------------------------------------- 1 | classdef LossWeighted < dagnn.Loss 2 | % LossWeighted 3 | % 4 | % The same as dagnn.Loss, but (optionally) allows to specify 5 | % instanceWeights as an additional input. 6 | % 7 | % inputs: scoresSP, labelsSP, [instanceWeightsSP] 8 | % outputs: loss 9 | % 10 | % Note: If you use instanceWeights to change the total weight of this 11 | % batch, then you shouldn't use the default extractStatsFn anymore, as 12 | % its average-loss depends on the number of boxes in the batch. 13 | % 14 | % Copyright by Holger Caesar, 2015 15 | 16 | properties (Transient) 17 | numSubBatches = 0; 18 | end 19 | 20 | methods 21 | function outputs = forward(obj, inputs, params) %#ok 22 | 23 | % Get inputs 24 | assert(numel(inputs) == 3); 25 | scoresSP = inputs{1}; 26 | labelsSP = inputs{2}; 27 | instanceWeightsSP = inputs{3}; 28 | 29 | % Check inputs 30 | assert(~isempty(labelsSP)); 31 | if ~isempty(instanceWeightsSP) 32 | assert(numel(instanceWeightsSP) == size(scoresSP, 4)); 33 | assert(numel(instanceWeightsSP) == size(labelsSP, ndims(labelsSP))); 34 | end 35 | 36 | % Compute loss 37 | outputs{1} = vl_nnloss(scoresSP, labelsSP, [], 'loss', obj.loss, 'instanceWeights', instanceWeightsSP); 38 | 39 | % Update statistics 40 | n = obj.numAveraged; 41 | m = n + size(inputs{1}, 4); 42 | obj.average = (n * obj.average + gather(outputs{1})) / m; 43 | obj.numAveraged = m; 44 | obj.numSubBatches = obj.numSubBatches + 1; 45 | assert(~isnan(obj.average)); 46 | end 47 | 48 | function [derInputs, derParams] = backward(obj, inputs, params, derOutputs) %#ok 49 | 50 | % Get inputs 51 | assert(numel(derOutputs) == 1); 52 | scoresSP = inputs{1}; 53 | labelsSP = inputs{2}; 54 | instanceWeightsSP = inputs{3}; 55 | 56 | derInputs{1} = vl_nnloss(scoresSP, labelsSP, derOutputs{1}, 'loss', obj.loss, 'instanceWeights', instanceWeightsSP); 57 | derInputs{2} = []; 58 | derInputs{3} = []; 59 | derParams = {}; 60 | end 61 | 62 | function obj = LossWeighted(varargin) 63 | obj = obj@dagnn.Loss(varargin{:}); 64 | end 65 | 66 | function reset(obj) 67 | reset@dagnn.Loss(obj); 68 | obj.numSubBatches = 0; 69 | end 70 | 71 | function forwardAdvanced(obj, layer) 72 | % Modification: Overrides standard forward pass to avoid giving up when any of 73 | % the inputs is empty. 74 | 75 | in = layer.inputIndexes; 76 | out = layer.outputIndexes; 77 | par = layer.paramIndexes; 78 | net = obj.net; 79 | inputs = {net.vars(in).value}; 80 | 81 | % clear inputs if not needed anymore 82 | for v = in 83 | net.numPendingVarRefs(v) = net.numPendingVarRefs(v) - 1; 84 | if net.numPendingVarRefs(v) == 0 85 | if ~net.vars(v).precious && ~net.computingDerivative && net.conserveMemory 86 | net.vars(v).value = []; 87 | end 88 | end 89 | end 90 | 91 | % call the simplified interface 92 | outputs = obj.forward(inputs, {net.params(par).value}); 93 | for oi = 1:numel(out) 94 | net.vars(out(oi)).value = outputs{oi}; 95 | end 96 | end 97 | end 98 | end --------------------------------------------------------------------------------