├── reviewResults.m
├── removeHits.m
├── submitHits.m
├── getResults.m
├── labelme.question
├── labelme.properties
├── reviewResults.sh
├── approveAndDeleteResults.sh
├── getResults.sh
├── run.sh
├── isSandboxMode.m
├── demo.m
└── generateLabelMeInputFile.m


/reviewResults.m:
--------------------------------------------------------------------------------
 1 | function reviewResults(sandbox)
 2 | 
 3 | if sandbox==isSandboxMode
 4 |   system('./reviewResults.sh')
 5 | else
 6 |   fname = fullfile(getenv('MTURK_CMD_HOME'),'bin','mturk.properties');
 7 |   error('sandbox flag is not set in agreement with the service_url variable in %s',fname);
 8 | end
 9 | 
10 | return;
11 | 
12 | % $$$ if sandbox
13 | % $$$     unix('./reviewResults.sh -sandbox')
14 | % $$$ else
15 | % $$$     unix('./reviewResults.sh')
16 | % $$$ end
17 | 


--------------------------------------------------------------------------------
/removeHits.m:
--------------------------------------------------------------------------------
 1 | function removeHits(sandbox)
 2 | 
 3 | if sandbox==isSandboxMode
 4 |   unix('./approveAndDeleteResults.sh')
 5 | else
 6 |   fname = fullfile(getenv('MTURK_CMD_HOME'),'bin','mturk.properties');
 7 |   error('sandbox flag is not set in agreement with the service_url variable in %s',fname);
 8 | end
 9 | 
10 | return;
11 | 
12 | % $$$ if sandbox
13 | % $$$     unix('./approveAndDeleteResults.sh -sandbox')
14 | % $$$ else
15 | % $$$     unix('./approveAndDeleteResults.sh')
16 | % $$$ end
17 | 


--------------------------------------------------------------------------------
/submitHits.m:
--------------------------------------------------------------------------------
 1 | function submitHits(sandbox)
 2 | 
 3 | if sandbox==isSandboxMode
 4 |   system('./run.sh')
 5 | else
 6 |   fname = fullfile(getenv('MTURK_CMD_HOME'),'bin','mturk.properties');
 7 |   error('\n\nsandbox flag is not set in agreement with the service_url variable in %s\n\n',fname);
 8 | end
 9 | 
10 | return;
11 | 
12 | %%%%%
13 | % The -sandbox flag does not work anymore.
14 | 
15 | % $$$ if sandbox
16 | % $$$     unix('./run.sh -sandbox ')
17 | % $$$ else
18 | % $$$     unix('./run.sh')
19 | % $$$ end
20 | 


--------------------------------------------------------------------------------
/getResults.m:
--------------------------------------------------------------------------------
 1 | function getResults(sandbox)
 2 | 
 3 | if sandbox==isSandboxMode
 4 |   system('./getResults.sh')
 5 | else
 6 |   fname = fullfile(getenv('MTURK_CMD_HOME'),'bin','mturk.properties');
 7 |   error('sandbox flag is not set in agreement with the service_url variable in %s',fname);
 8 | end
 9 | 
10 | return;
11 | 
12 | 
13 | % $$$ if sandbox
14 | % $$$     unix('./getResults.sh -sandbox')   % this will create the file labelme.results.xls
15 | % $$$ else
16 | % $$$     unix('./getResults.sh')
17 | % $$$ end
18 | 


--------------------------------------------------------------------------------
/labelme.question:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0"?>
 2 | <!-- Note the inclusion of the $urls variable which is defined as a field in the input file.  Apache Velocity is the template engine that is
 3 | used to perform the merging of variables into template files.  You can learn more about Velocity's capabilities at http://velocity.apache.org.
 4 | -->
 5 | <ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd">
 6 |     <ExternalURL>$urls</ExternalURL>
 7 |     <FrameHeight>550</FrameHeight>
 8 | </ExternalQuestion>
 9 | 
10 | 


--------------------------------------------------------------------------------
/labelme.properties:
--------------------------------------------------------------------------------
 1 | ######################################
 2 | ## External HIT Properties
 3 | ######################################
 4 | 
 5 | title:LabelMe: Label objects in this image
 6 | description:Please look at this image, outline an object, and enter the object's name.
 7 | keywords:LabelMe annotation image computer vision object recognition label
 8 | reward:0.01
 9 | assignments:1
10 | annotation:${urls}
11 | 
12 | ######################################
13 | ## HIT Timing Properties
14 | ######################################
15 | 
16 | # this Assignment Duration value is 60 * 60 = 1 hour
17 | assignmentduration:3600
18 | 
19 | # this HIT Lifetime value is 60*60*24*3 = 3 days
20 | hitlifetime:259200
21 | 
22 | # this Auto Approval period is 60*60*24*15 = 15 days
23 | autoapprovaldelay:1296000
24 | 


--------------------------------------------------------------------------------
/reviewResults.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env sh
 2 | #
 3 | # Copyright 2008 Amazon Technologies, Inc.
 4 | # 
 5 | # Licensed under the Amazon Software License (the "License");
 6 | # you may not use this file except in compliance with the License.
 7 | # You may obtain a copy of the License at:
 8 | # 
 9 | # http://aws.amazon.com/asl
10 | # 
11 | # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
12 | # OR CONDITIONS OF ANY KIND, either express or implied. See the
13 | # License for the specific language governing permissions and
14 | # limitations under the License.
15 | 
16 | DIR=$(pwd);
17 | cd $MTURK_CMD_HOME/bin
18 | ./reviewResults.sh $1 $2 $3 $4 $5 $6 $7 $8 $9 -resultsfile $DIR/labelme.results.xls
19 | cd $DIR
20 |  
21 | 
22 | #!/usr/bin/env bash
23 | #cd ../..
24 | #cd bin
25 | #./reviewResults.sh $1 $2 $3 $4 $5 $6 $7 $8 $9 -resultsfile ../samples/LabelMe/external_hit.results
26 | #cd ..
27 | #cd samples/LabelMe
28 | 


--------------------------------------------------------------------------------
/approveAndDeleteResults.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env sh
 2 | #
 3 | # Copyright 2008 Amazon Technologies, Inc.
 4 | # 
 5 | # Licensed under the Amazon Software License (the "License");
 6 | # you may not use this file except in compliance with the License.
 7 | # You may obtain a copy of the License at:
 8 | # 
 9 | # http://aws.amazon.com/asl
10 | # 
11 | # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
12 | # OR CONDITIONS OF ANY KIND, either express or implied. See the
13 | # License for the specific language governing permissions and
14 | # limitations under the License.
15 |  
16 | 
17 | DIR=$(pwd);
18 | cd $MTURK_CMD_HOME/bin
19 | ./deleteHITs.sh $1 $2 $3 $4 $5 $6 $7 $8 $9  -successfile $DIR/labelme.success -approve -expire 
20 | cd $DIR
21 | 
22 | #cd ../..
23 | #cd bin
24 | #./deleteHITs.sh $1 $2 $3 $4 $5 $6 $7 $8 $9  -successfile ../samples/LabelMe/external_hit.success -approve -expire 
25 | #cd ..
26 | #cd samples/LabelMe
27 | 


--------------------------------------------------------------------------------
/getResults.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env sh
 2 | #
 3 | # Copyright 2008 Amazon Technologies, Inc.
 4 | # 
 5 | # Licensed under the Amazon Software License (the "License");
 6 | # you may not use this file except in compliance with the License.
 7 | # You may obtain a copy of the License at:
 8 | # 
 9 | # http://aws.amazon.com/asl
10 | # 
11 | # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
12 | # OR CONDITIONS OF ANY KIND, either express or implied. See the
13 | # License for the specific language governing permissions and
14 | # limitations under the License.
15 | 
16 | DIR=$(pwd);
17 | cd $MTURK_CMD_HOME/bin
18 | ./getResults.sh $1 $2 $3 $4 $5 $6 $7 $8 $9 -successfile $DIR/labelme.success -outputfile $DIR/labelme.results.xls
19 | cd $DIR
20 |  
21 | 
22 | #cd ../..
23 | #cd bin
24 | #./getResults.sh $1 $2 $3 $4 $5 $6 $7 $8 $9 -successfile ../samples/LabelMe/external_hit.success -outputfile ../samples/LabelMe/external_hit.results
25 | #cd ..
26 | #cd samples/LabelMe
27 | 


--------------------------------------------------------------------------------
/run.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env sh
 2 | #
 3 | # Copyright 2008 Amazon Technologies, Inc.
 4 | # 
 5 | # Licensed under the Amazon Software License (the "License");
 6 | # you may not use this file except in compliance with the License.
 7 | # You may obtain a copy of the License at:
 8 | # 
 9 | # http://aws.amazon.com/asl
10 | # 
11 | # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
12 | # OR CONDITIONS OF ANY KIND, either express or implied. See the
13 | # License for the specific language governing permissions and
14 | # limitations under the License.
15 | 
16 | DIR=$(pwd);
17 | cd $MTURK_CMD_HOME/bin
18 | ./loadHITs.sh $1 $2 $3 $4 $5 $6 $7 $8 $9 -label $DIR/labelme -input $DIR/labelme.input -question $DIR/labelme.question -properties $DIR/labelme.properties
19 | cd $DIR
20 |  
21 | #echo $MTURK_CMD_HOME
22 | #cd ../..
23 | #cd bin
24 | #./loadHITs.sh $1 $2 $3 $4 $5 $6 $7 $8 $9 -label ../samples/LabelMe/external_hit -input ../samples/LabelMe/external_hit.input -question ../samples/LabelMe/external_hit.question -properties ../samples/LabelMe/external_hit.properties
25 | #cd ..
26 | #cd samples/LabelMe
27 | 


--------------------------------------------------------------------------------
/isSandboxMode.m:
--------------------------------------------------------------------------------
 1 | function isSandbox = isSandboxMode
 2 | % This function checks mturk.properties to see if we are in sandbox mode
 3 | % or live (pay money) mode.
 4 | 
 5 | urlSandbox = 'https://mechanicalturk.sandbox.amazonaws.com/?Service=AWSMechanicalTurkRequester';
 6 | urlLive = 'https://mechanicalturk.amazonaws.com/?Service=AWSMechanicalTurkRequester';
 7 | 
 8 | % Get location of aws-mturk-clt-1.3.0:
 9 | mturk_cmd_home = getenv('MTURK_CMD_HOME');
10 | 
11 | % Check to see if the MTURK_CMD_HOME environment variable has been set:
12 | if isempty(mturk_cmd_home)
13 |   error(sprintf('\n\nThe MTURK_CMD_HOME environment variable has not been set. To set within bash, run the following:\n\nexport MTURK_CMD_HOME=/your/path/to/aws-mturk-clt-1.3.0\n\n'));
14 |   return;
15 | end
16 | 
17 | % Variables to determine:
18 | isSet = 0; % Whether service_url is set.
19 | isSandbox = 0; % 1 - sandbox mode; 0 - live (pay money) mode
20 | 
21 | % Read mturk.properties:
22 | fname = fullfile(mturk_cmd_home,'bin','mturk.properties');
23 | fp = fopen(fname);
24 | while 1
25 |   tline = fgetl(fp);
26 |   if ~ischar(tline), break, end
27 |   loc = strfind(tline,'service_url');
28 |   if loc==1
29 |     if isSet
30 |       error('\n\nThere are multiple lines that set service_url inside:\n\n%s\n\nRemove or comment out all but one of the lines.',fname);
31 |       return;
32 |     end
33 |     isSet = 1;
34 |     if strfind(tline,urlSandbox)
35 |       isSandbox = 1;
36 |     elseif strfind(tline,urlLive)
37 |       isSandbox = 0;
38 |     else
39 |       error('\n\nservice_url is not set to a valid URL inside:\n\n%s\n\nSet to one of the following URLs:\n\nservice_url=%s # For live (pay money) mode\n\nservice_url=%s # For sandbox mode\n\n',fname,urlLive,urlSandbox);
40 |       return;
41 |     end
42 |   end
43 | end
44 | fclose(fp);
45 | 
46 | % Check if service_url has been set:
47 | if isSet == 0
48 |   error('\n\nNeed to set service_url inside:\n\n%s\n\nInclude one of the following lines:\n\nservice_url=%s # For live (pay money) mode\n\nservice_url=%s # For sandbox mode\n\n',fname,urlLive,urlSandbox);
49 |   return;
50 | end
51 | 


--------------------------------------------------------------------------------
/demo.m:
--------------------------------------------------------------------------------
 1 | % This script demonstrates how to submit tasks, retrieve results, and pay
 2 | % Mechanical Turk workers.
 3 | 
 4 | % For this to work, make sure the following environment variables are defined:
 5 | % MTURK_CMD_HOME=/your/path/to/aws-mturk-clt-1.3.0
 6 | % JAVA_HOME=/your/path/to/java
 7 | 
 8 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 9 | % STEP 0: Download the LabelMe Matlab Toolbox and add it to the path
10 | % http://labelme.csail.mit.edu/LabelMeToolbox
11 | addpath /your/path/to/LabelMeToolbox;
12 | 
13 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
14 | % STEP 1: PREDEFINE THE LIST OF TASKS (each image to label is one task)
15 | 
16 | % You will need your LabelMe username and collection.  To get this,
17 | % simply sign up for a LabelMe account (http://labelme.csail.mit.edu),
18 | % create a collection, and add images to the collection.  Finally, set
19 | % the following two variables with your LabelMe username and collection name.
20 | username = ENTER_YOUR_USERNAME_HERE;
21 | collection = ENTER_YOUR_COLLECTION_NAME_HERE;
22 | 
23 | % Set this to 0 to submit tasks to users (pay money mode).
24 | % Set this to 1 to debug in the sandbox (free, no money).
25 | %
26 | % The sandbox is free to use (you have fake credits on this server), so
27 | % you can test here to make sure everything works. When you are ready for
28 | % the real thing, set sandbox=0 and run the remaining commands in this
29 | % file. Make sure to add money to your Amazon account before running on
30 | % the real server.
31 | sandbox = 1;
32 | 
33 | % Make sure to also set also set the "service_url" variable inside
34 | % MTURK_CMD_HOME/bin/mturk.properties
35 | %
36 | % Sandbox mode:
37 | % service_url=https://mechanicalturk.sandbox.amazonaws.com/?Service=AWSMechanicalTurkRequester
38 | %
39 | % Live (pay money) mode:
40 | % service_url=https://mechanicalturk.amazonaws.com/?Service=AWSMechanicalTurkRequester
41 | 
42 | generateLabelMeInputFile(username,collection,sandbox);
43 | % HITS: you can remove specific tasks by editing the file 'labelme.input' and
44 | % deleting lines from it
45 | % PRICE: To change the price of a task, edit the file 'labelme.properties'.
46 | % Currently, each HIT is priced at $0.01 per image. Please see the
47 | % documentation for the Amazon Mechanical Turk Command Line Tools for
48 | % more information.
49 | 
50 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
51 | % STEP 2: SUBMIT HITS
52 | %
53 | submitHits(sandbox);
54 | % You will see messages indicating that the jobs are being submitted. At
55 | % the end, there will be a URL that points to a preview page for the
56 | % HIT. You can go to that URL and try out the HIT. All of the collected
57 | % annotations are stored on the LabelMe servers, so you can download them
58 | % immediately.
59 | %
60 | % Note that Mechanical Turk workers will see the following set of
61 | % instructions when they preview the HIT:
62 | %
63 | % http://labelme.csail.mit.edu/Release3.0/annotationTools/html/mt_instructions.html
64 | %
65 | % Once they accept the HIT, then they will be directed to the annotation
66 | % tool.  If you wish to change the instructions page that is shown during
67 | % preview mode, please see how to set the 'mt_intro' variable within the
68 | % "Advanced features" section on the LabelMe/Mechanical Turk instructions
69 | % page.
70 | return
71 | 
72 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
73 | % STEP 3: DOWNLOAD SUMMARY RESULTS and review them:
74 | %
75 | getResults(sandbox);
76 | % This will create the file 'labelme.results.xls'. This is a comma
77 | % separated file, which can be viewed with Excel. This file lists the
78 | % status of each HIT, as well as outputs from the annotation task, such
79 | % as number of annotations labeled, last object labeled, etc.
80 | return
81 | 
82 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
83 | % STEP 4: PAY WORKERS. It is important to pay the workers as soon as possible.
84 | %
85 | reviewResults(sandbox);
86 | return
87 | 
88 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
89 | % STEP 5: REMOVE HITS FROM AMAZON SERVER.
90 | %
91 | removeHits(sandbox);
92 | % When all of the HITs are completed, you can remove them from the Amazon
93 | % server.
94 | return
95 | 
96 | 
97 | 


--------------------------------------------------------------------------------
/generateLabelMeInputFile.m:
--------------------------------------------------------------------------------
  1 | function generateLabelMeInputFile(username,collection,sandbox,N,HOMEIMAGES,introPage,outputFile,Nimages)
  2 | % Generate Amazon Mechanical Turk input file with set of images for
  3 | % workers to label.
  4 | %
  5 | % To generate the Mechanical Turk input file, simply run the following in
  6 | % Matlab:
  7 | %
  8 | % generateLabelMeInputFile(username,collection)
  9 | %
 10 | % The script will produce a file called "labelme.input".  
 11 | %
 12 | % This function is more flexible.  Here are additional parameters that
 13 | % can be used:
 14 | %
 15 | % generateLabelMeInputFile(username,collection,sandbox,N,HOMEIMAGES,introPage,outputFile)
 16 | %
 17 | % Inputs:
 18 | % sandbox - Set to 1 to run on Mechanical Turk Sandbox to debug (0 is
 19 | %           default).
 20 | % N - Number of required annotations the worker must enter in the image
 21 | %     to complete the task.  If 'inf' is entered, then the worker may
 22 | %     enter as many annotations as they wish.
 23 | % HOMEIMAGES - Path to FOLDER.  By default, this is set to the LabelMe
 24 | %              server.
 25 | % introPage - Set this to point to a URL with instructions for the
 26 | %             worker.  By default, this is set to
 27 | %             http://labelme.csail.mit.edu/mt_instructions.html
 28 | % outputFile - Set the name of the output file.  By default, this is
 29 | %              'labelme.input'.  It is recommended that this does not
 30 | %              change (otherwise the scripts to send the job to
 31 | %              Mechanical Turk will not run).
 32 | % Nimages - use only the first Nimages from the folder (by default create
 33 | %              HITS for all images)
 34 | 
 35 | baseURL = 'http://labelme.csail.mit.edu/Release3.0';
 36 | 
 37 | folder = fullfile('users',username,collection);
 38 | 
 39 | if nargin < 3
 40 |   sandbox = 0;
 41 | end
 42 | if nargin < 4
 43 |   N = 'inf'; % The MT worker can label as many polygons as they wish.
 44 | end
 45 | if nargin < 5
 46 |   HOMEIMAGES = fullfile(baseURL,'Images');
 47 | end
 48 | if nargin < 6
 49 |   introPage = [];
 50 | end
 51 | if nargin < 7
 52 |   outputFile = 'labelme.input';
 53 | end
 54 | 
 55 | if sandbox~=isSandboxMode
 56 |   fname = fullfile(getenv('MTURK_CMD_HOME'),'bin','mturk.properties');
 57 |   error('\n\nsandbox flag is not set in agreement with the service_url variable in %s\n\n',fname);
 58 | end
 59 | 
 60 | if ~isstr(N)
 61 |   N = lower(num2str(N));
 62 | end
 63 | 
 64 | % Get set of images:
 65 | if any(findstr(HOMEIMAGES,'http://'))
 66 |   % From LabelMe server:
 67 |   page = fullfile(HOMEIMAGES,folder);
 68 |   page = strrep(page, '\', '/');
 69 |   folders = urlread(page);
 70 |   folders = folders(1:length(folders));
 71 |   j1 = findstr(lower(folders), '<a href="');
 72 |   j2 = findstr(lower(folders), '</a>');
 73 |   Nfolders = length(j1);
 74 |   
 75 |   fn = 0;
 76 |   filenames = [];
 77 |   for f = 1:Nfolders
 78 |     tmp = folders(j1(f)+9:j2(f)-1);
 79 |     fin = findstr(tmp, '"');
 80 |     if length(findstr(tmp(1:fin(end)-1), '.jpg'))>0
 81 |       fn = fn+1;
 82 |       filenames(fn).name = tmp(1:fin(end)-1);
 83 |     end
 84 |   end
 85 | else
 86 |   % From local directory:
 87 |   filenames = dir(fullfile(HOMEIMAGES,folder,'*.jpg'));
 88 | end
 89 | 
 90 | % Get extra variables:
 91 | extraVars = '';
 92 | if sandbox
 93 |   extraVars = '&mt_sandbox=true';
 94 | % $$$   extraVars = '&amp;mt_sandbox=true';
 95 | end
 96 | if ~isempty(introPage)
 97 |   extraVars = [extraVars sprintf('&mt_intro=%s',introPage)];
 98 | % $$$   extraVars = [extraVars sprintf('&amp;mt_intro=%s',introPage)];
 99 | end
100 | 
101 | if nargin < 8
102 |   Nimages = length(filenames);
103 | else
104 |   Nimages = min(Nimages, length(filenames));
105 | end
106 | 
107 | % Write to output file:
108 | fp = fopen(outputFile,'w');
109 | fprintf(fp,'urls\n');
110 | for i = 1:Nimages
111 |   fprintf(fp,'%s/tool.html?collection=LabelMe&mode=mt&N=%s&folder=%s&image=%s%s\n',baseURL,N,folder,filenames(i).name,extraVars);
112 | % $$$   fprintf(fp,'%s/tool.html?collection=LabelMe&amp;mode=mt&amp;N=%s&amp;folder=%s&amp;image=%s%s\n',baseURL,N,folder,filenames(i).name,extraVars);
113 | end
114 | fclose(fp);
115 | 
116 | 
117 | % create tasks page:
118 | for i = 1:Nimages
119 |   folderlist{i} = folder;
120 |   filelist{i} = filenames(i).name;
121 | end
122 | disp('Open labelme.input.html to see the progress of the annotations')
123 | LMphotoalbum(folderlist, filelist, 'labelme.input.html');
124 | 


--------------------------------------------------------------------------------