├── README.md
├── _assets
    └── model.png
├── executor.py
├── generate_invalid_video_list.py
├── get_results.py
├── merge_raw_results.py
├── requirement.txt
├── run_mc.py
├── run_mc_clevrer.py
├── run_oe_clevrer.py
├── scripts
    ├── test_mc_release.sh
    └── test_oe_release.sh
├── simulation.py
├── test_mc.json
├── test_oe.json
├── test_submit.json
├── tools
    └── transform_anno.py
└── utils
    └── utils.py


/README.md:
--------------------------------------------------------------------------------
 1 | # ComPhy
 2 | This repository holds the code for the paper.
 3 | 
 4 | > 
 5 | **ComPhy: Compositional Physical Reasoning ofObjects and Events from Videos**, (ICLR 2022)
 6 | >
 7 | 
 8 | 
 9 | 
10 | [Project Website](https://comphyreasoning.github.io/)
11 | 
12 | [Evaluation Server](https://eval.ai/web/challenges/challenge-page/1770/overview)
13 | ## Framework
14 | <div align="center">
15 |   <img src="_assets/model.png" width="100%">
16 | </div>
17 | 
18 | ## Code Preparation
19 | ```
20 | git clone https://github.com/comphyreasoning/compositional_physics_learner.git 
21 | ```
22 | 
23 | ## Installation
24 | ```
25 | pip install -r requirements
26 | ```
27 | 
28 | ## Data Preparation
29 | - Download videos, video annotation, questions from the [project website](https://comphyreasoning.github.io/).
30 | 
31 | ## Fast Evaluation
32 | - Download the regional proposals with attribute and physical property prediction from the anonymous [Google drive](https://drive.google.com/file/d/1LsOU-OfODxx8gPj3XdPvvThITGRY3Poy/view?usp=sharing)
33 | - Download the dynamic predictions from the anonymous [Google drive](https://drive.google.com/file/d/10JyHNW0dJrmnLbIbr8TOgHTRuKdQWflD/view?usp=sharing)
34 | - Run executor for factual questions.
35 | ```
36 | sh scripts/test_oe_release.sh
37 | ```
38 | - Run executor for multiple-choice questions.
39 | ```
40 | sh scripts/test_mc_release.sh
41 | ```
42 | - Submit results onto the [evaluation server](https://eval.ai/web/challenges/challenge-page/1770/overview).
43 | 
44 | 
45 | ## Supporting sub-modules
46 | ### Physical Property Learner and Dynamic predictor
47 | Please refer to [this repo](https://github.com/zfchenUnique/property_learner_predictor) for property learning and dynamics prediction.
48 | ### Perception
49 | This module uses the public [NS-VQA](https://github.com/kexinyi/ns-vqa.git)'s perception module object detection and visual attribute extraction.
50 | ### Program parser 
51 | This module uses the public [NS-VQA](https://github.com/kexinyi/ns-vqa.git)'s program parser module to tranform language into executable programs.
52 | 


--------------------------------------------------------------------------------
/_assets/model.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zfchenUnique/compositional_physics_learner/ba34512385ad3171c6176363055336f728a6a47e/_assets/model.png


--------------------------------------------------------------------------------
/executor.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import json
  3 | 
  4 | import pdb
  5 | from IPython.core import ultratb
  6 | import sys
  7 | sys.excepthook = ultratb.FormattedTB(call_pdb=True)
  8 | 
  9 | 
 10 | COLORS = ['gray', 'red', 'blue', 'green', 'brown', 'yellow', 'cyan', 'purple']
 11 | MATERIALS = ['metal', 'rubber']
 12 | SHAPES = ['sphere', 'cylinder', 'cube']
 13 | MASS = [1, 5]
 14 | CHARGES = [-1, 0, 1]
 15 | DIR_ANGLE_TH = 42  # threshold for allowed angle deviation wrt each directions
 16 | 
 17 | class Executor():
 18 |     """Symbolic program executor for V-CLEVR questions"""
 19 | 
 20 |     def __init__(self, sim, args=None):
 21 |         self._set_sim(sim)
 22 |         self._register_modules()
 23 |         self.args = args
 24 | 
 25 |     def run(self, pg, debug=False):
 26 |         exe_stack = []
 27 |         for m in pg:
 28 |             #if m=="unique":
 29 |             #    import pdb
 30 |            #     pdb.set_trace()
 31 |             if m in ['<END>', '<NULL>']:
 32 |             #if m in ['<END>']:
 33 |                 break
 34 |             if m not in ['<START>']:
 35 |                 if m not in self.modules:
 36 |                     exe_stack.append(m)
 37 |                 else:
 38 |                     argv = []
 39 |                     for i in range(self.modules[m]['nargs']):
 40 |                         if exe_stack:
 41 |                             argv.insert(0, exe_stack.pop())
 42 |                         else:
 43 |                             return 'error'
 44 |                     
 45 |                     step_output = self.modules[m]['func'](*argv)
 46 |                     if step_output == 'error':
 47 |                         #pdb.set_trace()
 48 |                         return 'error'
 49 |                     exe_stack.append(step_output)
 50 | 
 51 |                     if debug:
 52 |                         print('> %s%s' % (m, argv))
 53 |                         print(exe_stack)
 54 |                         print('\n')
 55 |         return str(exe_stack[-1])
 56 | 
 57 |     def _set_sim(self, sim):
 58 |         self.sim = sim
 59 |         self.all_objs = sim.get_visible_objs()
 60 |         self.existing_events = self._get_events(sim)
 61 |         self.unseens = self._get_unseen_events(sim)
 62 |         self.causal_traces = self._get_causal_traces(self.all_objs, self.existing_events)
 63 | 
 64 |     def _get_events(self, sim, drop_idx=None):
 65 |         events = [
 66 |             {
 67 |                 'type': 'start',
 68 |                 'frame': 0,
 69 |             },
 70 |             {
 71 |                 'type': 'end',
 72 |                 'frame': 125,
 73 |             },
 74 |         ]
 75 |         for io in sim.in_out:
 76 |             if io['frame'] < sim.n_vis_frames:
 77 |                 io_event = {
 78 |                     'type': io['type'],
 79 |                     'object': io['object'],
 80 |                     'frame': io['frame'],
 81 |                 }
 82 |                 if drop_idx is not None:
 83 |                     io_event = self._convert_event_idx_cf2gt(io_event, drop_idx)
 84 |                 events.append(io_event)
 85 |         for c in sim.collisions:
 86 |             if c['frame'] < sim.n_vis_frames:
 87 |                 col_event = {
 88 |                     'type': 'collision',
 89 |                     'object': c['object'],
 90 |                     'frame': c['frame']
 91 |                 }
 92 |                 if drop_idx is not None:
 93 |                     col_event = self._convert_event_idx_cf2gt(col_event, drop_idx)
 94 |                 events.append(col_event)
 95 |         return events
 96 | 
 97 |     def _get_unseen_events(self, sim):
 98 |         """Return a list of events (time_indicators) of events that are 
 99 |         going to happen
100 |         """
101 |         unseen_events = []
102 |         for io in self.sim.in_out:
103 |             if io['frame'] >= self.sim.n_vis_frames:
104 |                 io_event = {
105 |                     'type': io['type'],
106 |                     'object': [io['object']],
107 |                     'frame': io['frame'],
108 |                 }
109 |                 unseen_events.append(io_event)
110 |         for c in self.sim.collisions:
111 |             if c['frame'] >= self.sim.n_vis_frames:
112 |                 col_event = {
113 |                     'type': 'collision',
114 |                     'object': c['object'],
115 |                     'frame': c['frame'],
116 |                 }
117 |                 unseen_events.append(col_event)
118 |         return unseen_events
119 | 
120 |     def _get_causal_traces(self, objs, events):
121 |         """Compute the causal traces for each object"""
122 |         causal_traces = [[] for o in objs]
123 |         for e in self.existing_events:
124 |             if e['type'] not in ['start', 'end']:
125 |                 for o in e['object']:
126 |                     causal_traces[o].append(e)
127 |         for i, tr in enumerate(causal_traces):
128 |             tr = sorted(tr, key=lambda k: k['frame'])
129 |             causal_traces[i] = tr
130 |         return causal_traces
131 | 
132 |     def _convert_event_idx_cf2gt(self, event, drop_idx):
133 |         event_objs_converted = []
134 |         for o in event['object']:
135 |             if o >= drop_idx:
136 |                 event_objs_converted.append(o+1)
137 |             else:
138 |                 event_objs_converted.append(o)
139 |         event['object'] = event_objs_converted
140 |         return event
141 | 
142 |     def _register_modules(self):
143 |         self.modules = {
144 |             'objects': {'func': self.objects, 'nargs': 0},
145 |             'events': {'func': self.events, 'nargs': 0},
146 |             'unique': {'func': self.unique, 'nargs': 1},
147 |             'count': {'func': self.count, 'nargs': 1},
148 |             'exist': {'func': self.exist, 'nargs': 1},
149 |             'negate': {'func': self.negate, 'nargs': 1},
150 |             'belong_to': {'func': self.belong_to, 'nargs': 2},
151 |             'filter_color': {'func': self.filter_color, 'nargs': 2},
152 |             'filter_material': {'func': self.filter_material, 'nargs': 2},
153 |             'filter_shape': {'func': self.filter_shape, 'nargs': 2},
154 |             'filter_resting': {'func': self.filter_resting, 'nargs': 2},
155 |             'filter_moving': {'func': self.filter_moving, 'nargs': 2},
156 |             'filter_stationary': {'func': self.filter_stationary, 'nargs': 2},
157 |             'filter_start': {'func': self.filter_start, 'nargs': 1},
158 |             'filter_end': {'func': self.filter_end, 'nargs': 1},
159 |             'filter_in': {'func': self.filter_in, 'nargs': 2},
160 |             'filter_out': {'func': self.filter_out, 'nargs': 2},
161 |             'filter_collision': {'func': self.filter_collision, 'nargs': 2},
162 |             'filter_order': {'func': self.filter_order, 'nargs': 2},
163 |             'filter_before': {'func': self.filter_before, 'nargs': 2},
164 |             'filter_after': {'func': self.filter_after, 'nargs': 2},
165 |             'query_color': {'func': self.query_color, 'nargs': 1},
166 |             'query_material': {'func': self.query_material, 'nargs': 1},
167 |             'query_shape': {'func': self.query_shape, 'nargs': 1},
168 |             'query_direction': {'func': self.query_direction, 'nargs': 2},
169 |             'query_frame': {'func': self.query_frame, 'nargs': 1},
170 |             'query_object': {'func': self.query_object, 'nargs': 1},
171 |             'query_collision_partner': {'func': self.query_collision_partner, 'nargs': 2},
172 |             'filter_ancestor': {'func': self.filter_ancestor, 'nargs': 2},
173 |             'unseen_events': {'func': self.unseen_events, 'nargs': 0},
174 |             'all_events': {'func': self.all_events, 'nargs': 0},
175 |             'counterfact_events': {'func': self.counterfact_events, 'nargs': 2},
176 |             'filter_counterfact': {'func': self.filter_counterfact, 'nargs': 3},
177 |             ## for meta-clevrer
178 |             'filter_charged': {'func': self.filter_charged, 'nargs': 1},
179 |             'filter_mass': {'func': self.filter_mass, 'nargs': 1},
180 |             'is_heavier': {'func': self.is_heavier, 'nargs': 2},
181 |             'equal_charge': {'func': self.equal_charge, 'nargs': 2},
182 |             'filter_uncharged': {'func': self.filter_uncharged, 'nargs': 1},
183 |             'filter_opposite': {'func': self.filter_opposite, 'nargs': 1},
184 |             'filter_same': {'func': self.filter_charged, 'nargs': 1},
185 |             'query_both_shape': {'func': self.query_both_shape, 'nargs': 1},
186 |             'query_both_material': {'func': self.query_both_material, 'nargs': 1},
187 |             'query_both_color': {'func': self.query_both_color, 'nargs': 1},
188 |             'is_lighter': {'func': self.is_lighter, 'nargs': 2},
189 |             'not': {'func': self.negate, 'nargs': 1},
190 |             'filter_light': {'func': self.filter_light, 'nargs': 1},
191 |             'filter_heavy': {'func': self.filter_heavy, 'nargs': 1},
192 |         }
193 | 
194 |     # Module definitions
195 | 
196 | 
197 | 
198 |     ## Set / entry operators
199 |     def objects(self):
200 |         """
201 |         Return full object list
202 |         - args:
203 |         - return: objects(list)
204 |         """
205 |         return self.all_objs[:]
206 | 
207 |     def events(self):
208 |         """
209 |         Return full event list sorted in time order
210 |         - args:
211 |         - return: events(list)
212 |         """
213 |         events = self.existing_events[:]
214 |         events = sorted(events, key=lambda k: k['frame'])
215 |         return events
216 | 
217 |     def unique(self, input_list):
218 |         """
219 |         Return the only element of a list
220 |         - args: objects / events (list)
221 |         - return: object / event
222 |         """
223 |         if type(input_list) is not list:
224 |             return 'error'
225 |         if len(input_list) < 1:
226 |             return 'error'
227 |         if len(input_list) == 1:
228 |             return input_list[0]
229 |         # for query both
230 |         if len(input_list) ==2:
231 |             return input_list
232 |         if len(input_list) >=3:
233 |             return input_list[0]
234 | 
235 |     def count(self, input_list):
236 |         """
237 |         Return the number of objects / events in the input list
238 |         - args: objects / events (list)
239 |         - return: count(int)
240 |         """
241 |         if type(input_list) is not list:
242 |             return 'error'
243 |         return len(input_list)
244 | 
245 |     def exist(self, input_list):
246 |         """
247 |         Return if the input list is not empty
248 |         - args: objects / events (list)
249 |         - return: (bool)
250 |         """
251 |         if type(input_list) is not list:
252 |             return 'error'
253 |         if len(input_list) > 0:
254 |             return 'yes'
255 |         else:
256 |             return 'no'
257 | 
258 |     def negate(self, input_bool):
259 |         """
260 |         Negate the input yes / no statement
261 |         - args: input_bool(str)
262 |         - return: output_bool(str)
263 |         """
264 |         if input_bool == 'yes':
265 |             return 'no'
266 |         if input_bool == 'no':
267 |             return 'yes'
268 |         return 'error'
269 | 
270 |     def belong_to(self, input_entry, events):
271 |         """
272 |         Return if the input event / object belongs to the event list
273 |         - args: input_entry(dict / int), events(list)
274 |         - return: output_bool(str)
275 |         """
276 |         if type(events) is not list:
277 |             return 'error'
278 |         if type(input_entry) is dict:
279 |             for e in events:
280 |                 if e['type'] not in ['start', 'end']:
281 |                     if input_entry['type'] == e['type'] and \
282 |                        set(input_entry['object']) == set(e['object']):
283 |                         return 'yes'
284 |             return 'no'
285 |         elif type(input_entry) is int:
286 |             for e in events:
287 |                 if e['type'] not in ['start', 'end']:
288 |                     if input_entry in e['object']:
289 |                         return 'yes'
290 |             return 'no'
291 |         else:
292 |             return 'error'
293 | 
294 |     ## Object filters
295 |     def filter_color(self, objs, color):
296 |         """
297 |         Filter objects by color
298 |         - args: objects(list), color(str)
299 |         - return: objects(list)
300 |         """
301 |         if type(objs) is not list:
302 |             return 'error'
303 |         if len(objs) > 0 and type(objs[0]) is not int:
304 |             return 'error'
305 |         if color not in COLORS:
306 |             return 'error'
307 |         output_objs = []
308 |         for o in objs:
309 |             obj_attr = self.sim.get_static_attrs(o)
310 |             if obj_attr['color'] == color and self.sim.is_visible(o) :
311 |                 output_objs.append(o)
312 |         return output_objs
313 | 
314 |     def filter_material(self, objs, material):
315 |         """
316 |         Filter objects by material
317 |         - args: objects(list), material(str)
318 |         - return: objects(list)
319 |         """
320 |         if type(objs) is not list:
321 |             return 'error'
322 |         if len(objs) > 0 and type(objs[0]) is not int:
323 |             return 'error'
324 |         if material not in MATERIALS:
325 |             return 'error'
326 |         output_objs = []
327 |         for o in objs:
328 |             obj_attr = self.sim.get_static_attrs(o)
329 |             if obj_attr['material'] == material and self.sim.is_visible(o):
330 |                 output_objs.append(o)
331 |         return output_objs
332 | 
333 |     def filter_shape(self, objs, shape):
334 |         """
335 |         Filter objects by shape
336 |         - args: objects(list), shape(str)
337 |         - return: objects(list)
338 |         """
339 |         if type(objs) is not list:
340 |             return 'error'
341 |         if len(objs) > 0 and type(objs[0]) is not int:
342 |             return 'error'
343 |         if shape not in SHAPES:
344 |             return 'error'
345 |         output_objs = []
346 |         for o in objs:
347 |             obj_attr = self.sim.get_static_attrs(o)
348 |             if obj_attr['shape'] == shape and self.sim.is_visible(o):
349 |                 output_objs.append(o)
350 |         return output_objs
351 | 
352 |     def filter_resting(self, objs, frame):
353 |         """
354 |         Filter all resting objects in the input list
355 |         - args: objects(list), frame(int)
356 |         - return: objects(list)
357 |         """
358 |         if type(objs) is not list:
359 |             return 'error'
360 |         if type(frame) is not int and frame != 'null':
361 |             return 'error'
362 |         if frame == 'null':
363 |             frame = None
364 |         output_objs = []
365 |         for o in objs:
366 |             if self.sim.is_visible(o, frame_idx=frame) \
367 |                and not self.sim.is_moving(o, frame_idx=frame):
368 |                 output_objs.append(o)
369 |         return output_objs
370 | 
371 |     def filter_moving(self, objs, frame):
372 |         """
373 |         Filter all moving objects in the input list
374 |         - args: objects(list), frame(int)
375 |         - return: objects(list)
376 |         """
377 |         if type(objs) is not list:
378 |             return 'error'
379 |         if type(frame) is not int and frame != 'null':
380 |             return 'error'
381 |         if frame == 'null':
382 |             frame = None
383 |         output_objs = []
384 |         for o in objs:
385 |             if self.sim.is_visible(o, frame_idx=frame) and \
386 |                self.sim.is_moving(o, frame_idx=frame):
387 |                 output_objs.append(o)
388 |         return output_objs
389 | 
390 |     def filter_stationary(self, objs, frame):
391 |         """
392 |         Filter all moving objects in the input list
393 |         - args: objects(list), frame(int)
394 |         - return: objects(list)
395 |         """
396 |         if type(objs) is not list:
397 |             return 'error'
398 |         if type(frame) is not int and frame != 'null':
399 |             return 'error'
400 |         if frame == 'null':
401 |             frame = None
402 |         output_objs = []
403 |         for o in objs:
404 |             if self.sim.is_visible(o, frame_idx=frame) and \
405 |                not self.sim.is_moving(o, frame_idx=frame):
406 |                 output_objs.append(o)
407 |         return output_objs
408 | 
409 |     ## Event filters
410 |     def filter_start(self, events):
411 |         """
412 |         Find and return the start event from input list
413 |         - args: events(list)
414 |         - return: event(dict)
415 |         """
416 |         if type(events) is not list:
417 |             return 'error'
418 |         if len(events) > 0 and type(events[0]) is not dict:
419 |             return 'error'
420 |         for e in events:
421 |             if e['type'] == 'start':
422 |                 return e
423 |         return 'error'
424 | 
425 |     def filter_end(self, events):
426 |         """
427 |         Find and return the end event from input list
428 |         - args: events(list)
429 |         - return: event(dict)
430 |         """
431 |         if type(events) is not list:
432 |             return 'error'
433 |         if len(events) > 0 and type(events[0]) is not dict:
434 |             return 'error'
435 |         for e in events:
436 |             if e['type'] == 'end':
437 |                 return e
438 |         return error
439 | 
440 |     def filter_in(self, events, objs):
441 |         """
442 |         Return all incoming events that involve any of the objects
443 |         args: events(list), objects(list / int)
444 |         return: events(list)
445 |         """
446 |         if type(events) is not list:
447 |             return 'error'
448 |         if len(events) > 0 and type(events[0]) is not dict:
449 |             return 'error'
450 |         if type(objs) is not list:
451 |             objs = [objs]
452 |         if len(objs) > 0 and type(objs[0]) is not int:
453 |             return 'error'
454 |         output_events = []
455 |         for e in events:
456 |             if e['type'] == 'in':
457 |                 if e['object'][0] in objs:
458 |                     output_events.append(e)
459 |         return output_events
460 | 
461 |     def filter_out(self, events, objs):
462 |         """
463 |         Return all outgoing events that involve any of the objects
464 |         args: events(list), objects(list / int)
465 |         return: events(list)
466 |         """
467 |         if type(events) is not list:
468 |             return 'error'
469 |         if len(events) > 0 and type(events[0]) is not dict:
470 |             return 'error'
471 |         if type(objs) is not list:
472 |             objs = [objs]
473 |         if len(objs) > 0 and type(objs[0]) is not int:
474 |             return 'error'
475 |         output_events = []
476 |         for e in events:
477 |             if e['type'] == 'out':
478 |                 if e['object'][0] in objs:
479 |                     output_events.append(e)
480 |         return output_events
481 | 
482 |     def filter_collision(self, events, objs):
483 |         """
484 |         Return all collision events that involve any of the objects
485 |         args: events(list), objects(list / int)
486 |         return: events(list)
487 |         """
488 |         if type(events) is not list:
489 |             return 'error'
490 |         if len(events) > 0 and type(events[0]) is not dict:
491 |             return 'error'
492 |         if type(objs) is not list:
493 |             objs = [objs]
494 |         if len(objs) > 0 and type(objs[0]) is not int:
495 |             return 'error'
496 |         output_events = []
497 |         for e in events:
498 |             if e['type'] == 'collision':
499 |                 if e['object'][0] in objs or e['object'][1] in objs:
500 |                     output_events.append(e)
501 |         return output_events
502 | 
503 |     def filter_order(self, events, order):
504 |         """
505 |         Return the event at given order
506 |         - args: events(list), order(str)
507 |         - return: event(dict)
508 |         """
509 |         if type(events) is not list:
510 |             return 'error'
511 |         if len(events) > 0 and type(events[0]) is not dict:
512 |             return 'error'
513 |         if order not in ['first', 'second', 'last']:
514 |             return 'error'
515 |         idx_dict = {
516 |             'first': 0,
517 |             'second': 1,
518 |             'last': -1
519 |         }
520 |         idx = idx_dict[order]
521 |         if idx >= len(events) or len(events) == 0:
522 |             return 'error'
523 |         return events[idx]
524 | 
525 |     def filter_before(self, events, event):
526 |         """
527 |         Return all events before the designated event
528 |         - args: events(list), event(dict)
529 |         - return: events(list)
530 |         """
531 |         if type(events) is not list:
532 |             return 'error'
533 |         if len(events) > 0 and type(events[0]) is not dict:
534 |             return 'error'
535 |         if 'type' not in event:
536 |             return 'error'
537 |         output_events = []
538 |         for e in events:
539 |             if e['frame'] < event['frame']:
540 |                 output_events.append(e)
541 |         return output_events
542 | 
543 |     def filter_after(self, events, event):
544 |         """
545 |         Return all events after the designated event
546 |         - args: events(list), event(dict)
547 |         - return: events(list)
548 |         """
549 |         if type(events) is not list:
550 |             return 'error'
551 |         if len(events) > 0 and type(events[0]) is not dict:
552 |             return 'error'
553 |         if type(event) is not dict or 'type' not in event:
554 |             return 'error'
555 |         output_events = []
556 |         for e in events:
557 |             if e['frame'] > event['frame']:
558 |                 output_events.append(e)
559 |         return output_events
560 | 
561 |     ## query objects
562 |     def query_color(self, obj):
563 |         """
564 |         Return the color of the queried object
565 |         - args: obj(int)
566 |         - return: color(str)
567 |         """
568 |         if type(obj) is not int:
569 |             return 'error'
570 |         obj_attr = self.sim.get_static_attrs(obj)
571 |         return obj_attr['color']
572 | 
573 |     def query_material(self, obj):
574 |         """
575 |         Return the material of the queried object
576 |         - args: obj(int)
577 |         - return material(str)
578 |         """
579 |         if type(obj) is not int:
580 |             return 'error'
581 |         obj_attr = self.sim.get_static_attrs(obj)
582 |         return obj_attr['material']
583 | 
584 |     def query_shape(self, obj):
585 |         """
586 |         Return the shape of the queried object
587 |         - args: obj(int)
588 |         - return: shape(str)
589 |         """
590 |         if type(obj) is not int:
591 |             return 'error'
592 |         obj_attr = self.sim.get_static_attrs(obj)
593 |         return obj_attr['shape']
594 | 
595 |     def query_direction(self, obj, frame):
596 |         """
597 |         Return the direction of the queried object
598 |         - args: obj(int), frame(int)
599 |         - return: direction(str)
600 |         """
601 |         if frame=='null':
602 |             frame = None
603 |         if type(obj) is not int or (type(frame) is not int and frame is not  None):
604 |             return 'error'
605 |         if not self.sim.is_visible(obj, frame):
606 |             return 'error'
607 |         if self.sim.is_moving_left(obj, frame, angle_half_range=DIR_ANGLE_TH):
608 |             return 'left'
609 |         if self.sim.is_moving_right(obj, frame, angle_half_range=DIR_ANGLE_TH):
610 |             return 'right'
611 |         if self.sim.is_moving_up(obj, frame, angle_half_range=DIR_ANGLE_TH):
612 |             return 'up'
613 |         if self.sim.is_moving_down(obj, frame, angle_half_range=DIR_ANGLE_TH):
614 |             return 'down'
615 |         return 'error'
616 | 
617 |     ## query events
618 |     def query_frame(self, event):
619 |         """
620 |         Return the frame number of the queried event
621 |         - args: event(dict)
622 |         - return: frame(int)
623 |         """
624 |         if type(event) is not dict:
625 |             return 'error'
626 |         return event['frame']
627 | 
628 |     def query_object(self, in_out_event):
629 |         """
630 |         Return the object index that involves in the event
631 |         - args: event(dict)
632 |         - return: obj(int)
633 |         """
634 |         if type(in_out_event) is not dict:
635 |             return 'error'
636 |         if in_out_event['type'] not in ['in', 'out']:
637 |             return 'error'
638 |         return in_out_event['object'][0]
639 | 
640 |     def query_collision_partner(self, col_event, obj):
641 |         """
642 |         Return the collision partner of the input object
643 |         - args: col_event(dict), obj(int)
644 |         - return: obj(int)
645 |         """
646 |         if type(col_event) is not dict or type(obj) is not int:
647 |             return 'error'
648 |         if col_event['type'] != 'collision':
649 |             return 'error'
650 |         if obj not in col_event['object']:
651 |             return 'error'
652 |         col_objs = col_event['object']
653 |         if obj == col_objs[0]:
654 |             return col_objs[1]
655 |         else:
656 |             return col_objs[0]
657 | 
658 |     ## Explanatory
659 |     def _search_causes(self, target_event, causes):
660 |         """Recursively search for all ancestor events of the target event in the causal graph"""
661 |         next_step_events = []
662 |         for tr in self.causal_traces:
663 |             if target_event in tr:
664 |                 idx = tr.index(target_event)
665 |                 if idx > 0 and tr[idx-1] not in next_step_events:
666 |                     next_step_events.append(tr[idx-1])
667 |                     if tr[idx-1] not in causes:
668 |                         causes.append(tr[idx-1])
669 |         for e in next_step_events:
670 |             self._search_causes(e, causes)
671 | 
672 |     def filter_ancestor(self, events, event):
673 |         """
674 |         Filter all ancestors of the input event in the causal graph
675 |         - args: events(list), event(dict)
676 |         - return: events(list)
677 |         """
678 |         if type(events) is not list:
679 |             return 'error'
680 |         if len(events) > 0 and type(events[0]) is not dict:
681 |             return 'error'
682 |         if type(event) is not dict or 'type' not in event:
683 |             return 'error'
684 |         all_causes = []
685 |         self._search_causes(event, all_causes)
686 |         output = []
687 |         for e in events:
688 |             if e in all_causes:
689 |                 output.append(e)
690 |         return output
691 | 
692 |     ## Predictive
693 |     def unseen_events(self):
694 |         """
695 |         Return a complete list of all unseen events
696 |         - args: 
697 |         - return: events(list)
698 |         """
699 |         return self.unseens
700 | 
701 |     ## Counterfactual
702 |     def all_events(self):
703 |         """
704 |         Return all possible events
705 |         - args:
706 |         - return: events(list)
707 |         """
708 |         possible_events = []
709 |         for i in range(len(self.all_objs)):
710 |             for j in range(i+1, len(self.all_objs)):
711 |                 event = {
712 |                     'type': 'collision',
713 |                     'object': [self.all_objs[i], self.all_objs[j]],
714 |                 }
715 |                 possible_events.append(event)
716 |         return possible_events
717 | 
718 |     def counterfact_events(self, remove_obj, counterfact_property):
719 |         """
720 |         Return all events after removing the input object
721 |         - args: remove_obj(int)
722 |         - return: counterfact_events(list)
723 |         """
724 |         if type(remove_obj) is not int:
725 |             return 'error'
726 |         if remove_obj not in self.all_objs:
727 |             return 'error'
728 |         if not isinstance(counterfact_property, str ):
729 |             return 'error'
730 |         if 'lighter' in counterfact_property:
731 |             cf_id = str(remove_obj) +'_mass_1'
732 |         elif 'heavier' in counterfact_property: 
733 |             cf_id = str(remove_obj) +'_mass_5'
734 |         elif 'uncharge' in counterfact_property: 
735 |             cf_id = str(remove_obj) +'_charge_0'
736 |         elif 'opposite' in counterfact_property:
737 |             obj_info = self.sim.search_obj_info_by_id(remove_obj)
738 |             opp_charge = obj_info['charge'] * -1 
739 |             cf_id = str(remove_obj) +'_charge_' + str(opp_charge)
740 |         return self.sim.cf_events[cf_id]
741 | 
742 |     def filter_counterfact(self, events, remove_obj, counterfact_property):
743 |         """
744 |         Return all events from the input list that will happen when the object is dropped
745 |         - args: events(list), remove_obj(int)
746 |         - return: events(list)
747 |         """
748 |         if type(events) is not list:
749 |             return 'error'
750 |         if len(events) > 0 and type(events[0]) is not dict:
751 |             return 'error'
752 |         cf_events = self.counterfact_events(remove_obj, counterfact_property)
753 |         if cf_events == 'error':
754 |             return 'error'
755 |         cf_col_pairs = [set(e['object']) for e in cf_events if e['type'] == 'collision']
756 |         outputs = []
757 |         for e in events:
758 |             if set(e['object']) in cf_col_pairs:
759 |                 outputs.append(e)
760 |         return outputs
761 | 
762 |     def filter_charged(self, objs):
763 |         """
764 |         Filter all charged objects in the input list
765 |         - args: objects(list)
766 |         - return: objects(list)
767 |         """
768 |         if type(objs) is not list:
769 |             return 'error'
770 |         output_objs = []
771 |         for o in objs:
772 |             if self.sim.is_visible(o) and self.sim.is_charged(o):
773 |                 output_objs.append(o)
774 |         return output_objs
775 |     
776 |     def filter_mass(self, objs):
777 |         """
778 |         Filter all charged objects in the input list
779 |         - args: objects(list)
780 |         - return: masses(list)
781 |         """
782 |         if type(objs) is not list:
783 |             return 'error'
784 |         output_masses = []
785 |         for o in objs:
786 |             mass = self.sim.objs[o]['mass']
787 |             output_masses.append(mass)
788 |         return output_masses
789 | 
790 |     def is_heavier(self, mass1, mass2):
791 |         #pdb.set_trace()
792 |         if isinstance(mass1, list)  and len(mass1)==2:
793 |             if mass1[0]> mass1[1]:
794 |                 return 'yes'
795 |             else:
796 |                 return 'no'
797 |         elif isinstance(mass1, int) and isinstance(mass2, int):
798 |             if mass1 > mass2:
799 |                 return 'yes'
800 |             else:
801 |                 return 'no'
802 |         else:
803 |             return 'error'
804 | 
805 |     def equal_charge(self, obj1, obj2):
806 |         charge1 = self.sim.objs[obj1]['charge']
807 |         charge2 = self.sim.objs[obj2]['charge']
808 |         if charge1 !=0 and charge1==charge2:
809 |             return 'yes'
810 |         else:
811 |             return 'no'
812 |     
813 |     def filter_uncharged(self, objs):
814 |         """
815 |         Filter all charged objects in the input list
816 |         - args: objects(list)
817 |         - return: objects(list)
818 |         """
819 |         if type(objs) is not list:
820 |             return 'error'
821 |         output_objs = []
822 |         for o in objs:
823 |             if self.sim.is_visible(o) and (not self.sim.is_charged(o)):
824 |                 output_objs.append(o)
825 |         return output_objs
826 |     
827 |     def filter_opposite(self, objs):
828 |         """
829 |         Filter opposite charged objects in the input list
830 |         - args: objects(list)
831 |         - return: objects(list)
832 |         """
833 |         if type(objs) is not list:
834 |             return 'error'
835 |         charge_objs = self.filter_charged(objs)
836 |         if len(charge_objs)!=2:
837 |             return 'error'
838 |         elif self.sim.objs[charge_objs[0]]['charge'] * self.sim.objs[charge_objs[1]]['charge'] <0:
839 |             return  charge_objs
840 |         else:
841 |             return 'error'
842 | 
843 |     def filter_same(self, objs):
844 |         """
845 |         Filter the same charged objects in the input list
846 |         - args: objects(list)
847 |         - return: objects(list)
848 |         """
849 |         if type(objs) is not list:
850 |             return 'error'
851 |         charge_objs = self.filter_charged(objs)
852 |         if len(charge_objs)!=2:
853 |             return 'error'
854 |         elif self.sim.objs[charge_objs[0]]['charge'] * self.sim.objs[charge_objs[1]]['charge'] >0:
855 |             return  charge_objs
856 |         else:
857 |             return 'error'
858 | 
859 |     def query_both_shape(self, obj1, obj2):
860 |         """
861 |         Return the material of the queried object
862 |         - args: obj1(int), obj2(int)
863 |         - return shape1 and shape2 (str)
864 |         """
865 |         if type(obj1) is not int or type(obj2) is not int:
866 |             return 'error'
867 |         shape1 = self.query_shape(obj1)
868 |         shape2 = slef.query_shape(obj2)
869 |         return shape1 + ' and ' + shape2 
870 |     
871 |     def query_both_color(self, obj_list):
872 |         """
873 |         Return the colors of the queried objects 
874 |         - args: obj_list (list)
875 |         - return color1 and color2 (str)
876 |         """
877 |         if type(obj_list) is not list or len(obj_list)<2:
878 |             return 'error'
879 | 
880 |         obj1, obj2 = obj_list[0], obj_list[1]
881 |         if type(obj1) is not int or type(obj2) is not int:
882 |             return 'error'
883 |         color1 = self.query_color(obj1)
884 |         color2 = self.query_color(obj2)
885 |         return color1 + ' and ' + color2
886 | 
887 |     def query_both_material(self, obj_list):
888 |         """
889 |         Return the materials of the queried objects  
890 |         - args: obj_list (list)
891 |         - return material1 and material2 (str)
892 |         """
893 |         if type(obj_list) is not list or len(obj_list)<2:
894 |             return 'error'
895 |         obj1, obj2 = obj_list
896 |         if type(obj1) is not int or type(obj2) is not int:
897 |             return 'error'
898 |         material1 = self.query_material(obj1)
899 |         material2 = self.query_material(obj2)
900 |         return material1 + ' and ' + material2 
901 | 
902 |     def query_both_shape(self, obj_list):
903 |         """
904 |         Return the material of the queried object
905 |         - args: obj_list (list)
906 |         - return material1 and material2 (str)
907 |         """
908 |         if type(obj_list) is not list or len(obj_list)<2:
909 |             return 'error'
910 |         #if len(obj_list)!=2:
911 |         # Todo: select the most confident objects
912 |         obj1, obj2 = obj_list[0], obj_list[1]
913 |         if type(obj1) is not int or type(obj2) is not int:
914 |             return 'error'
915 |         shape1 = self.query_shape(obj1)
916 |         shape2 = self.query_shape(obj2)
917 |         return shape1 + ' and ' + shape2 
918 |     
919 |     def is_lighter(self, mass1, mass2):
920 |         if not isinstance(mass1, (int, float)) or (not isinstance(mass2, (int, float))):
921 |             return 'error'
922 |         if mass1 < mass2:
923 |             return 'yes'
924 |         else:
925 |             return 'no'
926 |     
927 |     def filter_light(self, objs):
928 |         """
929 |         Filter all light objects in the input list
930 |         - args: objects(list)
931 |         - return: objects(list)
932 |         """
933 |         if type(objs) is not list:
934 |             return 'error'
935 |         output_objs = []
936 |         for o in objs:
937 |             if self.sim.is_visible(o) and self.sim.is_light(o):
938 |                 output_objs.append(o)
939 |         return output_objs
940 | 
941 |     def filter_heavy(self, objs):
942 |         """
943 |         Filter all heavy objects in the input list
944 |         - args: objects(list)
945 |         - return: objects(list)
946 |         """
947 |         if type(objs) is not list:
948 |             return 'error'
949 |         output_objs = []
950 |         for o in objs:
951 |             if self.sim.is_visible(o) and (not self.sim.is_light(o)):
952 |                 output_objs.append(o)
953 |         return output_objs
954 | 


--------------------------------------------------------------------------------
/generate_invalid_video_list.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Run symbolic reasoning on open-ended questions
 3 | """
 4 | import os
 5 | import json
 6 | from tqdm import tqdm
 7 | import argparse
 8 | 
 9 | from executor import Executor
10 | from simulation import Simulation
11 | import pdb
12 | from utils.utils import print_monitor
13 | import numpy as np
14 | import sys
15 | 
16 | parser = argparse.ArgumentParser()
17 | parser.add_argument('--use_event_ann', default=1, type=int)
18 | parser.add_argument('--use_in', default=0, type=int)  # Interaction network
19 | parser.add_argument('--program_path', default='/home/tmp_user/code/clevrer_dataset_generation_v2/clevrer_question_generation/output/questions_v13_2/open_end_questions.json')
20 | parser.add_argument('--question_path', default='/home/tmp_user/code/clevrer_dataset_generation_v2/clevrer_question_generation/output/questions_v13_2/open_end_questions.json')
21 | parser.add_argument('--gt_flag', default=0, type=int)
22 | parser.add_argument('--ann_dir', default='/home/tmp_user/code/output/render_output/causal_v13')
23 | parser.add_argument('--track_dir', default='/home/tmp_user/code/output/render_output/causal_v13_coco_ann')
24 | parser.add_argument('--frame_diff', default=5, type=int)
25 | parser.add_argument('--mc_flag', default=0, type=int)
26 | parser.add_argument('--raw_motion_prediction_dir', default='')
27 | parser.add_argument('--invalid_video_fn', default = 'invalid_video_v14.txt')
28 | parser.add_argument('--start_id', default=0, type=int)
29 | parser.add_argument('--num_sim', default=5000, type=int)
30 | args = parser.parse_args()
31 | 
32 | question_path = args.question_path
33 | program_path = args.program_path 
34 | 
35 | with open(program_path) as f:
36 |     parsed_pgs = json.load(f)
37 | with open(question_path) as f:
38 |     anns = json.load(f)
39 | 
40 | total, correct = 0, 0
41 | 
42 | pbar = tqdm(range(args.num_sim))
43 | 
44 | acc_monitor = {}
45 | ans_swap = ''
46 | 
47 | if os.path.isfile(args.invalid_video_fn):
48 |     print("%s exists."%invalid_list)
49 |     sys.exit()
50 | else:
51 |     fh = open(args.invalid_video_fn, 'w')
52 | invalid_num = 0
53 | for ann_idx in pbar:
54 |     file_idx = ann_idx  + args.start_id
55 |     question_scene = anns[file_idx]
56 |     sim = Simulation(args, file_idx, use_event_ann=(args.use_event_ann != 0))
57 |     if len(sim.objs)!=len(sim.get_visible_objs()):
58 |         fh.write('%d\n'%file_idx)
59 |         fh.flush()
60 |         invalid_num +=1
61 | print('Invalid file num: %d\n'%invalid_num)
62 | fh.close()
63 | 


--------------------------------------------------------------------------------
/get_results.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Run symbolic reasoning on parsed programs and motion predictions
 3 | Output an answer prediction file for test evaluation
 4 | """
 5 | 
 6 | import os
 7 | import json
 8 | from tqdm import tqdm
 9 | import argparse
10 | 
11 | from executor import Executor
12 | from simulation import Simulation
13 | 
14 | 
15 | parser = argparse.ArgumentParser()
16 | parser.add_argument('--no_event', action='store_true', default=False)
17 | args = parser.parse_args()
18 | 
19 | 
20 | if not args.no_event:
21 |     raw_motion_dir = 'data/propnet_preds/with_edge_supervision'
22 | else:
23 |     raw_motion_dir = 'data/propnet_preds/without_edge_supervision'
24 |         
25 | question_path = 'data/test.json'
26 | mc_program_path = 'data/parsed_programs/mc_parsed_prog_1000_test.json'
27 | oe_program_path = 'data/parsed_programs/oe_parsed_prog_1000_test.json'
28 | 
29 | 
30 | with open(mc_program_path) as f:
31 |     mc_programs = json.load(f)
32 | with open(oe_program_path) as f:
33 |     oe_programs = json.load(f)
34 | with open(question_path) as f:
35 |     questions = json.load(f)
36 | 
37 | pred_map = {'yes': 'correct', 'no': 'wrong', 'error': 'error'}
38 | all_pred = []
39 | for i in tqdm(range(5000)):
40 |     ann_path = os.path.join(raw_motion_dir, 'sim_%05d.json' % (i + 15000))
41 |     sim = Simulation(ann_path, use_event_ann=True)
42 |     exe = Executor(sim)
43 |     
44 |     scene = {
45 |         'scene_index': questions[i]['scene_index'],
46 |         'questions': [],
47 |     }
48 |     for j, q in enumerate(questions[i]['questions']):
49 |         if q['question_type'] == 'descriptive':
50 |             assert oe_programs[i]['questions'][j]['question_id'] == q['question_id']
51 |             pg = oe_programs[i]['questions'][j]['program']
52 |             ans = exe.run(pg, debug=False)
53 |             ann = {
54 |                 'question_id': q['question_id'],
55 |                 'answer': ans,
56 |             }
57 |         else:
58 |             n_oe = mc_programs[i]['questions'][0]['question_id']
59 |             assert mc_programs[i]['questions'][j - n_oe]['question_id'] == q['question_id']
60 |             q_pg = mc_programs[i]['questions'][j - n_oe]['program']
61 |             ann = {
62 |                 'question_id': q['question_id'],
63 |                 'choices': [],
64 |             }
65 |             for k, c in enumerate(mc_programs[i]['questions'][j - n_oe]['choices']):
66 |                 c_pg = c['program']
67 |                 ans = exe.run(c_pg + q_pg, debug=False)
68 |                 ans = pred_map[ans]
69 |                 
70 |                 cann = {
71 |                     'choice_id': c['choice_id'],
72 |                     'answer': ans,
73 |                 }
74 |                 ann['choices'].append(cann)
75 |         scene['questions'].append(ann)
76 |     all_pred.append(scene)
77 |     
78 | with open('nsdr_pred.json', 'w') as fout:
79 |     json.dump(all_pred, fout)
80 | 


--------------------------------------------------------------------------------
/merge_raw_results.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import pdb
 3 | 
 4 | def merge_result():
 5 |     oe_result_path = './test_oe.json' 
 6 |     mc_result_path = './test_mc.json' 
 7 |     submit_result_path = './test_submit.json' 
 8 |     fh_oe = open(oe_result_path, 'rb')
 9 |     fh_mc = open(mc_result_path, 'rb')
10 |     fd_oe = json.load(fh_oe)
11 |     fd_mc = json.load(fh_mc)
12 |     assert len(fd_oe) == len(fd_mc)
13 |     start_id = 10000
14 |     vid_num = 2000
15 |     submit_list = []
16 |     for vid in range(start_id, start_id + vid_num):
17 |         tmp_dict = {'scene_index': vid, 'video_filename': 'sim_%05d.mp4'%(vid)}
18 |         vid_str = str(vid)
19 |         oe_dict = fd_oe[vid_str]
20 |         mc_dict = fd_mc[vid_str]
21 |         tmp_ques_list = []
22 |         oe_ques_key_list = sorted(list(oe_dict.keys()))
23 |         mc_ques_key_list = sorted(list(mc_dict.keys()))
24 |         for q_id in oe_ques_key_list:
25 |             tmp_q_pred = {'question_id': int(q_id), 'question_type': 'Factual', 'answer': oe_dict[q_id]}
26 |             tmp_ques_list.append(tmp_q_pred)
27 |         for q_id in mc_ques_key_list:
28 |             tmp_q_pred = {'question_id': int(q_id), 'question_type': mc_dict[q_id]['question_type']}
29 |             choice_list = []
30 |             for c_id, c_pred in enumerate(mc_dict[q_id]['choices']):
31 |                 tmp_choice = {'answer': c_pred, 'choice_id': c_id }
32 |                 choice_list.append(tmp_choice)
33 |             tmp_q_pred['choices'] = choice_list
34 |             tmp_ques_list.append(tmp_q_pred)
35 |         tmp_dict['questions'] = tmp_ques_list
36 |         submit_list.append(tmp_dict)
37 |     with open(submit_result_path, 'w') as fh_out:
38 |         json.dump(submit_list, fh_out)
39 |         #pdb.set_trace()
40 | 
41 | if __name__=='__main__':
42 |     merge_result()
43 | 


--------------------------------------------------------------------------------
/requirement.txt:
--------------------------------------------------------------------------------
 1 | h5py==2.10.0
 2 | matplotlib==3.3.0
 3 | numpy==1.19.1
 4 | Pillow==7.2.0
 5 | pycocotools==2.0.1
 6 | scikit_learn==0.23.1
 7 | scipy==1.5.2
 8 | scikit-image
 9 | torch==1.0.0
10 | torchvision==0.2.0
11 | opencv-python
12 | tqdm
13 | 


--------------------------------------------------------------------------------
/run_mc.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Run symbolic reasoning on multiple-choice questions
  3 | """
  4 | import os
  5 | import json
  6 | from tqdm import tqdm
  7 | import argparse
  8 | 
  9 | from executor import Executor
 10 | from simulation import Simulation
 11 | 
 12 | 
 13 | parser = argparse.ArgumentParser()
 14 | parser.add_argument('--n_progs', required=True)
 15 | parser.add_argument('--use_event_ann', default=1, type=int)
 16 | parser.add_argument('--use_in', default=0, type=int)  # Use interaction network
 17 | args = parser.parse_args()
 18 | 
 19 | 
 20 | if args.use_event_ann != 0:
 21 |     
 22 |     raw_motion_dir = 'data/propnet_preds/with_edge_supervision'
 23 | else:
 24 |     raw_motion_dir = 'data/propnet_preds/without_edge_supervision'
 25 | if args.use_in != 0:
 26 |     raw_motion_dir = 'data/propnet_preds/interaction_network'
 27 | 
 28 | 
 29 | question_path = 'data/validation.json'
 30 | if args.n_progs == 'all':
 31 |     program_path = 'data/parsed_programs/mc_allq_allc.json'
 32 | else:
 33 |     program_path = 'data/parsed_programs/mc_{}q_{}c_val_new.json'.format(args.n_progs, int(args.n_progs)*4)
 34 | 
 35 | 
 36 | print(raw_motion_dir)
 37 | print(program_path)
 38 | 
 39 | with open(program_path) as f:
 40 |     parsed_pgs = json.load(f)
 41 | with open(question_path) as f:
 42 |     anns = json.load(f)
 43 | 
 44 | total, correct = 0, 0
 45 | total_per_q, correct_per_q = 0, 0
 46 | total_expl, correct_expl = 0, 0
 47 | total_expl_per_q, correct_expl_per_q = 0, 0
 48 | total_pred, correct_pred = 0, 0
 49 | total_pred_per_q, correct_pred_per_q = 0, 0
 50 | total_coun, correct_coun = 0, 0
 51 | total_coun_per_q, correct_coun_per_q = 0, 0
 52 | 
 53 | pred_map = {'yes': 'correct', 'no': 'wrong', 'error': 'error'}
 54 | pbar = tqdm(range(5000))
 55 | for ann_idx in pbar:
 56 |     question_scene = anns[ann_idx]
 57 |     file_idx = ann_idx + 10000 
 58 |     ann_path = os.path.join(raw_motion_dir, 'sim_%05d.json' % file_idx)
 59 | 
 60 |     sim = Simulation(ann_path, use_event_ann=(args.use_event_ann != 0))
 61 |     exe = Executor(sim)
 62 |     valid_q_idx = 0
 63 |     for q_idx, q in enumerate(question_scene['questions']):
 64 |         question = q['question']
 65 |         q_type = q['question_type']
 66 |         if q_type == 'descriptive': # skip open-ended questions
 67 |             continue
 68 | 
 69 |         q_ann = parsed_pgs[str(file_idx)]['questions'][valid_q_idx]
 70 |         correct_question = True
 71 |         for c in q_ann['choices']:
 72 |             full_pg = c['program'] + q_ann['question_program']
 73 |             ans = c['answer']
 74 |             pred = exe.run(full_pg, debug=False)
 75 |             pred = pred_map[pred]
 76 |             # print(ans, pred)
 77 |             if ans == pred:
 78 |                 correct += 1
 79 |             else:
 80 |                 correct_question = False
 81 |             total += 1
 82 | 
 83 |             if q['question_type'].startswith('explanatory'):
 84 |                 if ans == pred:
 85 |                     correct_expl += 1
 86 |                 total_expl += 1
 87 | 
 88 |             if q['question_type'].startswith('predictive'):
 89 |                 # print(pred, ans)
 90 |                 if ans == pred:
 91 |                     correct_pred += 1
 92 |                 total_pred += 1
 93 | 
 94 |             if q['question_type'].startswith('counterfactual'):
 95 |                 if ans == pred:
 96 |                     correct_coun += 1
 97 |                 total_coun += 1
 98 | 
 99 |         if correct_question:
100 |             correct_per_q += 1
101 |         total_per_q += 1
102 | 
103 |         if q['question_type'].startswith('explanatory'):
104 |             if correct_question:
105 |                 correct_expl_per_q += 1
106 |             total_expl_per_q += 1
107 | 
108 |         if q['question_type'].startswith('predictive'):
109 |             if correct_question:
110 |                 correct_pred_per_q += 1
111 |             total_pred_per_q += 1
112 | 
113 |         if q['question_type'].startswith('counterfactual'):
114 |             if correct_question:
115 |                 correct_coun_per_q += 1
116 |             total_coun_per_q += 1
117 |         valid_q_idx += 1
118 |     # print('up to scene %d: %d / %d correct options, accuracy %f %%'
119 |     #       % (ann_idx, correct, total, (float(correct)*100/total)))
120 |     # print('up to scene %d: %d / %d correct questions, accuracy %f %%'
121 |     #       % (ann_idx, correct_per_q, total_per_q, (float(correct_per_q)*100/total_per_q)))
122 |     # print()
123 |     pbar.set_description('per choice {:f}, per questions {:f}'.format(float(correct)*100/total, float(correct_per_q)*100/total_per_q))
124 | 
125 | print('============ results ============')
126 | print('overall accuracy per option: %f %%' % (float(correct) * 100.0 / total))
127 | print('overall accuracy per question: %f %%' % (float(correct_per_q) * 100.0 / total_per_q))
128 | print('explanatory accuracy per option: %f %%' % (float(correct_expl) * 100.0 / total_expl))
129 | print('explanatory accuracy per question: %f %%' % (float(correct_expl_per_q) * 100.0 / total_expl_per_q))
130 | print('predictive accuracy per option: %f %%' % (float(correct_pred) * 100.0 / total_pred))
131 | print('predictive accuracy per question: %f %%' % (float(correct_pred_per_q) * 100.0 / total_pred_per_q))
132 | print('counterfactual accuracy per option: %f %%' % (float(correct_coun) * 100.0 / total_coun))
133 | print('counterfactual accuracy per question: %f %%' % (float(correct_coun_per_q) * 100.0 / total_coun_per_q))
134 | print('============ results ============')
135 | 
136 | output_ann = {
137 |     'total_options': total,
138 |     'correct_options': correct,
139 |     'total_questions': total_per_q,
140 |     'correct_questions': correct_per_q,
141 |     'total_explanatory_options': total_expl,
142 |     'correct_explanatory_options': correct_expl,
143 |     'total_explanatory_questions': total_expl_per_q,
144 |     'correct_explanatory_questions': correct_expl_per_q,
145 |     'total_predictive_options': total_pred,
146 |     'correct_predictive_options': correct_pred,
147 |     'total_predictive_questions': total_pred_per_q,
148 |     'correct_predictive_questions': correct_pred_per_q,
149 |     'total_counterfactual_options': total_coun,
150 |     'correct_counterfactual_options': correct_coun,
151 |     'total_counterfactual_questions': total_coun_per_q,
152 |     'correct_counterfactual_questions': correct_coun_per_q,
153 | }
154 | 
155 | output_file = 'result_mc.json'
156 | if args.use_in != 0:
157 |     output_file = 'result_mc_in.json'
158 | with open(output_file, 'w') as fout:
159 |     json.dump(output_ann, fout)
160 | 


--------------------------------------------------------------------------------
/run_mc_clevrer.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Run symbolic reasoning on multiple-choice questions
  3 | """
  4 | import os
  5 | import json
  6 | from tqdm import tqdm
  7 | import argparse
  8 | 
  9 | from executor import Executor
 10 | from simulation import Simulation
 11 | import pdb
 12 | import numpy as np
 13 | EPS = 0.000001
 14 | 
 15 | parser = argparse.ArgumentParser()
 16 | parser.add_argument('--use_event_ann', default=1, type=int)
 17 | parser.add_argument('--use_in', default=0, type=int)  # Use interaction network
 18 | parser.add_argument('--program_path', default='/home/zfchen/code/clevrer_dataset_generation_v2/clevrer_question_generation/output/questions_v13_2/open_end_questions.json')
 19 | parser.add_argument('--question_path', default='/home/zfchen/code/clevrer_dataset_generation_v2/clevrer_question_generation/output/questions_v13_2/open_end_questions.json')
 20 | parser.add_argument('--gt_flag', default=0, type=int)
 21 | parser.add_argument('--ann_dir', default='/home/zfchen/code/output/render_output/causal_v13')
 22 | parser.add_argument('--track_dir', default='/home/zfchen/code/output/render_output/causal_v13_coco_ann')
 23 | parser.add_argument('--raw_motion_prediction_dir', default='')
 24 | parser.add_argument('--frame_diff', default=5, type=int)
 25 | parser.add_argument('--mc_flag', default=1, type=int)
 26 | parser.add_argument('--start_id', default=0, type=int)
 27 | parser.add_argument('--num_sim', default=5000, type=int)
 28 | parser.add_argument('--file_idx_offset', default=10000, type=int)
 29 | parser.add_argument('--result_out_fn', default = 'result_split1.json')
 30 | args = parser.parse_args()
 31 | 
 32 | question_path = args.question_path
 33 | program_path = args.program_path 
 34 | 
 35 | print(question_path)
 36 | print(program_path)
 37 | 
 38 | with open(program_path) as f:
 39 |     parsed_pgs = json.load(f)
 40 | with open(question_path) as f:
 41 |     anns = json.load(f)
 42 | 
 43 | total, correct = 0, 0
 44 | total_per_q, correct_per_q = 0, 0
 45 | total_expl, correct_expl = 0, 0
 46 | total_expl_per_q, correct_expl_per_q = 0, 0
 47 | total_pred, correct_pred = 0, 0
 48 | total_pred_per_q, correct_pred_per_q = 0, 0
 49 | total_coun, correct_coun = 0, 0
 50 | total_coun_per_q, correct_coun_per_q = 0, 0
 51 | 
 52 | total_coun_mass, correct_coun_mass = 0, 0
 53 | total_coun_per_q_mass, correct_coun_per_q_mass = 0, 0
 54 | total_coun_charge, correct_coun_charge = 0, 0
 55 | total_coun_per_q_charge, correct_coun_per_q_charge = 0, 0
 56 | 
 57 | pred_map = {'yes': 'correct', 'no': 'wrong', 'error': 'error'}
 58 | pbar = tqdm(range(args.num_sim))
 59 | prediction_dict = {}
 60 | 
 61 | for ann_idx in pbar:
 62 |     file_idx = ann_idx + args.start_id 
 63 |     question_scene = anns[file_idx-args.file_idx_offset]
 64 |     
 65 |     sim = Simulation(args, file_idx, n_vis_frames=120, use_event_ann=(args.use_event_ann != 0))
 66 |     tmp_pred = {}
 67 |     exe = Executor(sim)
 68 |     valid_q_idx = 0
 69 |     for q_idx, q in enumerate(question_scene['questions']):
 70 |         #print('%d\n'%q_idx)
 71 |         question = q['question']
 72 |         if 'question_type' not in q:
 73 |             continue
 74 |         q_type = q['question_type']
 75 |         q_ann = parsed_pgs[file_idx-args.file_idx_offset]['questions'][q_idx]
 76 |         correct_question = True
 77 |         pred_list = []
 78 |         if 'choices' in q_ann:
 79 |             for c in q_ann['choices']:
 80 |                 full_pg = c['program'] + q_ann['program']
 81 |                 ans = c['answer'] if 'answer' in c else None
 82 |                 pred = exe.run(full_pg, debug=False)
 83 |                 pred = pred_map[pred]
 84 |                 if ans is not None and  ans == pred:
 85 |                     correct += 1
 86 |                 else:
 87 |                     correct_question = False
 88 |                 total += 1
 89 |                 
 90 |                 if q['question_type'].startswith('predictive'):
 91 |                     if ans is not None and  ans == pred:
 92 |                         correct_pred += 1
 93 |                     total_pred += 1
 94 | 
 95 |                 if q['question_type'].startswith('counterfactual'):
 96 |                     if ans is not None and  ans == pred:
 97 |                         correct_coun += 1
 98 |                     total_coun += 1
 99 |                 pred_list.append(pred)
100 |         ques_id = q['question_id']
101 |         tmp_pred[ques_id] = {'choices': pred_list, 'question_type': q['question_type'] }
102 | 
103 |         if correct_question:
104 |             correct_per_q += 1
105 |         total_per_q += 1
106 | 
107 |         if q['question_type'].startswith('predictive'):
108 |             if correct_question:
109 |                 correct_pred_per_q += 1
110 |             total_pred_per_q += 1
111 | 
112 |         if q['question_type'].startswith('counterfactual'):
113 |             if correct_question:
114 |                 correct_coun_per_q += 1
115 |             total_coun_per_q += 1
116 |         valid_q_idx += 1
117 |     prediction_dict[file_idx] =  tmp_pred
118 |     pbar.set_description('per choice {:f}, per questions {:f}'.format(float(correct)*100/max(total, EPS), float(correct_per_q)*100/max(total_per_q, EPS)))
119 | 
120 | if ans is not None:
121 |     print('============ results ============')
122 |     print('overall accuracy per option: %f %%' % (float(correct) * 100.0 / total))
123 |     print('overall accuracy per question: %f %%' % (float(correct_per_q) * 100.0 / total_per_q))
124 |     print('predictive accuracy per option: %f %%' % (float(correct_pred) * 100.0 / max( total_pred, EPS)))
125 |     print('predictive accuracy per question: %f %%' % (float(correct_pred_per_q) * 100.0 / max(total_pred_per_q, EPS)))
126 |     print('counterfactual accuracy per option: %f %%' % (float(correct_coun) * 100.0 / max(total_coun, EPS)))
127 |     print('counterfactual accuracy per question: %f %%' % (float(correct_coun_per_q) * 100.0 / max(total_coun_per_q, EPS)))
128 |     print('============ results ============')
129 | 
130 | output_file = args.result_out_fn
131 | if output_file!='':
132 |     with open(output_file, 'w') as fout:
133 |         json.dump(prediction_dict, fout)
134 | 


--------------------------------------------------------------------------------
/run_oe_clevrer.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Run symbolic reasoning on open-ended questions
  3 | """
  4 | import os
  5 | import json
  6 | from tqdm import tqdm
  7 | import argparse
  8 | 
  9 | from executor import Executor
 10 | from simulation import Simulation
 11 | import pdb
 12 | from utils.utils import print_monitor
 13 | import numpy as np
 14 | 
 15 | parser = argparse.ArgumentParser()
 16 | parser.add_argument('--use_event_ann', default=1, type=int)
 17 | parser.add_argument('--use_in', default=0, type=int)  # Interaction network
 18 | parser.add_argument('--program_path', default='/home/zfchen/code/clevrer_dataset_generation_v2/clevrer_question_generation/output/questions_v13_2/open_end_questions.json')
 19 | parser.add_argument('--question_path', default='/home/zfchen/code/clevrer_dataset_generation_v2/clevrer_question_generation/output/questions_v13_2/open_end_questions.json')
 20 | parser.add_argument('--gt_flag', default=0, type=int)
 21 | parser.add_argument('--ann_dir', default='/home/zfchen/code/output/render_output/causal_v13')
 22 | parser.add_argument('--track_dir', default='/home/zfchen/code/output/render_output/causal_v13_coco_ann')
 23 | parser.add_argument('--frame_diff', default=5, type=int)
 24 | parser.add_argument('--mc_flag', default=0, type=int)
 25 | parser.add_argument('--raw_motion_prediction_dir', default='')
 26 | parser.add_argument('--start_id', default=0, type=int)
 27 | parser.add_argument('--num_sim', default=5000, type=int)
 28 | parser.add_argument('--ann_offset', default=0, type=int)
 29 | parser.add_argument('--gt_ann_dir', default='/home/zfchen/code/output/render_output_vislab3/v16/render')
 30 | parser.add_argument('--ambi_ques_list_fn', default = './data/ambi_ques_v16.txt')
 31 | parser.add_argument('--gen_ambi_flag', default=0, type=int)
 32 | parser.add_argument('--save_prediction_fn', default = '')
 33 | args = parser.parse_args()
 34 | 
 35 | question_path = args.question_path
 36 | program_path = args.program_path 
 37 | 
 38 | with open(program_path) as f:
 39 |     parsed_pgs = json.load(f)
 40 | with open(question_path) as f:
 41 |     anns = json.load(f)
 42 | 
 43 | if args.gen_ambi_flag:
 44 |     fh = open(args.ambi_ques_list_fn, 'w')
 45 | 
 46 | total, correct = 0, 0
 47 | 
 48 | pbar = tqdm(range(args.num_sim))
 49 | 
 50 | acc_monitor = {}
 51 | acc_monitor_2 = {}
 52 | ans_swap = ''
 53 | prediction_dict = {}
 54 | 
 55 | for ann_idx in pbar:
 56 |     file_idx = ann_idx  + args.start_id
 57 |     ques_idx = file_idx - args.ann_offset
 58 |     question_scene = anns[ques_idx]
 59 |     sim = Simulation(args, file_idx, use_event_ann=(args.use_event_ann != 0))
 60 |     exe = Executor(sim)
 61 |     tmp_dict = {}
 62 |     for q_idx, q in enumerate(question_scene['questions']):
 63 |         if 'question_type' in q and 'multiple_choice' in q['question_type']:
 64 |             continue
 65 |         
 66 |         question = q['question']
 67 |         parsed_pg = parsed_pgs[ques_idx]['questions'][q_idx]['program']
 68 |         q_type = parsed_pg[-1]
 69 |         if q_type+'_acc' not in acc_monitor:
 70 |             acc_monitor[q_type+'_acc'] = 0
 71 |             acc_monitor[q_type+'_total'] = 1
 72 |         else:
 73 |             acc_monitor[q_type+'_total'] +=1
 74 | 
 75 |         q_fam = q['question_family']
 76 |         if q_fam+'_acc' not in acc_monitor_2:
 77 |             acc_monitor_2[q_fam+'_acc'] = 0
 78 |             acc_monitor_2[q_fam+'_total'] = 1
 79 |         else:
 80 |             acc_monitor_2[q_fam+'_total'] +=1
 81 | 
 82 |         pred = exe.run(parsed_pg, debug=False)
 83 |         ques_id = q['question_id']
 84 |         tmp_dict[ques_id] = pred
 85 | 
 86 |         ans = q['answer'] if 'answer' in q else None
 87 |         if ans is not None and pred == ans:
 88 |             correct += 1
 89 |             acc_monitor[q_type+'_acc'] +=1
 90 |             acc_monitor_2[q_fam+'_acc'] +=1
 91 |         elif ans is not None and 'and' in ans: # for query_both     
 92 |             eles = ans.split(' ')
 93 |             if len(eles)==3:
 94 |                 ans_swap  = eles[2] + ' and ' + eles[0]
 95 |                 if pred == ans_swap:
 96 |                     correct +=1
 97 |                     acc_monitor[q_type+'_acc'] +=1
 98 |                     acc_monitor_2[q_fam+'_acc'] +=1
 99 |         total += 1
100 |     pbar.set_description('acc: {:f}%%'.format(float(correct)*100/total))
101 |     prediction_dict[file_idx] = tmp_dict
102 | 
103 | if ans is not None:
104 |     print_monitor(acc_monitor)
105 |     print_monitor(acc_monitor_2)
106 |     print('overall accuracy per question: %f %%' % (float(correct) * 100.0 / total))
107 | if args.save_prediction_fn !='':
108 |     with open(args.save_prediction_fn, 'w') as fh:
109 |         json.dump(prediction_dict, fh)
110 |     print('Saving predictions to %s\n'%(args.save_prediction_fn))
111 | 


--------------------------------------------------------------------------------
/scripts/test_mc_release.sh:
--------------------------------------------------------------------------------
 1 | QUES_PATH='/home/tmp_user/code/output/comPhy/questions/test.json'
 2 | ANN_DIR='/home/tmp_user/code/output/render_output_vislab3/v16_test/prediction_prp_mass_charge'
 3 | #PRED_DIR='/home/tmp_user/code/output/render_output_vislab3/v16_test/predictions_motion_prp_no_ref_v2'
 4 | PRED_DIR='/home/tmp_user/code/output/render_output_vislab3/v16_test/predictions_motion_prp_no_ref_v2_vis50_new'
 5 | python run_mc_clevrer.py \
 6 |     --gt_flag 0 \
 7 |     --use_event_ann 0 \
 8 |     --raw_motion_prediction_dir ${PRED_DIR} \
 9 |     --ann_dir ${ANN_DIR} \
10 |     --start_id 10000 \
11 |     --num_sim 2000 \
12 |     --program_path  ${QUES_PATH} \
13 |     --question_path ${QUES_PATH} \
14 |     --result_out_fn test_mc.json \
15 |     #--program_path /home/tmp_user/code/clevrer_dataset_generation_v2/clevrer_question_generation/output/questions_v16_3_1_test/multiple_choice_questions.json \
16 |     #--question_path /home/tmp_user/code/clevrer_dataset_generation_v2/clevrer_question_generation/output/questions_v16_3_1_test/multiple_choice_questions.json \
17 | 


--------------------------------------------------------------------------------
/scripts/test_oe_release.sh:
--------------------------------------------------------------------------------
 1 | QUES_PATH='/home/zfchen/code/output/comPhy/questions/test.json'
 2 | ANN_DIR='/home/zfchen/code/output/render_output_vislab3/v16_test/prediction_prp_mass_charge'
 3 | PRED_DIR='/home/zfchen/code/output/render_output_vislab3/v16_test/predictions_motion_prp_no_ref_v2_vis50_new'
 4 | #PRED_DIR='/home/zfchen/code/output/render_output_vislab3/v16_test/predictions_motion_prp_no_ref_v2'
 5 | python run_oe_clevrer.py \
 6 |     --gt_flag 0 \
 7 |     --use_event_ann 0 \
 8 |     --program_path ${QUES_PATH} \
 9 |     --question_path ${QUES_PATH} \
10 |     --raw_motion_prediction_dir ${PRED_DIR} \
11 |     --ann_dir ${ANN_DIR} \
12 |     --start_id 10000 \
13 |     --num_sim 2000 \
14 |     --ann_offset 10000 \
15 |     --save_prediction_fn test_oe_v2.json \
16 | 


--------------------------------------------------------------------------------
/simulation.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import json
  3 | import numpy as np
  4 | from utils import utils
  5 | import pdb
  6 | import json
  7 | 
  8 | MOVING_V_TH = 0.25  # threshold above which an object is moving
  9 | DIR_ANGLE_TH = 20  # threshold for allowed angle deviation wrt each directions
 10 | FRAME_DIFF = 5
 11 | 
 12 | DV_TH = 0.003
 13 | #DIST_TH = 20
 14 | DIST_TH = 68
 15 | #DIST_TH = 60
 16 | print(DIST_TH)
 17 | print(DV_TH)
 18 | EPS = 0.000001
 19 | 
 20 | class Simulation():
 21 | 
 22 |     def __init__(self, args, sim_id, n_vis_frames=125, use_event_ann=True):
 23 |         
 24 |         objs, preds, edges = utils.load_ann(sim_id, args)
 25 |         self.objs = objs
 26 |         self.preds = preds
 27 |         self.edges = edges # for proposal handling
 28 |         self.num_objs = len(self.objs)
 29 |         self.sim_id = sim_id
 30 |        
 31 |         self.args = args
 32 |         FRAME_DIFF = args.frame_diff
 33 |         self.frame_diff = FRAME_DIFF
 34 |         self.n_vis_frames = n_vis_frames
 35 |         self.moving_v_th = MOVING_V_TH
 36 |         self._init_sim_no_event()
 37 |         if use_event_ann:
 38 |             self._init_collision_gt()
 39 | 
 40 |     def get_what_if_id(self, pred):
 41 |         p = pred
 42 |         if p['what_if'] == -1:
 43 |             return -1
 44 |         if 'mass' in p:
 45 |             cf_id = str(p['what_if']) +'_mass_' + str(p['mass'])
 46 |         if 'charge' in p:
 47 |             cf_id = str(p['what_if']) +'_charge_' +str(p['charge'])
 48 |         return cf_id
 49 | 
 50 |     def get_visible_objs(self):
 51 |         #return [o['id'] for o in self.objs]
 52 |         if self.args.gt_flag:
 53 |             obj_list = [o['id'] for o in self.objs if self.is_visible(o['id'])]
 54 |         else:
 55 |             obj_list = [o['id'] for o in self.objs]
 56 |         return obj_list
 57 | 
 58 |     def get_static_attrs(self, obj_idx):
 59 |         for o in self.objs:
 60 |             if o['id'] == obj_idx:
 61 |                 attrs = {
 62 |                     'color': o['color'],
 63 |                     'material': o['material'],
 64 |                     'shape': o['shape'],
 65 |                 }
 66 |                 return attrs
 67 |         raise ValueError('Invalid object index')
 68 | 
 69 |     def is_visible(self, obj_idx, frame_idx=None, ann_idx=None, what_if=-1):
 70 |         if frame_idx is not None or ann_idx is not None:
 71 |             frame_ann = self._get_frame_ann(frame_idx, ann_idx, what_if)
 72 |             for o in frame_ann['objects']:
 73 |                 oid = self._get_obj_idx(o)
 74 |                 if oid == obj_idx:
 75 |                     return True
 76 |             return False
 77 |         else:
 78 |             for i, t in enumerate(self.preds[0]['trajectory']):
 79 |                 if t['frame_index'] < self.n_vis_frames and \
 80 |                    self.is_visible(obj_idx, ann_idx=i, what_if=what_if):
 81 |                     return True
 82 |             return False
 83 | 
 84 |     def is_moving(self, obj_idx, frame_idx=None, ann_idx=None):
 85 |         if frame_idx is not None or ann_idx is not None:
 86 |             frame_ann = self._get_frame_ann(frame_idx, ann_idx)
 87 |             for o in frame_ann['objects']:
 88 |                 oid = self._get_obj_idx(o)
 89 |                 if oid == obj_idx:
 90 |                     speed = np.linalg.norm([o['vx'], o['vy']])
 91 |                     return speed > self.moving_v_th
 92 |             print(frame_idx, ann_idx)
 93 |             raise ValueError('Invalid object index')
 94 |         else:
 95 |             for i, t in enumerate(self.preds[0]['trajectory']):
 96 |                 if t['frame_index'] >= self.n_vis_frames:
 97 |                     break
 98 |                 if self.is_visible(obj_idx, ann_idx=i) and \
 99 |                    self.is_moving(obj_idx, ann_idx=i):
100 |                     return True
101 |             return False
102 | 
103 |     def is_moving_up(self, obj_idx, frame_idx=None, ann_idx=None,
104 |                        angle_half_range=DIR_ANGLE_TH):
105 |         if frame_idx is not None:
106 |             frame_ann = self._get_frame_ann(frame_idx, ann_idx)
107 |             for o in frame_ann['objects']:
108 |                 oid = self._get_obj_idx(o)
109 |                 if oid == obj_idx:
110 |                     theta = np.arctan(o['vy'] / (o['vx']+EPS)) * 180 / np.pi
111 |                     if o['vx'] < 0:
112 |                         theta += 180
113 |                     return theta > 270 - angle_half_range or \
114 |                            theta < -90 + angle_half_range
115 |         else: 
116 |             valid_frm_list = []
117 |             for f in range(0, self.n_vis_frames, FRAME_DIFF):
118 |                 if self.is_visible(obj_idx, f)  and  \
119 |                         self.is_moving(obj_idx, f) and \
120 |                    not self.is_moving_up(
121 |                        obj_idx, f, angle_half_range=angle_half_range):
122 |                     valid_frm_list.append(0)
123 |                 else:
124 |                     valid_frm_list.append(1)
125 |             ratio = sum(valid_frm_list) / (len(valid_frm_list)+0.000001)
126 |             if ratio >=0.5:
127 |                 return True
128 |             else:
129 |                 return False
130 |         raise ValueError('Invalid object index')
131 | 
132 |     def is_moving_down(self, obj_idx, frame_idx=None, ann_idx=None,
133 |                        angle_half_range=DIR_ANGLE_TH):
134 |         if frame_idx is not None:
135 |             frame_ann = self._get_frame_ann(frame_idx, ann_idx)
136 |             for o in frame_ann['objects']:
137 |                 oid = self._get_obj_idx(o)
138 |                 if oid == obj_idx:
139 |                     theta = np.arctan(o['vy'] / (o['vx']+EPS)) * 180 / np.pi
140 |                     if o['vx'] < 0:
141 |                         theta += 180
142 |                     return theta > 90 - angle_half_range and \
143 |                            theta < 90 + angle_half_range
144 |         else: 
145 |             valid_frm_list = []
146 |             for f in range(0, self.n_vis_frames, FRAME_DIFF):
147 |                 if self.is_visible(obj_idx, f)  and  \
148 |                         self.is_moving(obj_idx, f) and \
149 |                    not self.is_moving_down(
150 |                        obj_idx, f, angle_half_range=angle_half_range):
151 |                     valid_frm_list.append(0)
152 |                 else:
153 |                     valid_frm_list.append(1)
154 |             ratio = sum(valid_frm_list) / (len(valid_frm_list)+0.000001)
155 |             if ratio >=0.5:
156 |                 return True
157 |             else:
158 |                 return False
159 |         raise ValueError('Invalid object index')
160 | 
161 |     def is_moving_left(self, obj_idx, frame_idx=None, ann_idx=None,
162 |                        angle_half_range=DIR_ANGLE_TH):
163 |         if frame_idx is not None:
164 |             frame_ann = self._get_frame_ann(frame_idx, ann_idx)
165 |             for o in frame_ann['objects']:
166 |                 oid = self._get_obj_idx(o)
167 |                 if oid == obj_idx:
168 |                     theta = np.arctan(o['vy'] / (o['vx'] + EPS)) * 180 / np.pi
169 |                     if o['vx'] < 0:
170 |                         theta += 180
171 |                     return theta > 180 - angle_half_range and \
172 |                            theta < 180 + angle_half_range
173 |         else: 
174 |             valid_frm_list = []
175 |             for f in range(0, self.n_vis_frames, FRAME_DIFF):
176 |                 if self.is_visible(obj_idx, f)  and  \
177 |                         self.is_moving(obj_idx, f) and \
178 |                    not self.is_moving_left(
179 |                        obj_idx, f, angle_half_range=angle_half_range):
180 |                     valid_frm_list.append(0)
181 |                 else:
182 |                     valid_frm_list.append(1)
183 |             ratio = sum(valid_frm_list) / (len(valid_frm_list)+0.000001)
184 |             if ratio >=0.5:
185 |                 return True
186 |             else:
187 |                 return False
188 |         raise ValueError('Invalid object index')
189 | 
190 |     def is_moving_right(self, obj_idx, frame_idx=None, ann_idx=None,
191 |                        angle_half_range=DIR_ANGLE_TH):
192 |         if frame_idx is not None:
193 |             frame_ann = self._get_frame_ann(frame_idx, ann_idx)
194 |             for o in frame_ann['objects']:
195 |                 oid = self._get_obj_idx(o)
196 |                 if oid == obj_idx:
197 |                     theta = np.arctan(o['vy'] / (o['vx']+EPS)) * 180 / np.pi
198 |                     if o['vx'] < 0:
199 |                         theta += 180
200 |                     return theta < 0 + angle_half_range and \
201 |                            theta > 0 - angle_half_range
202 |         else: 
203 |             valid_frm_list = []
204 |             for f in range(0, self.n_vis_frames, FRAME_DIFF):
205 |                 if self.is_visible(obj_idx, f)  and  \
206 |                         self.is_moving(obj_idx, f) and \
207 |                    not self.is_moving_right(
208 |                        obj_idx, f, angle_half_range=angle_half_range):
209 |                     valid_frm_list.append(0)
210 |                 else:
211 |                     valid_frm_list.append(1)
212 |             ratio = sum(valid_frm_list) / (len(valid_frm_list)+0.000001)
213 |             if ratio >=0.5:
214 |                 return True
215 |             else:
216 |                 return False
217 |         raise ValueError('Invalid object index')
218 | 
219 |     def _init_sim_no_event(self):
220 |         self.in_out = []
221 |         self.collisions = []
222 |         self.cf_events = {}
223 |         for k, p in enumerate(self.preds):
224 |             what_if_id =  self.get_what_if_id(p)
225 |             for i, t in enumerate(p['trajectory']):
226 |                 for o in t['objects']:
227 |                     o['id'] = self._get_obj_idx(o)
228 |                     vxs, vys = [], []
229 |                     if i != 0 and not self.is_visible(o['id'], ann_idx=i-1, what_if=what_if_id):
230 |                         if k == 0:
231 |                             self.in_out.append({'frame': t['frame_index'], 'type': 'in', 'object': [o['id']]})
232 |                     elif i != 0:
233 |                         x_prev, y_prev = self._get_obj_location(o['id'], ann_idx=i-1, what_if=what_if_id)
234 |                         vxs.append((o['x'] - x_prev) / self.frame_diff)
235 |                         vys.append((o['y'] - y_prev) / self.frame_diff)
236 |                     if i != len(p['trajectory']) - 1 and not self.is_visible(o['id'], ann_idx=i+1, what_if = what_if_id):
237 |                         if k == 0:
238 |                             self.in_out.append({'frame': p['trajectory'][i+1]['frame_index'], 'type': 'out', 'object': [o['id']]})
239 |                     elif i != len(p['trajectory']) - 1:
240 |                         x_next, y_next = self._get_obj_location(o['id'], ann_idx=i+1, what_if=what_if_id)
241 |                         vxs.append((x_next - o['x']) / self.frame_diff)
242 |                         vys.append((y_next - o['y']) / self.frame_diff)
243 |                     if len(vxs) != 0:
244 |                         o['vx'] = np.average(vxs)
245 |                         o['vy'] = np.average(vys)
246 |                     else:
247 |                         o['vx'], o['vy'] = 0, 0
248 |             if p['what_if'] == -1:
249 |                 #print('debug')
250 |                 #self.collisions = self._get_col_proposals_counterfact()
251 |                 self.collisions = self._get_col_proposals()
252 |             else:
253 |                 self.cf_events[what_if_id] = self._get_col_proposals(what_if_id)
254 | 
255 |     def _init_collision_gt(self):
256 |         ann_path = os.path.join(self.args.gt_ann_dir, 'causal_sim', 'sim_%05d'%self.sim_id, 'annotations', 'annotation.json')
257 |         with open(ann_path, 'r') as fh:
258 |             ann = json.load(fh)
259 |         self.collisions = []
260 |         for c in ann['collisions']:
261 |             obj1_idx = c['object_idxs'][0]
262 |             obj2_idx = c['object_idxs'][1]
263 |             self.collisions.append({
264 |                     'type': 'collision',
265 |                     'object': [obj1_idx, obj2_idx],
266 |                     'frame': int(c['time'] * 25),
267 |                 })
268 | 
269 |     def _init_sim(self):
270 |         self.in_out = []
271 |         self.collisions = []
272 |         p = self.preds[0]
273 |         for i, t in enumerate(p['trajectory']):
274 |             for o in t['objects']:
275 |                 o['id'] = self._get_obj_idx(o)
276 |                 vxs, vys = [], []
277 |                 if i != 0 and not self.is_visible(o['id'], ann_idx=i-1):
278 |                     self.in_out.append({'frame': t['frame_index'], 'type': 'in', 'object': [o['id']]})
279 |                 elif i != 0:
280 |                     x_prev, y_prev = self._get_obj_location(o['id'], ann_idx=i-1)
281 |                     vxs.append((o['x'] - x_prev) / self.frame_diff)
282 |                     vys.append((o['y'] - y_prev) / self.frame_diff)
283 |                 if i != len(p['trajectory']) - 1 and not self.is_visible(o['id'], ann_idx=i+1):
284 |                     self.in_out.append({'frame': p['trajectory'][i+1]['frame_index'], 'type': 'out', 'object': [o['id']]})
285 |                 elif i != len(p['trajectory']) - 1:
286 |                     x_next, y_next = self._get_obj_location(o['id'], ann_idx=i+1)
287 |                     vxs.append((x_next - o['x']) / self.frame_diff)
288 |                     vys.append((y_next - o['y']) / self.frame_diff)
289 |                 if len(vxs) != 0:
290 |                     o['vx'] = np.average(vxs)
291 |                     o['vy'] = np.average(vys)
292 |                 else:
293 |                     o['vx'], o['vy'] = 0, 0
294 |         for c in p['collisions']:
295 |             obj1_idx = self._get_obj_idx(c['objects'][0])
296 |             obj2_idx = self._get_obj_idx(c['objects'][1])
297 |             self.collisions.append({
298 |                     'type': 'collision',
299 |                     'object': [obj1_idx, obj2_idx],
300 |                     'frame': c['frame'],
301 |                 })
302 | 
303 |         self.cf_events = {}
304 |         for j in range(1, len(self.preds)):
305 |             assert self.preds[j]['what_if'] != -1
306 |             self.cf_events[self.preds[j]['what_if']] = []
307 |             for c in self.preds[j]['collisions']:
308 |                 obj1_idx = self._get_obj_idx(c['objects'][0])
309 |                 obj2_idx = self._get_obj_idx(c['objects'][1])
310 |                 self.cf_events[self.preds[j]['what_if']].append({
311 |                         'type': 'collision',
312 |                         'object': [obj1_idx, obj2_idx],
313 |                         'frame': c['frame'],
314 |                     })    
315 | 
316 |     def _get_obj_idx(self, obj):
317 |         for o in self.objs:
318 |             if o['color'] == obj['color'] and \
319 |                o['material'] == obj['material'] and \
320 |                o['shape'] == obj['shape']:
321 |                 return o['id']
322 |         return -1
323 | 
324 |     def search_obj_info_by_id(self, obj_id):
325 |         for idx, obj_info in enumerate(self.objs):
326 |             if obj_info['id']==obj_id:
327 |                 return obj_info
328 |         return -1
329 | 
330 |     def _get_frame_ann(self, frame_idx=None, ann_idx=None, what_if=-1):
331 |         assert ann_idx is not None or frame_idx is not None
332 |         target = None
333 |         if frame_idx is not None:
334 |             for t in self.search_pred_by_cf_id(what_if)['trajectory']:
335 |                 if t['frame_index'] == frame_idx:
336 |                     target = t
337 |                     break
338 |         else:
339 |             target = self.search_pred_by_cf_id(what_if)['trajectory'][ann_idx]
340 |         if target is None:
341 |             raise ValueError('Invalid input frame')
342 |         return target
343 | 
344 |     def _get_obj_location(self, obj_idx, frame_idx=None, ann_idx=None, what_if=-1):
345 |         assert self.is_visible(obj_idx, frame_idx, ann_idx, what_if=what_if)
346 |         frame_ann = self._get_frame_ann(frame_idx, ann_idx, what_if)
347 |         for o in frame_ann['objects']:
348 |             if self._get_obj_idx(o) == obj_idx:
349 |                 return o['x'], o['y']
350 | 
351 |     def search_pred_by_cf_id(self, what_if):
352 |         if what_if==-1:
353 |             return  self.preds[what_if+1]
354 |         obj_id, prop, prop_val = int(what_if.split('_')[0]), what_if.split('_')[1], int(what_if.split('_')[2])
355 |         for pred_id, pred_info in enumerate(self.preds):
356 |             what_if = pred_info['what_if']
357 |             if obj_id!=what_if:
358 |                 continue
359 |             if prop in pred_info and pred_info[prop]==prop_val:
360 |                 return pred_info
361 |         return 'error'
362 | 
363 |     def get_trace(self, obj, what_if=-1):
364 |         output = []
365 |         pred = self.search_pred_by_cf_id(what_if)
366 |         for t in pred['trajectory']:
367 |             for o in t['objects']:
368 |                 if o['id'] == obj:
369 |                     o['frame'] = t['frame_index']
370 |                     output.append(o)
371 |         return output
372 | 
373 |     def _get_col_frame_proposals(self, obj, what_if=-1):
374 |         proposed_frames = []
375 |         trace = self.get_trace(obj, what_if)
376 |         dvs = []
377 |         for i, o in enumerate(trace):
378 |             if i > 0:
379 |                 dvx = o['vx'] - trace[i-1]['vx']
380 |                 dvy = o['vy'] - trace[i-1]['vy']
381 |                 dv = np.linalg.norm([dvx, dvy])
382 |             else:
383 |                 dv = 0 
384 |             dvs.append(dv)
385 |         for j, dv in enumerate(dvs):
386 |             if j != 0 and j != len(dvs)-1:
387 |                 if dv > dvs[j-1] and dv > dvs[j+1] and dv > DV_TH and dv < 5 and self.is_visible(obj, frame_idx=trace[j]['frame'], what_if=what_if):
388 |                     proposed_frames.append(trace[j]['frame'])
389 |         return proposed_frames
390 | 
391 |     def _get_closest_obj(self, obj, frame_idx, what_if=-1):
392 |         assert self.is_visible(obj, frame_idx=frame_idx, what_if=what_if)
393 |         xo, yo = self._get_obj_location(obj, frame_idx=frame_idx, what_if=what_if)
394 |         obj_idxs = [o['id'] for o in self.objs]
395 |         min_dist = 99999
396 |         closest_obj = -1
397 |         for io in obj_idxs:
398 |             if io != obj and self.is_visible(io, frame_idx=frame_idx, what_if=what_if):
399 |                 x, y = self._get_obj_location(io, frame_idx=frame_idx, what_if=what_if)
400 |                 dist = np.linalg.norm([x-xo, y-yo])
401 |                 if dist < min_dist:
402 |                     min_dist = dist
403 |                     closest_obj = io
404 |         return closest_obj, min_dist
405 | 
406 |     def _get_closest_obj_list(self, obj, frame_idx, what_if=-1):
407 |         assert self.is_visible(obj, frame_idx=frame_idx, what_if=what_if)
408 |         xo, yo = self._get_obj_location(obj, frame_idx=frame_idx, what_if=what_if)
409 |         obj_idxs = [o['id'] for o in self.objs]
410 |         min_dist = 99999
411 |         closest_obj = -1
412 |         closest_obj_list  = []
413 |         min_dist_list  = []
414 |         for io in obj_idxs:
415 |             if io != obj and self.is_visible(io, frame_idx=frame_idx, what_if=what_if):
416 |                 x, y = self._get_obj_location(io, frame_idx=frame_idx, what_if=what_if)
417 |                 dist = np.linalg.norm([x-xo, y-yo])
418 |                 min_dist_list.append(dist)
419 |                 closest_obj_list.append(io)
420 |         return closest_obj_list, min_dist_list
421 | 
422 |     def _get_col_proposals_counterfact(self, what_if=-1):
423 |         cols = []
424 |         col_pairs = []
425 |         obj_idxs = [o['id'] for o in self.objs]
426 |         for io in obj_idxs:
427 |             col_frames = self._get_col_frame_proposals(io, what_if)
428 |             for f in col_frames:
429 |                 partner_list, dist_list = self._get_closest_obj_list(io, f, what_if)
430 |                 for partner, dist in zip(partner_list, dist_list):
431 |                     #if what_if==-1:
432 |                     #    print('frame: %d, object indexes: %d %d, dist: %f\n'%(f, io, partner, dist))
433 |                     if dist < DIST_TH and {io, partner} not in col_pairs:
434 |                         col_event = {
435 |                             'type': 'collision',
436 |                             'object': [io, partner],
437 |                             'frame': f,
438 |                         }
439 |                         cols.append(col_event)
440 |                         col_pairs.append({io, partner})
441 |         return cols
442 | 
443 |     def _get_col_proposals(self, what_if=-1):
444 |         cols = []
445 |         col_pairs = []
446 |         obj_idxs = [o['id'] for o in self.objs]
447 |         for io in obj_idxs:
448 |             col_frames = self._get_col_frame_proposals(io, what_if)
449 |             for f in col_frames:
450 |                 partner, dist = self._get_closest_obj(io, f, what_if)
451 |                 #if what_if==-1:
452 |                 #    print('frame: %d, object indexes: %d %d, dist: %f\n'%(f, io, partner, dist))
453 |                 if dist < DIST_TH and {io, partner} not in col_pairs:
454 |                     col_event = {
455 |                         'type': 'collision',
456 |                         'object': [io, partner],
457 |                         'frame': f,
458 |                     }
459 |                     cols.append(col_event)
460 |                     col_pairs.append({io, partner})
461 |         return cols
462 |     
463 |     def is_charged(self, obj_idx):
464 |         return self.objs[obj_idx]['charge']!=0
465 |     
466 |     def is_light(self, obj_idx):
467 |         return self.objs[obj_idx]['mass']==1
468 | 


--------------------------------------------------------------------------------
/test_oe.json:
--------------------------------------------------------------------------------
1 | {"10000": {"0": "yellow", "1": "metal", "2": "purple"}, "10001": {"0": "cube", "1": "right", "2": "yes"}, "10002": {"0": "error", "1": "error", "2": "yes"}, "10003": {"0": "1", "1": "metal", "2": "rubber"}, "10004": {"0": "yes", "1": "sphere", "2": "red"}, "10005": {"0": "error", "1": "yes", "2": "yes"}, "10006": {"0": "cube", "1": "3", "2": "green and red"}, "10007": {"0": "metal", "1": "error", "2": "purple"}, "10008": {"0": "purple", "1": "cube", "2": "no"}, "10009": {"0": "gray and purple", "1": "0", "2": "sphere"}, "10010": {"0": "yes", "1": "no", "2": "error"}, "10011": {"0": "error", "1": "right", "2": "yes"}, "10012": {"0": "cylinder", "1": "metal", "2": "purple"}, "10013": {"0": "right", "1": "yes", "2": "cyan"}, "10014": {"0": "error", "1": "yes", "2": "error"}, "10015": {"0": "cube", "1": "no", "2": "no"}, "10016": {"0": "error", "1": "error", "2": "2"}, "10017": {"0": "red and yellow", "1": "no", "2": "yellow"}, "10018": {"0": "cube and sphere", "1": "sphere", "2": "red"}, "10019": {"0": "sphere", "1": "yes", "2": "error"}, "10020": {"0": "rubber", "1": "yes", "2": "0"}, "10021": {"0": "cube", "1": "cylinder", "2": "cube"}, "10022": {"0": "error", "1": "no", "2": "yes"}, "10023": {"0": "0", "1": "purple"}, "10024": {"0": "purple", "1": "0", "2": "red"}, "10025": {"0": "cube and cube", "1": "no", "2": "brown and cyan"}, "10026": {"0": "no", "1": "cylinder", "2": "no"}, "10027": {"0": "yes", "1": "rubber", "2": "sphere and cube"}, "10028": {"0": "rubber", "1": "rubber", "2": "blue and cyan"}, "10029": {"0": "blue and gray", "1": "1", "2": "sphere and cylinder"}, "10030": {"0": "no", "1": "0", "2": "cylinder"}, "10031": {"0": "error", "1": "cube", "2": "rubber"}, "10032": {"0": "cube", "1": "rubber", "2": "yes"}, "10033": {"0": "rubber", "1": "rubber", "2": "yes"}, "10034": {"0": "no", "1": "rubber", "2": "up"}, "10035": {"0": "yellow", "1": "cyan", "2": "yellow"}, "10036": {"0": "brown and cyan", "1": "cylinder", "2": "yes"}, "10037": {"0": "yellow", "1": "sphere", "2": "0"}, "10038": {"0": "cube", "1": "left", "2": "gray and green"}, "10039": {"0": "0", "1": "sphere", "2": "sphere"}, "10040": {"0": "sphere and cube", "1": "brown", "2": "gray"}, "10041": {"0": "cylinder and cube", "1": "yes", "2": "red"}, "10042": {"0": "yes", "1": "brown and gray", "2": "brown"}, "10043": {"0": "no", "1": "error", "2": "down"}, "10044": {"0": "yes", "1": "cube", "2": "metal"}, "10045": {"0": "rubber", "1": "metal", "2": "rubber"}, "10046": {"0": "error", "1": "yellow", "2": "3", "3": "brown and purple", "4": "yellow", "5": "0", "6": "cube", "7": "gray", "8": "no", "9": "no"}, "10047": {"0": "error", "1": "1", "2": "no"}, "10048": {"0": "yellow", "1": "no", "2": "cylinder"}, "10049": {"0": "no", "1": "rubber", "2": "red and green"}, "10050": {"0": "gray", "1": "yes", "2": "down"}, "10051": {"0": "gray", "1": "no", "2": "cylinder"}, "10052": {"0": "error", "1": "rubber", "2": "no"}, "10053": {"0": "gray", "1": "0", "2": "error"}, "10054": {"0": "yes", "1": "metal"}, "10055": {"0": "no", "1": "sphere and cube", "2": "no"}, "10056": {"0": "cyan", "1": "cube and cylinder", "2": "no"}, "10057": {"0": "yes", "1": "cube", "2": "down"}, "10058": {"0": "no", "1": "3", "2": "yes"}, "10059": {"0": "cube", "1": "blue", "2": "blue and purple"}, "10060": {"0": "yes", "1": "no", "2": "rubber"}, "10061": {"0": "error", "1": "rubber", "2": "metal"}, "10062": {"0": "no", "1": "gray and blue", "2": "cylinder"}, "10063": {"0": "yes", "1": "metal", "2": "sphere"}, "10064": {"0": "brown", "1": "brown", "2": "no"}, "10065": {"0": "error", "1": "green", "2": "0", "3": "cyan and green", "4": "error", "5": "1", "6": "error", "7": "error", "8": "no", "9": "no"}, "10066": {"0": "cyan and green", "1": "green", "2": "cube and cylinder"}, "10067": {"0": "1", "1": "no", "2": "metal"}, "10068": {"0": "blue", "1": "blue", "2": "no", "3": "blue and green", "4": "no", "5": "blue", "6": "metal", "7": "no", "8": "yes"}, "10069": {"0": "rubber", "1": "cube and cube", "2": "cube"}, "10070": {"0": "purple", "1": "brown", "2": "1"}, "10071": {"0": "metal", "1": "no"}, "10072": {"0": "no", "1": "purple", "2": "brown"}, "10073": {"0": "yellow", "1": "1", "2": "no"}, "10074": {"0": "sphere", "1": "error", "2": "1", "3": "green and brown", "4": "0", "5": "green", "6": "green", "7": "0", "8": "2"}, "10075": {"0": "error", "1": "purple", "2": "0"}, "10076": {"0": "yellow", "1": "red", "2": "2", "3": "sphere and cylinder", "4": "yellow and green", "5": "yes", "6": "rubber", "7": "sphere", "8": "no", "9": "yes"}, "10077": {"0": "red and blue", "1": "down", "2": "no"}, "10078": {"0": "no", "1": "cube", "2": "cyan and green"}, "10079": {"0": "3", "1": "purple", "2": "metal"}, "10080": {"0": "cyan and yellow", "1": "yellow", "2": "no"}, "10081": {"0": "down", "1": "sphere", "2": "no"}, "10082": {"0": "yes", "1": "purple", "2": "metal"}, "10083": {"0": "blue and purple", "1": "green"}, "10084": {"0": "0", "1": "1", "2": "gray"}, "10085": {"0": "no", "1": "brown and cyan", "2": "yes"}, "10086": {"0": "blue", "1": "yes", "2": "yes"}, "10087": {"0": "rubber", "1": "red", "2": "yes"}, "10088": {"0": "red and red", "1": "1", "2": "2"}, "10089": {"0": "error", "1": "right", "2": "3"}, "10090": {"0": "brown and gray", "1": "metal", "2": "cube"}, "10091": {"0": "3", "1": "cylinder", "2": "sphere"}, "10092": {"0": "metal", "1": "no", "2": "red and gray"}, "10093": {"0": "yellow", "1": "3", "2": "cylinder"}, "10094": {"0": "purple", "1": "red", "2": "green"}, "10095": {"0": "down", "1": "red", "2": "cube"}, "10096": {"0": "brown", "1": "left", "2": "brown"}, "10097": {"0": "2", "1": "error"}, "10098": {"0": "no", "1": "error", "2": "gray"}, "10099": {"0": "0", "1": "no", "2": "2"}, "10100": {"0": "sphere", "1": "metal", "2": "no", "3": "red and purple", "4": "no", "5": "metal", "6": "purple", "7": "no", "8": "yes"}, "10101": {"0": "yes", "1": "2", "2": "metal"}, "10102": {"0": "rubber", "1": "0", "2": "yes"}, "10103": {"0": "yellow", "1": "brown", "2": "brown and red"}, "10104": {"0": "sphere and cylinder", "1": "cylinder", "2": "blue and purple"}, "10105": {"0": "cylinder and cylinder", "1": "yes", "2": "0"}, "10106": {"0": "cylinder", "1": "red and blue", "2": "yes"}, "10107": {"0": "green", "1": "green", "2": "no"}, "10108": {"0": "down", "1": "red", "2": "cube and cylinder"}, "10109": {"0": "cylinder", "1": "rubber", "2": "0"}, "10110": {"0": "cylinder and sphere", "1": "error", "2": "yellow"}, "10111": {"0": "cyan and red", "1": "0", "2": "cyan"}, "10112": {"0": "no", "1": "up", "2": "brown and cyan"}, "10113": {"0": "2", "1": "up", "2": "cylinder"}, "10114": {"0": "cube", "1": "cube", "2": "no"}, "10115": {"0": "sphere and cube", "1": "cyan and yellow", "2": "sphere"}, "10116": {"0": "sphere", "1": "error", "2": "0", "3": "error", "4": "sphere and cylinder", "5": "yes", "6": "gray", "7": "sphere", "8": "yes", "9": "no"}, "10117": {"0": "rubber", "1": "1", "2": "error"}, "10118": {"0": "1", "1": "gray", "2": "yes"}, "10119": {"0": "metal", "1": "yellow", "2": "yes"}, "10120": {"0": "0", "1": "yes", "2": "no"}, "10121": {"0": "cylinder and cube", "1": "no", "2": "error"}, "10122": {"0": "rubber", "1": "no", "2": "error"}, "10123": {"0": "0", "1": "cube", "2": "cylinder"}, "10124": {"0": "no", "1": "sphere and cylinder", "2": "sphere"}, "10125": {"0": "yes", "1": "cylinder", "2": "metal"}, "10126": {"0": "error", "1": "0", "2": "sphere"}, "10127": {"0": "gray and purple", "1": "purple", "2": "error"}, "10128": {"0": "purple", "1": "cylinder", "2": "yes", "3": "cyan and purple", "4": "cube and sphere", "5": "0", "6": "cyan", "7": "cube", "8": "no", "9": "no"}, "10129": {"0": "2", "1": "error", "2": "no"}, "10130": {"0": "rubber", "1": "no", "2": "no"}, "10131": {"0": "cylinder", "1": "rubber", "2": "0"}, "10132": {"0": "cube and cylinder", "1": "brown and blue", "2": "rubber"}, "10133": {"0": "no", "1": "yes", "2": "error"}, "10134": {"0": "red", "1": "purple", "2": "cube"}, "10135": {"0": "brown", "1": "yes", "2": "no"}, "10136": {"0": "metal", "1": "left", "2": "blue"}, "10137": {"0": "3", "1": "no", "2": "no"}, "10138": {"0": "cylinder", "1": "rubber", "2": "blue"}, "10139": {"0": "no", "1": "rubber", "2": "metal"}, "10140": {"0": "up", "1": "no", "2": "cube"}, "10141": {"0": "rubber", "1": "cylinder and cube", "2": "no"}, "10142": {"0": "red", "1": "yes", "2": "cylinder and sphere"}, "10143": {"0": "cylinder", "1": "0", "2": "down"}, "10144": {"0": "rubber", "1": "sphere", "2": "no", "3": "sphere and cylinder", "4": "brown and yellow", "5": "no", "6": "metal", "7": "gray", "8": "yes", "9": "yes"}, "10145": {"0": "purple and yellow", "1": "metal", "2": "metal"}, "10146": {"0": "rubber", "1": "no", "2": "sphere"}, "10147": {"0": "cube", "1": "purple", "2": "yellow"}, "10148": {"0": "yes", "1": "cube", "2": "red"}, "10149": {"0": "rubber", "1": "yes", "2": "cyan"}, "10150": {"0": "cylinder and cube", "1": "metal", "2": "cyan"}, "10151": {"0": "rubber", "1": "no", "2": "gray and purple"}, "10152": {"0": "2", "1": "error", "2": "left"}, "10153": {"0": "error", "1": "error", "2": "gray"}, "10154": {"0": "gray", "1": "cube", "2": "yes"}, "10155": {"0": "rubber", "1": "brown", "2": "sphere and cylinder"}, "10156": {"0": "1", "1": "metal", "2": "rubber"}, "10157": {"0": "red", "1": "1", "2": "sphere"}, "10158": {"0": "cylinder", "1": "cyan", "2": "3", "3": "cyan and purple", "4": "sphere and cylinder", "5": "2", "6": "down", "7": "cube", "8": "yes", "9": "no"}, "10159": {"0": "cube", "1": "cube", "2": "1", "3": "error", "4": "error", "5": "no", "6": "metal", "7": "cube", "8": "no", "9": "yes"}, "10160": {"0": "brown", "1": "no", "2": "2"}, "10161": {"0": "yellow", "1": "gray", "2": "blue"}, "10162": {"0": "up", "1": "up", "2": "gray"}, "10163": {"0": "yes", "1": "yes", "2": "blue"}, "10164": {"0": "rubber", "1": "metal", "2": "yes"}, "10165": {"0": "cylinder and sphere", "1": "up", "2": "no"}, "10166": {"0": "rubber", "1": "error", "2": "2", "3": "cube and cylinder", "4": "green and yellow", "5": "yes", "6": "cylinder", "7": "cube", "8": "yes", "9": "no"}, "10167": {"0": "cylinder", "1": "1", "2": "sphere"}, "10168": {"0": "brown", "1": "right", "2": "2"}, "10169": {"0": "yes", "1": "no", "2": "error"}, "10170": {"0": "no", "1": "cube", "2": "sphere"}, "10171": {"0": "1", "1": "green and purple", "2": "rubber"}, "10172": {"0": "metal", "1": "left", "2": "error"}, "10173": {"0": "0", "1": "cylinder", "2": "cylinder"}, "10174": {"0": "metal", "1": "no", "2": "no"}, "10175": {"0": "no", "1": "yellow", "2": "blue and cyan"}, "10176": {"0": "cylinder", "1": "no", "2": "0"}, "10177": {"0": "no", "1": "cube", "2": "3"}, "10178": {"0": "metal", "1": "rubber", "2": "error"}, "10179": {"0": "gray and brown", "1": "yes", "2": "yes"}, "10180": {"0": "no", "1": "metal", "2": "no"}, "10181": {"0": "brown", "1": "brown and brown", "2": "down"}, "10182": {"0": "cube", "1": "sphere and cube", "2": "no"}, "10183": {"0": "rubber", "1": "no", "2": "cyan and green"}, "10184": {"0": "2", "1": "gray", "2": "green"}, "10185": {"0": "rubber", "1": "1"}, "10186": {"0": "error", "1": "yes", "2": "error"}, "10187": {"0": "gray and red", "1": "yes"}, "10188": {"0": "right", "1": "no", "2": "gray"}, "10189": {"0": "no", "1": "error", "2": "no"}, "10190": {"0": "no", "1": "yes", "2": "cube and cylinder"}, "10191": {"0": "brown and cyan", "1": "no", "2": "1"}, "10192": {"0": "gray and red", "1": "cylinder", "2": "red"}, "10193": {"0": "no", "1": "error", "2": "yellow"}, "10194": {"0": "gray", "1": "yes", "2": "2"}, "10195": {"0": "yes", "1": "blue", "2": "sphere"}, "10196": {"0": "cube and sphere", "1": "green and purple", "2": "yes"}, "10197": {"0": "sphere and cube", "1": "cyan", "2": "brown"}, "10198": {"0": "gray and purple", "1": "error", "2": "yes"}, "10199": {"0": "1", "1": "purple", "2": "cube"}, "10200": {"0": "0", "1": "yellow and red", "2": "yellow"}, "10201": {"0": "cube and sphere", "1": "cylinder", "2": "yes"}, "10202": {"0": "1", "1": "yellow", "2": "brown and gray"}, "10203": {"0": "sphere and cube", "1": "cylinder", "2": "brown"}, "10204": {"0": "left", "1": "yes", "2": "cylinder and sphere"}, "10205": {"0": "yes", "1": "no", "2": "rubber"}, "10206": {"0": "4", "1": "cylinder", "2": "rubber"}, "10207": {"0": "2", "1": "cylinder", "2": "blue"}, "10208": {"0": "metal", "1": "blue and purple", "2": "0"}, "10209": {"0": "gray", "1": "purple", "2": "yes"}, "10210": {"0": "sphere", "1": "gray", "2": "no", "3": "yellow and gray", "4": "cube and sphere", "5": "1", "6": "error", "7": "down", "8": "no", "9": "yes"}, "10211": {"0": "yes", "1": "brown", "2": "error"}, "10212": {"0": "no", "1": "cyan", "2": "cyan and gray"}, "10213": {"0": "cube", "1": "0", "2": "purple"}, "10214": {"0": "error", "1": "no", "2": "cylinder"}, "10215": {"0": "yes", "1": "cylinder and cube", "2": "0"}, "10216": {"0": "green and purple", "1": "metal", "2": "red"}, "10217": {"0": "blue", "1": "sphere and cube", "2": "cube"}, "10218": {"0": "error", "1": "no", "2": "metal"}, "10219": {"0": "no", "1": "cyan"}, "10220": {"0": "error", "1": "metal", "2": "cylinder"}, "10221": {"0": "yes", "1": "cylinder", "2": "no"}, "10222": {"0": "no", "1": "rubber", "2": "error"}, "10223": {"0": "yes", "1": "rubber", "2": "sphere"}, "10224": {"0": "0", "1": "right", "2": "cylinder"}, "10225": {"0": "green and brown", "1": "sphere and cylinder", "2": "cube"}, "10226": {"0": "cyan", "1": "cube", "2": "no", "3": "cyan", "4": "cube and cylinder", "5": "1", "6": "cylinder", "7": "yellow", "8": "yes", "9": "no"}, "10227": {"0": "right", "1": "no"}, "10228": {"0": "error", "1": "rubber", "2": "red"}, "10229": {"0": "cyan", "1": "error", "2": "error"}, "10230": {"0": "red", "1": "yes", "2": "blue and purple"}, "10231": {"0": "yes", "1": "cyan", "2": "rubber"}, "10232": {"0": "error", "1": "sphere and cube", "2": "down"}, "10233": {"0": "cube and cube", "1": "metal"}, "10234": {"0": "no", "1": "yes", "2": "0"}, "10235": {"0": "blue", "1": "0"}, "10236": {"0": "cube and cylinder", "1": "blue", "2": "cube"}, "10237": {"0": "sphere", "1": "error", "2": "green"}, "10238": {"0": "1", "1": "brown and cyan", "2": "blue"}, "10239": {"0": "rubber", "1": "no", "2": "red and brown"}, "10240": {"0": "rubber", "1": "green and red", "2": "yes"}, "10241": {"0": "brown", "1": "error", "2": "rubber"}, "10242": {"0": "red", "1": "no", "2": "cyan and red"}, "10243": {"0": "green", "1": "green and yellow", "2": "green"}, "10244": {"0": "no", "1": "rubber", "2": "right"}, "10245": {"0": "cube and cylinder", "1": "yes", "2": "cylinder"}, "10246": {"0": "cube and cylinder", "1": "rubber", "2": "no"}, "10247": {"0": "error", "1": "yes", "2": "1"}, "10248": {"0": "error", "1": "purple", "2": "purple"}, "10249": {"0": "error", "1": "green", "2": "yes"}, "10250": {"0": "cube", "1": "no", "2": "sphere"}, "10251": {"0": "gray", "1": "yes", "2": "0"}, "10252": {"0": "3", "1": "sphere", "2": "error"}, "10253": {"0": "purple and red", "1": "yes", "2": "2"}, "10254": {"0": "yes", "1": "error", "2": "sphere"}, "10255": {"0": "purple", "1": "metal", "2": "sphere"}, "10256": {"0": "sphere", "1": "cube and sphere", "2": "1"}, "10257": {"0": "cylinder and cube", "1": "rubber", "2": "cyan"}, "10258": {"0": "cube", "1": "right", "2": "no"}, "10259": {"0": "0", "1": "gray", "2": "metal"}, "10260": {"0": "error", "1": "cube", "2": "no", "3": "blue and yellow", "4": "cylinder and cube", "5": "no", "6": "error", "7": "error", "8": "no", "9": "yes"}, "10261": {"0": "left", "1": "down", "2": "red and yellow"}, "10262": {"0": "purple", "1": "yellow", "2": "1"}, "10263": {"0": "error", "1": "green", "2": "error"}, "10264": {"0": "yes", "1": "metal"}, "10265": {"0": "no", "1": "gray", "2": "left"}, "10266": {"0": "0", "1": "error"}, "10267": {"0": "error", "1": "rubber", "2": "blue and purple"}, "10268": {"0": "2", "1": "rubber", "2": "cyan"}, "10269": {"0": "cylinder", "1": "red", "2": "gray"}, "10270": {"0": "green", "1": "cylinder", "2": "error"}, "10271": {"0": "right", "1": "red and yellow", "2": "0"}, "10272": {"0": "error", "1": "sphere", "2": "1"}, "10273": {"0": "yes", "1": "sphere", "2": "error"}, "10274": {"0": "blue", "1": "cyan", "2": "0"}, "10275": {"0": "metal", "1": "cyan", "2": "error", "3": "error", "4": "cyan", "5": "yes", "6": "error", "7": "error", "8": "yes", "9": "error"}, "10276": {"0": "cylinder", "1": "red", "2": "cylinder"}, "10277": {"0": "error", "1": "2"}, "10278": {"0": "left", "1": "brown", "2": "1"}, "10279": {"0": "no", "1": "yes", "2": "error"}, "10280": {"0": "metal", "1": "2", "2": "sphere"}, "10281": {"0": "cube and sphere", "1": "yes", "2": "red"}, "10282": {"0": "red", "1": "blue and purple", "2": "cube and cube"}, "10283": {"0": "yes", "1": "cube", "2": "rubber"}, "10284": {"0": "yes", "1": "cylinder and cube", "2": "4"}, "10285": {"0": "rubber", "1": "yellow", "2": "2", "3": "cyan and yellow", "4": "0", "5": "cylinder", "6": "cube", "7": "yes", "8": "yes"}, "10286": {"0": "left", "1": "rubber", "2": "yes"}, "10287": {"0": "metal", "1": "no", "2": "4"}, "10288": {"0": "error", "1": "sphere and cylinder", "2": "yes"}, "10289": {"0": "error", "1": "sphere", "2": "rubber"}, "10290": {"0": "yes", "1": "no", "2": "up"}, "10291": {"0": "right", "1": "cyan and yellow", "2": "error"}, "10292": {"0": "metal", "1": "blue", "2": "0"}, "10293": {"0": "error", "1": "brown"}, "10294": {"0": "yes", "1": "no", "2": "yes"}, "10295": {"0": "blue", "1": "sphere", "2": "yes"}, "10296": {"0": "yellow", "1": "no", "2": "brown and yellow"}, "10297": {"0": "sphere", "1": "0", "2": "cyan"}, "10298": {"0": "cylinder", "1": "rubber"}, "10299": {"0": "right", "1": "0", "2": "yellow"}, "10300": {"0": "brown", "1": "cylinder", "2": "blue"}, "10301": {"0": "gray and green", "1": "cylinder", "2": "no"}, "10302": {"0": "blue", "1": "error", "2": "error"}, "10303": {"0": "2", "1": "gray and purple", "2": "cube and sphere"}, "10304": {"0": "yes", "1": "cylinder", "2": "error"}, "10305": {"0": "cyan", "1": "yes", "2": "blue and cyan"}, "10306": {"0": "cube", "1": "error", "2": "3"}, "10307": {"0": "no", "1": "yellow", "2": "cylinder"}, "10308": {"0": "no", "1": "rubber", "2": "no"}, "10309": {"0": "cube and cube", "1": "yes", "2": "no"}, "10310": {"0": "left", "1": "1", "2": "cylinder"}, "10311": {"0": "gray", "1": "gray and red", "2": "no"}, "10312": {"0": "error", "1": "no", "2": "no"}, "10313": {"0": "no", "1": "error", "2": "cyan and brown"}, "10314": {"0": "cylinder", "1": "green", "2": "green"}, "10315": {"0": "error", "1": "error", "2": "1"}, "10316": {"0": "brown", "1": "error", "2": "purple"}, "10317": {"0": "cyan", "1": "yes", "2": "brown"}, "10318": {"0": "error", "1": "yes", "2": "cylinder"}, "10319": {"0": "cyan", "1": "sphere", "2": "no"}, "10320": {"0": "rubber", "1": "blue", "2": "yes"}, "10321": {"0": "cyan and yellow", "1": "cylinder", "2": "rubber"}, "10322": {"0": "gray", "1": "sphere", "2": "no"}, "10323": {"0": "brown and red", "1": "cylinder", "2": "red"}, "10324": {"0": "red", "1": "red and brown", "2": "sphere"}, "10325": {"0": "sphere", "1": "metal", "2": "no"}, "10326": {"0": "yes", "1": "blue", "2": "cube and sphere"}, "10327": {"0": "error", "1": "yellow", "2": "yes"}, "10328": {"0": "yellow", "1": "cylinder", "2": "yellow and cyan"}, "10329": {"0": "yellow", "1": "brown", "2": "yellow"}, "10330": {"0": "cyan", "1": "sphere", "2": "rubber"}, "10331": {"0": "right", "1": "0", "2": "metal"}, "10332": {"0": "cube", "1": "cube and sphere"}, "10333": {"0": "cylinder", "1": "no", "2": "gray and red"}, "10334": {"0": "sphere", "1": "metal", "2": "red"}, "10335": {"0": "no", "1": "error", "2": "metal"}, "10336": {"0": "sphere and cube", "1": "cube", "2": "no"}, "10337": {"0": "blue and purple", "1": "0", "2": "brown"}, "10338": {"0": "green", "1": "cube", "2": "error", "3": "gray and purple", "4": "sphere and cube", "5": "3", "6": "purple", "7": "sphere", "8": "no", "9": "no"}, "10339": {"0": "cube", "1": "purple"}, "10340": {"0": "error", "1": "cube", "2": "error"}, "10341": {"0": "cyan", "1": "metal", "2": "cube and cylinder"}, "10342": {"0": "green", "1": "no"}, "10343": {"0": "yes", "1": "no", "2": "1"}, "10344": {"0": "metal", "1": "error", "2": "error"}, "10345": {"0": "yes", "1": "no", "2": "yes"}, "10346": {"0": "error", "1": "right", "2": "up"}, "10347": {"0": "sphere", "1": "brown and yellow", "2": "1"}, "10348": {"0": "brown", "1": "no", "2": "error"}, "10349": {"0": "3", "1": "left", "2": "yellow"}, "10350": {"0": "0", "1": "green", "2": "cylinder"}, "10351": {"0": "yes", "1": "cylinder", "2": "brown and cyan"}, "10352": {"0": "cube", "1": "metal", "2": "cylinder"}, "10353": {"0": "0", "1": "rubber", "2": "3"}, "10354": {"0": "metal", "1": "2", "2": "cylinder"}, "10355": {"0": "brown", "1": "no", "2": "brown"}, "10356": {"0": "cyan", "1": "yes", "2": "sphere and cube"}, "10357": {"0": "cyan", "1": "no", "2": "no"}, "10358": {"0": "metal", "1": "sphere", "2": "2"}, "10359": {"0": "no", "1": "metal", "2": "red"}, "10360": {"0": "cube and cylinder", "1": "0", "2": "error"}, "10361": {"0": "error", "1": "cylinder", "2": "green"}, "10362": {"0": "sphere", "1": "error"}, "10363": {"0": "sphere and cube", "1": "yes", "2": "2"}, "10364": {"0": "error", "1": "gray", "2": "yes", "3": "error", "4": "0", "5": "down", "6": "yellow", "7": "no", "8": "no"}, "10365": {"0": "error", "1": "sphere", "2": "2"}, "10366": {"0": "no", "1": "rubber", "2": "purple"}, "10367": {"0": "error", "1": "error", "2": "no"}, "10368": {"0": "red", "1": "yes", "2": "cube"}, "10369": {"0": "no", "1": "rubber", "2": "yes"}, "10370": {"0": "yes", "1": "cylinder", "2": "3"}, "10371": {"0": "error", "1": "yes", "2": "error"}, "10372": {"0": "blue and yellow", "1": "cyan", "2": "3"}, "10373": {"0": "error", "1": "error", "2": "2"}, "10374": {"0": "no", "1": "cube and sphere", "2": "1"}, "10375": {"0": "3", "1": "error", "2": "down"}, "10376": {"0": "sphere", "1": "0", "2": "cyan and purple"}, "10377": {"0": "1", "1": "yellow", "2": "rubber"}, "10378": {"0": "green", "1": "cube and sphere", "2": "1"}, "10379": {"0": "rubber", "1": "rubber", "2": "yellow"}, "10380": {"0": "down", "1": "metal", "2": "cube"}, "10381": {"0": "1", "1": "yes"}, "10382": {"0": "sphere and cylinder", "1": "purple", "2": "green and purple"}, "10383": {"0": "sphere", "1": "metal", "2": "yes", "3": "blue", "4": "blue and blue", "5": "0", "6": "gray", "7": "error", "8": "yes", "9": "yes"}, "10384": {"0": "brown", "1": "no", "2": "sphere"}, "10385": {"0": "rubber", "1": "cube", "2": "yellow"}, "10386": {"0": "sphere", "1": "sphere", "2": "no"}, "10387": {"0": "no", "1": "brown", "2": "cube and cylinder"}, "10388": {"0": "1", "1": "no"}, "10389": {"0": "yes", "1": "right", "2": "cylinder"}, "10390": {"0": "yellow", "1": "green", "2": "blue"}, "10391": {"0": "2", "1": "blue", "2": "cylinder"}, "10392": {"0": "error", "1": "0", "2": "cube"}, "10393": {"0": "error", "1": "error", "2": "error"}, "10394": {"0": "3", "1": "green", "2": "gray"}, "10395": {"0": "2", "1": "yes", "2": "yes"}, "10396": {"0": "yellow", "1": "2", "2": "cube"}, "10397": {"0": "rubber", "1": "no", "2": "no"}, "10398": {"0": "yes", "1": "red", "2": "error"}, "10399": {"0": "error", "1": "yes", "2": "metal"}, "10400": {"0": "yes", "1": "yes", "2": "rubber"}, "10401": {"0": "yes", "1": "cube", "2": "brown and gray"}, "10402": {"0": "metal", "1": "cyan", "2": "cube and cube"}, "10403": {"0": "error", "1": "yes"}, "10404": {"0": "4", "1": "error", "2": "error"}, "10405": {"0": "cyan and yellow", "1": "cyan", "2": "gray"}, "10406": {"0": "sphere and cylinder", "1": "error"}, "10407": {"0": "brown", "1": "0", "2": "2"}, "10408": {"0": "cylinder and cube", "1": "yes", "2": "cyan"}, "10409": {"0": "sphere", "1": "yellow and gray", "2": "up"}, "10410": {"0": "metal", "1": "no", "2": "cube"}, "10411": {"0": "cube and cylinder", "1": "cylinder", "2": "rubber"}, "10412": {"0": "0", "1": "no", "2": "error"}, "10413": {"0": "blue", "1": "rubber", "2": "no"}, "10414": {"0": "metal", "1": "purple", "2": "sphere"}, "10415": {"0": "3", "1": "no", "2": "0"}, "10416": {"0": "error", "1": "cyan", "2": "cube"}, "10417": {"0": "brown", "1": "cylinder", "2": "0", "3": "cyan and brown", "4": "cylinder and cube", "5": "0", "6": "cyan", "7": "blue", "8": "0", "9": "0"}, "10418": {"0": "metal", "1": "sphere and cube", "2": "brown"}, "10419": {"0": "metal", "1": "3", "2": "2"}, "10420": {"0": "error", "1": "purple", "2": "0"}, "10421": {"0": "yes", "1": "cyan", "2": "yes"}, "10422": {"0": "rubber", "1": "no", "2": "blue"}, "10423": {"0": "cube", "1": "purple and brown", "2": "up"}, "10424": {"0": "red", "1": "yellow", "2": "yellow and red"}, "10425": {"0": "brown and purple", "1": "error", "2": "no"}, "10426": {"0": "yellow", "1": "rubber", "2": "yes"}, "10427": {"0": "red and brown", "1": "red", "2": "yes"}, "10428": {"0": "error", "1": "cube and sphere", "2": "red"}, "10429": {"0": "blue", "1": "brown and yellow", "2": "cylinder"}, "10430": {"0": "cylinder", "1": "sphere", "2": "green"}, "10431": {"0": "rubber", "1": "1", "2": "rubber"}, "10432": {"0": "cube", "1": "0", "2": "no"}, "10433": {"0": "blue", "1": "yes", "2": "0"}, "10434": {"0": "metal", "1": "rubber", "2": "1"}, "10435": {"0": "sphere", "1": "error", "2": "sphere"}, "10436": {"0": "metal", "1": "no", "2": "brown"}, "10437": {"0": "cube and cylinder", "1": "yes", "2": "gray"}, "10438": {"0": "cube", "1": "no", "2": "cylinder"}, "10439": {"0": "2", "1": "no", "2": "error"}, "10440": {"0": "error", "1": "rubber", "2": "yes"}, "10441": {"0": "cylinder", "1": "down", "2": "metal"}, "10442": {"0": "error", "1": "brown", "2": "right"}, "10443": {"0": "brown", "1": "yes", "2": "red"}, "10444": {"0": "right", "1": "rubber", "2": "blue"}, "10445": {"0": "cyan and green", "1": "cyan", "2": "yellow"}, "10446": {"0": "left", "1": "yes", "2": "brown"}, "10447": {"0": "error", "1": "error", "2": "1"}, "10448": {"0": "cyan", "1": "red", "2": "cyan and red"}, "10449": {"0": "red", "1": "metal", "2": "yes"}, "10450": {"0": "metal", "1": "0", "2": "sphere and cylinder"}, "10451": {"0": "yes", "1": "yes", "2": "red"}, "10452": {"0": "error", "1": "error", "2": "metal"}, "10453": {"0": "brown", "1": "red", "2": "yes"}, "10454": {"0": "gray", "1": "sphere and cylinder", "2": "1"}, "10455": {"0": "cube and cube", "1": "yes", "2": "blue and green"}, "10456": {"0": "error", "1": "no", "2": "error"}, "10457": {"0": "cube and cube", "1": "left", "2": "blue"}, "10458": {"0": "0", "1": "gray", "2": "rubber"}, "10459": {"0": "0", "1": "metal", "2": "error"}, "10460": {"0": "2", "1": "yellow", "2": "cube and sphere"}, "10461": {"0": "0", "1": "sphere", "2": "cube"}, "10462": {"0": "sphere", "1": "metal", "2": "cylinder and sphere"}, "10463": {"0": "cube", "1": "no", "2": "yes"}, "10464": {"0": "yellow", "1": "yes", "2": "yes"}, "10465": {"0": "0", "1": "no", "2": "metal"}, "10466": {"0": "purple", "1": "yes", "2": "purple and green"}, "10467": {"0": "1", "1": "cyan", "2": "metal"}, "10468": {"0": "yes", "1": "3", "2": "rubber"}, "10469": {"0": "green", "1": "cylinder", "2": "cylinder"}, "10470": {"0": "blue", "1": "cube", "2": "sphere"}, "10471": {"0": "yes", "1": "rubber", "2": "right"}, "10472": {"0": "0", "1": "cylinder", "2": "blue"}, "10473": {"0": "2", "1": "metal", "2": "green"}, "10474": {"0": "rubber", "1": "yellow", "2": "purple"}, "10475": {"0": "brown and red", "1": "rubber", "2": "metal"}, "10476": {"0": "sphere", "1": "yes", "2": "2"}, "10477": {"0": "no", "1": "0", "2": "purple"}, "10478": {"0": "no", "1": "purple", "2": "no"}, "10479": {"0": "red", "1": "sphere", "2": "error"}, "10480": {"0": "metal", "1": "yes", "2": "purple and green"}, "10481": {"0": "0", "1": "yellow and red", "2": "1"}, "10482": {"0": "metal", "1": "cyan", "2": "rubber"}, "10483": {"0": "cube", "1": "1", "2": "cube"}, "10484": {"0": "no", "1": "yes", "2": "no"}, "10485": {"0": "no", "1": "yellow", "2": "metal"}, "10486": {"0": "red", "1": "3", "2": "error"}, "10487": {"0": "blue", "1": "cube", "2": "cylinder"}, "10488": {"0": "error", "1": "sphere", "2": "gray"}, "10489": {"0": "metal", "1": "2", "2": "yes"}, "10490": {"0": "red and yellow", "1": "yes", "2": "cube"}, "10491": {"0": "cylinder", "1": "metal", "2": "gray"}, "10492": {"0": "1", "1": "0", "2": "down"}, "10493": {"0": "error", "1": "cube", "2": "yes"}, "10494": {"0": "left", "1": "yes", "2": "3"}, "10495": {"0": "cylinder", "1": "yes", "2": "error"}, "10496": {"0": "blue", "1": "cylinder", "2": "rubber"}, "10497": {"0": "green", "1": "cyan", "2": "up"}, "10498": {"0": "cylinder", "1": "metal", "2": "no"}, "10499": {"0": "gray", "1": "blue and green", "2": "cube and cylinder"}, "10500": {"0": "metal", "1": "green", "2": "yellow and cyan"}, "10501": {"0": "yes", "1": "brown and purple", "2": "cube"}, "10502": {"0": "1", "1": "no", "2": "cube and sphere"}, "10503": {"0": "brown", "1": "purple and red", "2": "cylinder"}, "10504": {"0": "cube", "1": "yes", "2": "cube and cylinder"}, "10505": {"0": "2", "1": "sphere", "2": "no"}, "10506": {"0": "yellow", "1": "error", "2": "yes"}, "10507": {"0": "brown", "1": "cube", "2": "error"}, "10508": {"0": "error", "1": "yes", "2": "sphere"}, "10509": {"0": "red", "1": "no", "2": "cube"}, "10510": {"0": "0", "1": "cylinder", "2": "brown"}, "10511": {"0": "4", "1": "cube", "2": "error"}, "10512": {"0": "no", "1": "3", "2": "yellow and cyan"}, "10513": {"0": "no", "1": "yes", "2": "rubber"}, "10514": {"0": "yes", "1": "left", "2": "rubber"}, "10515": {"0": "cube", "1": "error", "2": "0"}, "10516": {"0": "brown", "1": "cube", "2": "purple"}, "10517": {"0": "yes", "1": "blue and cyan", "2": "metal"}, "10518": {"0": "rubber", "1": "0", "2": "2"}, "10519": {"0": "green and red", "1": "yes", "2": "no"}, "10520": {"0": "sphere", "1": "error", "2": "sphere"}, "10521": {"0": "error", "1": "blue", "2": "yes"}, "10522": {"0": "cyan", "1": "error", "2": "yes"}, "10523": {"0": "0", "1": "cube and cylinder", "2": "error"}, "10524": {"0": "no", "1": "no", "2": "rubber"}, "10525": {"0": "no", "1": "cube", "2": "no"}, "10526": {"0": "0", "1": "0", "2": "yes"}, "10527": {"0": "metal", "1": "cube", "2": "error"}, "10528": {"0": "no", "1": "yellow", "2": "error"}, "10529": {"0": "error", "1": "yes", "2": "error"}, "10530": {"0": "error", "1": "metal", "2": "brown"}, "10531": {"0": "cyan", "1": "green", "2": "error"}, "10532": {"0": "sphere and cylinder", "1": "yellow", "2": "metal"}, "10533": {"0": "blue and cyan", "1": "no", "2": "yes"}, "10534": {"0": "error", "1": "2", "2": "cube and sphere"}, "10535": {"0": "cube", "1": "metal", "2": "no", "3": "error", "4": "error", "5": "0", "6": "sphere", "7": "sphere", "8": "yes", "9": "yes"}, "10536": {"0": "1", "1": "metal", "2": "yes"}, "10537": {"0": "sphere", "1": "brown", "2": "red"}, "10538": {"0": "sphere", "1": "down", "2": "yes"}, "10539": {"0": "left", "1": "error", "2": "1"}, "10540": {"0": "rubber", "1": "cube and sphere", "2": "metal"}, "10541": {"0": "yes", "1": "3", "2": "brown"}, "10542": {"0": "no", "1": "cylinder", "2": "sphere and cube"}, "10543": {"0": "yes", "1": "error", "2": "cube and sphere"}, "10544": {"0": "red", "1": "error", "2": "red"}, "10545": {"0": "blue and green", "1": "cube and sphere", "2": "cube"}, "10546": {"0": "purple", "1": "sphere", "2": "blue"}, "10547": {"0": "yes", "1": "rubber", "2": "metal"}, "10548": {"0": "3", "1": "brown and cyan", "2": "metal"}, "10549": {"0": "yes", "1": "red", "2": "error"}, "10550": {"0": "cyan", "1": "metal", "2": "cylinder"}, "10551": {"0": "error", "1": "1", "2": "rubber"}, "10552": {"0": "yes", "1": "yes", "2": "yes"}, "10553": {"0": "2", "1": "cylinder and cube", "2": "sphere"}, "10554": {"0": "yes", "1": "error", "2": "error"}, "10555": {"0": "cyan", "1": "0", "2": "up"}, "10556": {"0": "yellow", "1": "no", "2": "rubber"}, "10557": {"0": "gray", "1": "cyan", "2": "1"}, "10558": {"0": "gray", "1": "right", "2": "brown"}, "10559": {"0": "cylinder and cube", "1": "no", "2": "cylinder"}, "10560": {"0": "no", "1": "brown", "2": "cylinder"}, "10561": {"0": "cube and cylinder", "1": "green and red", "2": "yellow"}, "10562": {"0": "error", "1": "gray", "2": "yellow"}, "10563": {"0": "cyan", "1": "error", "2": "0"}, "10564": {"0": "cube", "1": "error"}, "10565": {"0": "rubber", "1": "sphere", "2": "error"}, "10566": {"0": "sphere", "1": "gray", "2": "0"}, "10567": {"0": "error", "1": "metal"}, "10568": {"0": "brown and purple", "1": "0", "2": "gray"}, "10569": {"0": "red and green", "1": "rubber", "2": "left"}, "10570": {"0": "error", "1": "green", "2": "2"}, "10571": {"0": "yes", "1": "rubber", "2": "yes"}, "10572": {"0": "2", "1": "red", "2": "1"}, "10573": {"0": "cube", "1": "1", "2": "blue and purple"}, "10574": {"0": "metal", "1": "no", "2": "error"}, "10575": {"0": "cylinder and cube", "1": "rubber", "2": "purple"}, "10576": {"0": "up", "1": "error", "2": "cylinder"}, "10577": {"0": "down", "1": "cyan", "2": "cube and sphere"}, "10578": {"0": "error", "1": "cube and cylinder", "2": "gray and green"}, "10579": {"0": "rubber", "1": "rubber", "2": "cube and cylinder"}, "10580": {"0": "cube", "1": "sphere", "2": "no"}, "10581": {"0": "purple and blue", "1": "1", "2": "error"}, "10582": {"0": "gray and red", "1": "cyan", "2": "right"}, "10583": {"0": "0", "1": "left", "2": "no"}, "10584": {"0": "green", "1": "no", "2": "error"}, "10585": {"0": "sphere", "1": "cyan", "2": "cube and sphere"}, "10586": {"0": "error", "1": "0", "2": "no"}, "10587": {"0": "1", "1": "cylinder", "2": "rubber"}, "10588": {"0": "blue", "1": "blue and green", "2": "2"}, "10589": {"0": "error", "1": "no", "2": "cyan"}, "10590": {"0": "0", "1": "rubber", "2": "brown"}, "10591": {"0": "sphere", "1": "rubber", "2": "no", "3": "green and red", "4": "cylinder and sphere", "5": "yes", "6": "sphere", "7": "green", "8": "no", "9": "1"}, "10592": {"0": "cylinder", "1": "sphere and cube", "2": "purple and cyan"}, "10593": {"0": "blue and brown", "1": "cylinder", "2": "yes"}, "10594": {"0": "yes", "1": "yes", "2": "error"}, "10595": {"0": "sphere", "1": "0", "2": "4"}, "10596": {"0": "no", "1": "1", "2": "purple and green"}, "10597": {"0": "error", "1": "no", "2": "sphere"}, "10598": {"0": "no", "1": "cube", "2": "metal"}, "10599": {"0": "blue", "1": "0", "2": "blue and purple"}, "10600": {"0": "error", "1": "yes", "2": "down"}, "10601": {"0": "down", "1": "brown and gray", "2": "rubber"}, "10602": {"0": "error", "1": "error", "2": "2", "3": "green and red", "4": "cube and cylinder", "5": "yes", "6": "no", "7": "1"}, "10603": {"0": "1", "1": "yes", "2": "cube"}, "10604": {"0": "blue", "1": "rubber", "2": "purple"}, "10605": {"0": "metal", "1": "error", "2": "rubber"}, "10606": {"0": "error", "1": "1", "2": "no"}, "10607": {"0": "brown", "1": "2", "2": "brown"}, "10608": {"0": "error", "1": "metal", "2": "error"}, "10609": {"0": "yellow", "1": "yes", "2": "yellow and green"}, "10610": {"0": "no", "1": "error", "2": "no"}, "10611": {"0": "gray", "1": "purple", "2": "2"}, "10612": {"0": "red", "1": "no", "2": "cube and sphere"}, "10613": {"0": "rubber", "1": "no", "2": "purple"}, "10614": {"0": "error", "1": "1", "2": "1"}, "10615": {"0": "yes", "1": "sphere", "2": "cylinder"}, "10616": {"0": "gray and purple", "1": "no", "2": "yes"}, "10617": {"0": "cylinder", "1": "error", "2": "cylinder"}, "10618": {"0": "yes", "1": "brown", "2": "yes"}, "10619": {"0": "brown", "1": "0", "2": "brown and yellow"}, "10620": {"0": "metal", "1": "metal", "2": "no"}, "10621": {"0": "gray and yellow", "1": "cube", "2": "gray"}, "10622": {"0": "error", "1": "rubber", "2": "yes"}, "10623": {"0": "0", "1": "brown and yellow", "2": "cylinder"}, "10624": {"0": "rubber", "1": "cube", "2": "no"}, "10625": {"0": "cube", "1": "2", "2": "cyan and purple"}, "10626": {"0": "sphere and cylinder", "1": "rubber", "2": "1"}, "10627": {"0": "brown", "1": "cube and sphere", "2": "brown"}, "10628": {"0": "no", "1": "rubber", "2": "rubber"}, "10629": {"0": "blue", "1": "error", "2": "sphere and cube"}, "10630": {"0": "cylinder", "1": "yes", "2": "rubber"}, "10631": {"0": "metal", "1": "brown and gray", "2": "cube and cylinder"}, "10632": {"0": "blue", "1": "no", "2": "cylinder"}, "10633": {"0": "yes", "1": "rubber", "2": "metal"}, "10634": {"0": "rubber", "1": "right", "2": "gray and brown"}, "10635": {"0": "error", "1": "right", "2": "0"}, "10636": {"0": "error", "1": "green", "2": "sphere"}, "10637": {"0": "blue", "1": "no", "2": "error"}, "10638": {"0": "rubber", "1": "red", "2": "blue and cyan"}, "10639": {"0": "cube and cube", "1": "1", "2": "0"}, "10640": {"0": "error", "1": "blue and purple"}, "10641": {"0": "cylinder", "1": "0"}, "10642": {"0": "brown", "1": "rubber", "2": "yellow"}, "10643": {"0": "red", "1": "blue", "2": "cube and cylinder"}, "10644": {"0": "cube", "1": "metal", "2": "1", "3": "no", "4": "brown", "5": "cube", "6": "no", "7": "yes"}, "10645": {"0": "rubber", "1": "rubber", "2": "green"}, "10646": {"0": "cylinder and sphere", "1": "yes", "2": "yes"}, "10647": {"0": "sphere", "1": "no", "2": "green"}, "10648": {"0": "blue", "1": "red", "2": "no"}, "10649": {"0": "rubber", "1": "metal", "2": "cylinder and sphere"}, "10650": {"0": "blue and brown", "1": "cylinder", "2": "metal"}, "10651": {"0": "green", "1": "brown and green", "2": "error"}, "10652": {"0": "yes", "1": "no", "2": "cylinder"}, "10653": {"0": "error", "1": "cylinder", "2": "no"}, "10654": {"0": "cube", "1": "no", "2": "gray"}, "10655": {"0": "cube", "1": "red and green", "2": "0"}, "10656": {"0": "cyan", "1": "error", "2": "no"}, "10657": {"0": "purple", "1": "yellow", "2": "yes"}, "10658": {"0": "0", "1": "rubber", "2": "cyan and green"}, "10659": {"0": "red", "1": "purple", "2": "cylinder"}, "10660": {"0": "sphere", "1": "right", "2": "1"}, "10661": {"0": "0", "1": "no"}, "10662": {"0": "error", "1": "error", "2": "cylinder and cube"}, "10663": {"0": "no", "1": "brown", "2": "sphere and cylinder"}, "10664": {"0": "1", "1": "rubber", "2": "error"}, "10665": {"0": "yellow and green", "1": "1", "2": "cylinder"}, "10666": {"0": "4", "1": "cyan", "2": "yes"}, "10667": {"0": "purple and red", "1": "rubber", "2": "0"}, "10668": {"0": "3", "1": "no", "2": "sphere and cube"}, "10669": {"0": "rubber", "1": "gray", "2": "error"}, "10670": {"0": "yes", "1": "cube and cylinder", "2": "1"}, "10671": {"0": "metal", "1": "error", "2": "brown"}, "10672": {"0": "1", "1": "cube", "2": "cyan"}, "10673": {"0": "sphere", "1": "yes", "2": "error"}, "10674": {"0": "sphere and cube", "1": "yes"}, "10675": {"0": "error", "1": "rubber", "2": "rubber"}, "10676": {"0": "sphere", "1": "error", "2": "gray"}, "10677": {"0": "sphere", "1": "yellow", "2": "no"}, "10678": {"0": "cylinder and cube", "1": "cube", "2": "cube"}, "10679": {"0": "error", "1": "gray", "2": "error"}, "10680": {"0": "1", "1": "no", "2": "cyan"}, "10681": {"0": "left", "1": "gray", "2": "1"}, "10682": {"0": "no", "1": "left", "2": "error"}, "10683": {"0": "error", "1": "1", "2": "sphere"}, "10684": {"0": "red", "1": "blue", "2": "gray"}, "10685": {"0": "rubber", "1": "left", "2": "yes"}, "10686": {"0": "no", "1": "cyan", "2": "3"}, "10687": {"0": "error", "1": "no"}, "10688": {"0": "gray", "1": "2"}, "10689": {"0": "green", "1": "yes", "2": "no"}, "10690": {"0": "blue", "1": "yes", "2": "2"}, "10691": {"0": "2", "1": "rubber", "2": "red"}, "10692": {"0": "yellow", "1": "cube", "2": "cylinder and sphere"}, "10693": {"0": "brown and purple", "1": "yes", "2": "yes"}, "10694": {"0": "metal", "1": "metal", "2": "yellow and brown"}, "10695": {"0": "rubber", "1": "rubber", "2": "cyan"}, "10696": {"0": "blue", "1": "gray", "2": "0"}, "10697": {"0": "error", "1": "no", "2": "sphere"}, "10698": {"0": "cube", "1": "sphere", "2": "yes"}, "10699": {"0": "brown and cyan", "1": "down", "2": "cube and cylinder"}, "10700": {"0": "error", "1": "cube", "2": "no"}, "10701": {"0": "gray", "1": "yes", "2": "error"}, "10702": {"0": "yes", "1": "rubber", "2": "sphere"}, "10703": {"0": "2", "1": "yellow", "2": "cylinder"}, "10704": {"0": "metal", "1": "yes", "2": "0"}, "10705": {"0": "5", "1": "green", "2": "green"}, "10706": {"0": "no", "1": "cylinder and cube", "2": "down"}, "10707": {"0": "metal", "1": "no", "2": "yellow"}, "10708": {"0": "rubber", "1": "cylinder and cube", "2": "no"}, "10709": {"0": "yes", "1": "cylinder and sphere", "2": "blue"}, "10710": {"0": "purple", "1": "brown", "2": "error"}, "10711": {"0": "purple", "1": "brown and purple", "2": "error"}, "10712": {"0": "yes", "1": "no", "2": "1"}, "10713": {"0": "gray", "1": "rubber", "2": "yellow"}, "10714": {"0": "2", "1": "gray and red", "2": "red"}, "10715": {"0": "red", "1": "3", "2": "no"}, "10716": {"0": "error", "1": "no", "2": "error"}, "10717": {"0": "sphere and cylinder", "1": "cube", "2": "3"}, "10718": {"0": "no", "1": "error", "2": "cube"}, "10719": {"0": "metal", "1": "gray and red", "2": "2"}, "10720": {"0": "metal", "1": "yes", "2": "purple and cyan"}, "10721": {"0": "up", "1": "cylinder", "2": "brown"}, "10722": {"0": "red", "1": "no", "2": "down"}, "10723": {"0": "2", "1": "cube", "2": "cyan"}, "10724": {"0": "cube", "1": "yes", "2": "green"}, "10725": {"0": "error", "1": "red", "2": "right"}, "10726": {"0": "3", "1": "brown", "2": "error"}, "10727": {"0": "error", "1": "metal", "2": "metal"}, "10728": {"0": "yellow", "1": "yes", "2": "no"}, "10729": {"0": "blue and gray", "1": "error", "2": "no"}, "10730": {"0": "error", "1": "metal", "2": "error"}, "10731": {"0": "cylinder", "1": "cylinder and cube", "2": "rubber"}, "10732": {"0": "red", "1": "1", "2": "yes"}, "10733": {"0": "error", "1": "metal", "2": "yes", "3": "error", "4": "gray and green", "5": "yes", "6": "cube", "7": "green", "8": "no", "9": "no"}, "10734": {"0": "cube and cylinder", "1": "2", "2": "cylinder"}, "10735": {"0": "brown and cyan", "1": "sphere and cylinder", "2": "1"}, "10736": {"0": "green", "1": "yes", "2": "gray and gray"}, "10737": {"0": "cyan", "1": "rubber", "2": "yes"}, "10738": {"0": "no", "1": "metal", "2": "right"}, "10739": {"0": "yes", "1": "gray", "2": "gray"}, "10740": {"0": "yes", "1": "error", "2": "error"}, "10741": {"0": "0", "1": "cube", "2": "cube"}, "10742": {"0": "no", "1": "1", "2": "gray and cyan"}, "10743": {"0": "green and red", "1": "yes"}, "10744": {"0": "1", "1": "yes", "2": "no"}, "10745": {"0": "no", "1": "purple", "2": "green"}, "10746": {"0": "0", "1": "0", "2": "brown and red"}, "10747": {"0": "red", "1": "error", "2": "0"}, "10748": {"0": "blue", "1": "blue", "2": "sphere"}, "10749": {"0": "error", "1": "error", "2": "error"}, "10750": {"0": "yes", "1": "cube and sphere", "2": "red and gray"}, "10751": {"0": "rubber", "1": "gray", "2": "purple and brown"}, "10752": {"0": "yes", "1": "2", "2": "yellow and green"}, "10753": {"0": "right", "1": "metal", "2": "sphere and cube"}, "10754": {"0": "rubber", "1": "0", "2": "0"}, "10755": {"0": "0", "1": "2", "2": "rubber"}, "10756": {"0": "brown and red", "1": "yes", "2": "error"}, "10757": {"0": "error", "1": "brown and purple", "2": "error"}, "10758": {"0": "0", "1": "gray", "2": "up"}, "10759": {"0": "rubber", "1": "yes", "2": "error"}, "10760": {"0": "cube", "1": "cyan", "2": "1", "3": "cube", "4": "cube and sphere", "5": "0", "6": "cyan", "7": "cyan", "8": "yes", "9": "yes"}, "10761": {"0": "purple", "1": "1", "2": "1"}, "10762": {"0": "1", "1": "brown", "2": "yes"}, "10763": {"0": "green", "1": "metal", "2": "yes"}, "10764": {"0": "1", "1": "green", "2": "green"}, "10765": {"0": "cube and cylinder", "1": "0", "2": "1"}, "10766": {"0": "cube and sphere", "1": "metal", "2": "metal"}, "10767": {"0": "green and gray", "1": "cyan", "2": "cyan"}, "10768": {"0": "error", "1": "cube", "2": "error"}, "10769": {"0": "left", "1": "0", "2": "error"}, "10770": {"0": "left", "1": "error", "2": "blue"}, "10771": {"0": "sphere", "1": "0", "2": "cube"}, "10772": {"0": "0", "1": "error", "2": "error"}, "10773": {"0": "metal", "1": "sphere", "2": "1"}, "10774": {"0": "metal", "1": "yes", "2": "error"}, "10775": {"0": "cyan", "1": "green", "2": "cyan and gray"}, "10776": {"0": "cylinder", "1": "yes", "2": "green and blue"}, "10777": {"0": "metal", "1": "error", "2": "cylinder"}, "10778": {"0": "yes", "1": "cube", "2": "red and cyan"}, "10779": {"0": "down", "1": "error", "2": "down"}, "10780": {"0": "2", "1": "no", "2": "blue and green"}, "10781": {"0": "error", "1": "gray and gray", "2": "down"}, "10782": {"0": "error", "1": "right"}, "10783": {"0": "rubber", "1": "yes", "2": "metal"}, "10784": {"0": "cube", "1": "down", "2": "brown and cyan"}, "10785": {"0": "error", "1": "metal", "2": "yes"}, "10786": {"0": "3", "1": "cyan"}, "10787": {"0": "no", "1": "metal", "2": "sphere"}, "10788": {"0": "green and purple", "1": "error", "2": "yellow"}, "10789": {"0": "error", "1": "no"}, "10790": {"0": "yes", "1": "blue", "2": "blue"}, "10791": {"0": "1", "1": "gray and blue", "2": "rubber"}, "10792": {"0": "purple", "1": "cube", "2": "1"}, "10793": {"0": "cylinder", "1": "0", "2": "metal"}, "10794": {"0": "yellow", "1": "yes", "2": "yellow"}, "10795": {"0": "3", "1": "error", "2": "error"}, "10796": {"0": "rubber", "1": "rubber", "2": "blue and green"}, "10797": {"0": "sphere", "1": "rubber", "2": "2"}, "10798": {"0": "yes", "1": "error", "2": "error"}, "10799": {"0": "no", "1": "yes", "2": "sphere"}, "10800": {"0": "metal", "1": "rubber", "2": "yes"}, "10801": {"0": "3", "1": "error", "2": "blue and gray"}, "10802": {"0": "brown", "1": "cylinder", "2": "brown and blue"}, "10803": {"0": "blue", "1": "gray", "2": "up"}, "10804": {"0": "right", "1": "error", "2": "error"}, "10805": {"0": "cube", "1": "cube", "2": "no"}, "10806": {"0": "blue", "1": "no", "2": "yes"}, "10807": {"0": "sphere", "1": "down", "2": "yes"}, "10808": {"0": "1", "1": "no", "2": "metal"}, "10809": {"0": "yes", "1": "no", "2": "error"}, "10810": {"0": "sphere", "1": "3", "2": "error"}, "10811": {"0": "no", "1": "cylinder and sphere", "2": "metal"}, "10812": {"0": "rubber", "1": "sphere and cube", "2": "sphere"}, "10813": {"0": "1", "1": "yellow", "2": "red"}, "10814": {"0": "yes", "1": "metal", "2": "yellow"}, "10815": {"0": "sphere", "1": "sphere", "2": "gray and green"}, "10816": {"0": "green and gray", "1": "green", "2": "down"}, "10817": {"0": "yes", "1": "purple", "2": "yes"}, "10818": {"0": "brown and cyan", "1": "no", "2": "cylinder"}, "10819": {"0": "sphere", "1": "metal", "2": "yes", "3": "purple", "4": "sphere and cylinder", "5": "0", "6": "error", "7": "right", "8": "no", "9": "yes"}, "10820": {"0": "sphere and cube", "1": "no", "2": "rubber"}, "10821": {"0": "cube", "1": "rubber", "2": "2", "3": "brown and yellow", "4": "cylinder and sphere", "5": "no", "6": "cylinder", "7": "up", "8": "no", "9": "no"}, "10822": {"0": "cube and cylinder", "1": "no", "2": "gray and yellow"}, "10823": {"0": "error", "1": "0", "2": "metal"}, "10824": {"0": "purple", "1": "cube", "2": "rubber"}, "10825": {"0": "cylinder", "1": "metal", "2": "red"}, "10826": {"0": "cube", "1": "no", "2": "rubber"}, "10827": {"0": "yes", "1": "green", "2": "yes"}, "10828": {"0": "error", "1": "yes", "2": "sphere"}, "10829": {"0": "green", "1": "brown", "2": "green"}, "10830": {"0": "4", "1": "yes", "2": "error"}, "10831": {"0": "cylinder", "1": "error", "2": "cube"}, "10832": {"0": "rubber", "1": "cyan and purple", "2": "0"}, "10833": {"0": "rubber", "1": "right", "2": "cyan and gray"}, "10834": {"0": "cyan", "1": "1", "2": "cube"}, "10835": {"0": "green", "1": "cylinder", "2": "3"}, "10836": {"0": "sphere", "1": "left", "2": "0"}, "10837": {"0": "gray", "1": "sphere", "2": "yes"}, "10838": {"0": "rubber", "1": "cube", "2": "sphere and cylinder"}, "10839": {"0": "error", "1": "right", "2": "gray"}, "10840": {"0": "rubber", "1": "yes", "2": "metal"}, "10841": {"0": "no", "1": "sphere", "2": "cube"}, "10842": {"0": "sphere", "1": "green", "2": "rubber"}, "10843": {"0": "green and yellow", "1": "brown", "2": "1"}, "10844": {"0": "yes", "1": "error", "2": "sphere"}, "10845": {"0": "red", "1": "error", "2": "0", "3": "blue and red", "4": "yes", "5": "left", "6": "right", "7": "no", "8": "no"}, "10846": {"0": "yes", "1": "cyan", "2": "no"}, "10847": {"0": "0", "1": "left", "2": "purple"}, "10848": {"0": "left", "1": "2", "2": "cube"}, "10849": {"0": "no", "1": "cylinder", "2": "error"}, "10850": {"0": "no", "1": "rubber", "2": "cube and cylinder"}, "10851": {"0": "3", "1": "red", "2": "left"}, "10852": {"0": "0", "1": "gray", "2": "sphere"}, "10853": {"0": "sphere", "1": "no", "2": "cube"}, "10854": {"0": "yes", "1": "gray", "2": "cube"}, "10855": {"0": "0", "1": "no", "2": "left"}, "10856": {"0": "error", "1": "no", "2": "green"}, "10857": {"0": "no", "1": "gray", "2": "metal"}, "10858": {"0": "sphere", "1": "error", "2": "error"}, "10859": {"0": "2", "1": "cyan", "2": "error"}, "10860": {"0": "cube", "1": "rubber", "2": "cylinder"}, "10861": {"0": "2", "1": "metal", "2": "metal"}, "10862": {"0": "sphere", "1": "error", "2": "yes", "3": "gray and purple", "4": "error", "5": "yes", "6": "blue", "7": "gray", "8": "yes", "9": "no"}, "10863": {"0": "rubber", "1": "blue", "2": "4"}, "10864": {"0": "yes", "1": "metal", "2": "rubber"}, "10865": {"0": "yes", "1": "no", "2": "cylinder"}, "10866": {"0": "0", "1": "green and brown", "2": "error"}, "10867": {"0": "yes", "1": "yes", "2": "error"}, "10868": {"0": "error", "1": "yes", "2": "purple"}, "10869": {"0": "yes", "1": "no"}, "10870": {"0": "error", "1": "brown", "2": "0"}, "10871": {"0": "rubber", "1": "red", "2": "red and yellow"}, "10872": {"0": "sphere", "1": "yellow", "2": "sphere"}, "10873": {"0": "cylinder", "1": "cyan and purple", "2": "no"}, "10874": {"0": "green", "1": "green", "2": "error"}, "10875": {"0": "no", "1": "red", "2": "no"}, "10876": {"0": "yes", "1": "brown", "2": "yes"}, "10877": {"0": "error", "1": "cyan and gray", "2": "error"}, "10878": {"0": "yes", "1": "2", "2": "down"}, "10879": {"0": "up", "1": "yes", "2": "yes"}, "10880": {"0": "rubber", "1": "no", "2": "error"}, "10881": {"0": "no", "1": "left", "2": "rubber"}, "10882": {"0": "left", "1": "red", "2": "down"}, "10883": {"0": "rubber", "1": "rubber", "2": "gray and purple"}, "10884": {"0": "yes", "1": "error", "2": "rubber"}, "10885": {"0": "error", "1": "2", "2": "cube"}, "10886": {"0": "yellow", "1": "cube and cylinder", "2": "2"}, "10887": {"0": "yes", "1": "cylinder", "2": "cube"}, "10888": {"0": "error", "1": "0", "2": "error"}, "10889": {"0": "metal", "1": "no", "2": "brown"}, "10890": {"0": "no", "1": "purple and brown", "2": "sphere"}, "10891": {"0": "1", "1": "yellow", "2": "purple"}, "10892": {"0": "rubber", "1": "left", "2": "yes"}, "10893": {"0": "yes", "1": "green", "2": "yes"}, "10894": {"0": "cube", "1": "brown and purple", "2": "0"}, "10895": {"0": "error", "1": "sphere", "2": "no"}, "10896": {"0": "sphere", "1": "no", "2": "2"}, "10897": {"0": "error", "1": "0", "2": "cube"}, "10898": {"0": "no", "1": "rubber", "2": "cyan"}, "10899": {"0": "rubber", "1": "gray", "2": "1"}, "10900": {"0": "yes", "1": "metal", "2": "cyan"}, "10901": {"0": "green and gray", "1": "error", "2": "1"}, "10902": {"0": "cylinder", "1": "error", "2": "no"}, "10903": {"0": "metal", "1": "1", "2": "cube"}, "10904": {"0": "1", "1": "left", "2": "yellow and blue"}, "10905": {"0": "sphere", "1": "yellow", "2": "yes"}, "10906": {"0": "metal", "1": "no", "2": "yes"}, "10907": {"0": "gray", "1": "error", "2": "yes"}, "10908": {"0": "2", "1": "blue", "2": "cube"}, "10909": {"0": "cyan", "1": "cylinder", "2": "no"}, "10910": {"0": "red", "1": "cube", "2": "red"}, "10911": {"0": "error", "1": "0", "2": "metal"}, "10912": {"0": "cylinder and cube", "1": "no", "2": "red"}, "10913": {"0": "sphere and cylinder", "1": "brown and red", "2": "green"}, "10914": {"0": "error", "1": "rubber", "2": "metal"}, "10915": {"0": "error", "1": "cube", "2": "yes"}, "10916": {"0": "red", "1": "error", "2": "purple and red"}, "10917": {"0": "metal", "1": "cube and cube", "2": "yellow and blue"}, "10918": {"0": "up", "1": "brown", "2": "2"}, "10919": {"0": "cube", "1": "error", "2": "error"}, "10920": {"0": "error", "1": "yes", "2": "error"}, "10921": {"0": "2", "1": "cube", "2": "blue"}, "10922": {"0": "metal", "1": "0", "2": "metal"}, "10923": {"0": "no", "1": "blue", "2": "1"}, "10924": {"0": "metal", "1": "cylinder", "2": "yes"}, "10925": {"0": "cylinder and sphere", "1": "no", "2": "cyan"}, "10926": {"0": "error", "1": "no", "2": "left"}, "10927": {"0": "cube and sphere", "1": "1", "2": "no"}, "10928": {"0": "0", "1": "no", "2": "cyan"}, "10929": {"0": "yes", "1": "cyan", "2": "cylinder"}, "10930": {"0": "red", "1": "yes", "2": "yes"}, "10931": {"0": "error", "1": "no", "2": "cylinder"}, "10932": {"0": "green", "1": "0", "2": "cylinder"}, "10933": {"0": "rubber", "1": "up", "2": "no"}, "10934": {"0": "cube and cylinder", "1": "yellow", "2": "3"}, "10935": {"0": "cube", "1": "cube", "2": "no"}, "10936": {"0": "brown and green", "1": "left", "2": "cube and sphere"}, "10937": {"0": "cube", "1": "0", "2": "blue"}, "10938": {"0": "left", "1": "rubber", "2": "1"}, "10939": {"0": "sphere", "1": "blue", "2": "yes"}, "10940": {"0": "no", "1": "rubber", "2": "cylinder"}, "10941": {"0": "cylinder", "1": "cylinder", "2": "error"}, "10942": {"0": "sphere", "1": "0", "2": "error"}, "10943": {"0": "right", "1": "rubber", "2": "yes"}, "10944": {"0": "rubber", "1": "purple", "2": "left"}, "10945": {"0": "cylinder", "1": "rubber", "2": "1", "3": "blue and brown", "4": "sphere", "5": "no", "6": "sphere", "7": "green", "8": "no", "9": "yes"}, "10946": {"0": "blue and cyan", "1": "cylinder", "2": "yes"}, "10947": {"0": "yes", "1": "purple and blue", "2": "sphere"}, "10948": {"0": "cyan", "1": "rubber", "2": "0", "3": "cyan and yellow", "4": "cylinder and sphere", "5": "yes", "6": "metal", "7": "blue", "8": "no", "9": "yes"}, "10949": {"0": "brown", "1": "error", "2": "metal"}, "10950": {"0": "yes", "1": "0", "2": "brown and purple"}, "10951": {"0": "cyan and yellow", "1": "yellow", "2": "right"}, "10952": {"0": "cube and cylinder", "1": "yes", "2": "1"}, "10953": {"0": "no", "1": "yes", "2": "sphere"}, "10954": {"0": "gray and blue", "1": "blue", "2": "no"}, "10955": {"0": "0", "1": "blue", "2": "no"}, "10956": {"0": "error", "1": "rubber", "2": "cube"}, "10957": {"0": "error", "1": "cube and cylinder", "2": "brown"}, "10958": {"0": "yes", "1": "3"}, "10959": {"0": "green", "1": "error", "2": "metal"}, "10960": {"0": "cube", "1": "yes", "2": "error"}, "10961": {"0": "cylinder", "1": "no", "2": "rubber"}, "10962": {"0": "green", "1": "cyan", "2": "0"}, "10963": {"0": "red", "1": "cube", "2": "3", "3": "green and red", "4": "cylinder and sphere", "5": "0", "6": "right", "7": "error", "8": "yes", "9": "yes"}, "10964": {"0": "sphere", "1": "error", "2": "cube"}, "10965": {"0": "metal", "1": "cylinder and sphere", "2": "2"}, "10966": {"0": "green", "1": "green and blue", "2": "yes"}, "10967": {"0": "2", "1": "red", "2": "2"}, "10968": {"0": "brown", "1": "right", "2": "3"}, "10969": {"0": "yes", "1": "no", "2": "cylinder"}, "10970": {"0": "yes", "1": "error", "2": "gray"}, "10971": {"0": "cylinder", "1": "yes", "2": "purple"}, "10972": {"0": "blue", "1": "yes", "2": "purple"}, "10973": {"0": "sphere", "1": "down", "2": "error"}, "10974": {"0": "0", "1": "error", "2": "error"}, "10975": {"0": "0", "1": "no", "2": "brown and cyan"}, "10976": {"0": "metal", "1": "rubber", "2": "metal"}, "10977": {"0": "cyan", "1": "yes", "2": "yes"}, "10978": {"0": "no", "1": "1", "2": "blue and gray"}, "10979": {"0": "error", "1": "cube", "2": "up"}, "10980": {"0": "0", "1": "gray and yellow", "2": "cylinder"}, "10981": {"0": "blue and cyan", "1": "0", "2": "2"}, "10982": {"0": "brown", "1": "error", "2": "rubber"}, "10983": {"0": "1", "1": "cube", "2": "error"}, "10984": {"0": "error", "1": "rubber", "2": "0"}, "10985": {"0": "no", "1": "metal", "2": "cube and sphere"}, "10986": {"0": "yes", "1": "cylinder", "2": "sphere"}, "10987": {"0": "no", "1": "no", "2": "error"}, "10988": {"0": "yes", "1": "green", "2": "green"}, "10989": {"0": "purple and yellow", "1": "yes", "2": "sphere"}, "10990": {"0": "down", "1": "yes", "2": "gray"}, "10991": {"0": "sphere and cylinder", "1": "yes", "2": "sphere"}, "10992": {"0": "rubber", "1": "no", "2": "rubber"}, "10993": {"0": "yes", "1": "cube", "2": "cube"}, "10994": {"0": "cube", "1": "brown", "2": "brown"}, "10995": {"0": "blue", "1": "green", "2": "error"}, "10996": {"0": "cube", "1": "no", "2": "no"}, "10997": {"0": "purple", "1": "2", "2": "error"}, "10998": {"0": "error", "1": "brown", "2": "brown"}, "10999": {"0": "green", "1": "yes"}, "11000": {"0": "cylinder", "1": "2", "2": "up"}, "11001": {"0": "no", "1": "1", "2": "sphere"}, "11002": {"0": "metal", "1": "brown", "2": "blue"}, "11003": {"0": "blue", "1": "no", "2": "sphere"}, "11004": {"0": "brown", "1": "error", "2": "error"}, "11005": {"0": "cylinder", "1": "sphere", "2": "yes"}, "11006": {"0": "0", "1": "cyan", "2": "yes"}, "11007": {"0": "no", "1": "left", "2": "gray"}, "11008": {"0": "0", "1": "metal", "2": "yes"}, "11009": {"0": "2", "1": "error", "2": "metal"}, "11010": {"0": "1", "1": "yellow", "2": "red"}, "11011": {"0": "up", "1": "no", "2": "rubber"}, "11012": {"0": "red", "1": "gray", "2": "0"}, "11013": {"0": "gray", "1": "2", "2": "0"}, "11014": {"0": "no", "1": "error", "2": "cyan"}, "11015": {"0": "gray", "1": "error", "2": "0", "3": "cyan", "4": "gray", "5": "1", "6": "yellow", "7": "down", "8": "no", "9": "no"}, "11016": {"0": "error", "1": "3", "2": "yellow"}, "11017": {"0": "2", "1": "metal", "2": "red"}, "11018": {"0": "cylinder", "1": "sphere", "2": "blue"}, "11019": {"0": "no", "1": "rubber", "2": "rubber"}, "11020": {"0": "red", "1": "blue", "2": "red"}, "11021": {"0": "gray", "1": "2"}, "11022": {"0": "sphere", "1": "no", "2": "red"}, "11023": {"0": "2", "1": "purple", "2": "cube"}, "11024": {"0": "metal", "1": "1", "2": "cube"}, "11025": {"0": "metal", "1": "no", "2": "gray"}, "11026": {"0": "error", "1": "error", "2": "2"}, "11027": {"0": "no", "1": "down", "2": "1"}, "11028": {"0": "rubber", "1": "error", "2": "1"}, "11029": {"0": "cylinder", "1": "green"}, "11030": {"0": "cube", "1": "error", "2": "blue"}, "11031": {"0": "1", "1": "1", "2": "left"}, "11032": {"0": "brown", "1": "brown", "2": "rubber"}, "11033": {"0": "2", "1": "rubber", "2": "rubber"}, "11034": {"0": "right", "1": "0", "2": "down"}, "11035": {"0": "cube", "1": "1"}, "11036": {"0": "0", "1": "0"}, "11037": {"0": "error", "1": "purple", "2": "1"}, "11038": {"0": "no", "1": "2", "2": "error"}, "11039": {"0": "1", "1": "gray", "2": "cyan"}, "11040": {"0": "blue", "1": "cylinder", "2": "blue"}, "11041": {"0": "5", "1": "cube", "2": "right"}, "11042": {"0": "down", "1": "right", "2": "4"}, "11043": {"0": "cyan", "1": "yes", "2": "cyan"}, "11044": {"0": "sphere", "1": "0", "2": "no"}, "11045": {"0": "0", "1": "purple"}, "11046": {"0": "red", "1": "blue", "2": "3"}, "11047": {"0": "cube", "1": "purple", "2": "0"}, "11048": {"0": "1", "1": "metal", "2": "cylinder"}, "11049": {"0": "no", "1": "0", "2": "sphere"}, "11050": {"0": "0", "1": "metal"}, "11051": {"0": "green", "1": "1", "2": "3"}, "11052": {"0": "no", "1": "left", "2": "up"}, "11053": {"0": "cube", "1": "rubber", "2": "cylinder"}, "11054": {"0": "yes", "1": "yes", "2": "yes"}, "11055": {"0": "rubber", "1": "purple", "2": "cyan"}, "11056": {"0": "sphere", "1": "cyan", "2": "1"}, "11057": {"0": "green", "1": "no", "2": "cube"}, "11058": {"0": "gray", "1": "yes", "2": "1"}, "11059": {"0": "error", "1": "up", "2": "brown"}, "11060": {"0": "yellow", "1": "2", "2": "sphere"}, "11061": {"0": "cube", "1": "gray", "2": "1"}, "11062": {"0": "1", "1": "sphere", "2": "error"}, "11063": {"0": "yes", "1": "4", "2": "right"}, "11064": {"0": "0", "1": "purple", "2": "rubber"}, "11065": {"0": "rubber", "1": "metal", "2": "error"}, "11066": {"0": "sphere", "1": "metal", "2": "no"}, "11067": {"0": "no", "1": "yes", "2": "2"}, "11068": {"0": "0", "1": "yes", "2": "3"}, "11069": {"0": "2", "1": "1", "2": "cube"}, "11070": {"0": "blue", "1": "yes", "2": "2"}, "11071": {"0": "cylinder", "1": "1", "2": "cube"}, "11072": {"0": "purple", "1": "no", "2": "no"}, "11073": {"0": "yes", "1": "rubber", "2": "1"}, "11074": {"0": "0", "1": "metal", "2": "sphere"}, "11075": {"0": "brown", "1": "cyan", "2": "cylinder"}, "11076": {"0": "yellow", "1": "yes", "2": "0"}, "11077": {"0": "0", "1": "cube"}, "11078": {"0": "rubber", "1": "purple", "2": "1"}, "11079": {"0": "metal", "1": "yes", "2": "rubber"}, "11080": {"0": "down", "1": "green", "2": "0"}, "11081": {"0": "4", "1": "red", "2": "rubber"}, "11082": {"0": "1", "1": "error", "2": "rubber"}, "11083": {"0": "0", "1": "cube", "2": "cylinder"}, "11084": {"0": "rubber", "1": "sphere", "2": "0"}, "11085": {"0": "0", "1": "cylinder", "2": "error"}, "11086": {"0": "yes", "1": "yes", "2": "yes"}, "11087": {"0": "error", "1": "error", "2": "1"}, "11088": {"0": "left", "1": "brown", "2": "4"}, "11089": {"0": "sphere", "1": "1", "2": "yellow"}, "11090": {"0": "yellow", "1": "0", "2": "2"}, "11091": {"0": "yellow", "1": "yes", "2": "brown"}, "11092": {"0": "3", "1": "metal", "2": "metal"}, "11093": {"0": "metal", "1": "yes", "2": "brown"}, "11094": {"0": "1", "1": "sphere", "2": "green"}, "11095": {"0": "rubber", "1": "cyan", "2": "cylinder"}, "11096": {"0": "error", "1": "green", "2": "cube"}, "11097": {"0": "rubber", "1": "yes", "2": "cube"}, "11098": {"0": "sphere", "1": "yellow", "2": "cube"}, "11099": {"0": "red", "1": "no", "2": "error"}, "11100": {"0": "cube", "1": "cylinder", "2": "blue"}, "11101": {"0": "no", "1": "0", "2": "sphere"}, "11102": {"0": "yes", "1": "red", "2": "down"}, "11103": {"0": "2", "1": "blue", "2": "blue"}, "11104": {"0": "no", "1": "no", "2": "cube"}, "11105": {"0": "3", "1": "no", "2": "right"}, "11106": {"0": "yellow", "1": "cube", "2": "sphere"}, "11107": {"0": "no", "1": "error", "2": "cyan"}, "11108": {"0": "up", "1": "0"}, "11109": {"0": "metal", "1": "1", "2": "4"}, "11110": {"0": "red", "1": "yes", "2": "green"}, "11111": {"0": "purple", "1": "error", "2": "rubber"}, "11112": {"0": "1", "1": "metal", "2": "gray"}, "11113": {"0": "metal"}, "11114": {"0": "metal", "1": "1", "2": "red"}, "11115": {"0": "sphere", "1": "sphere", "2": "4"}, "11116": {"0": "0", "1": "yellow", "2": "blue"}, "11117": {"0": "cylinder", "1": "yellow", "2": "metal"}, "11118": {"0": "sphere", "1": "0", "2": "error"}, "11119": {"0": "red", "1": "1", "2": "cube"}, "11120": {"0": "rubber", "1": "green", "2": "no"}, "11121": {"0": "2", "1": "cyan", "2": "purple"}, "11122": {"0": "cylinder", "1": "1", "2": "green"}, "11123": {"0": "sphere", "1": "0", "2": "no"}, "11124": {"0": "0", "1": "0", "2": "no"}, "11125": {"0": "no", "1": "metal", "2": "no"}, "11126": {"0": "3", "1": "rubber", "2": "left"}, "11127": {"0": "sphere", "1": "1"}, "11128": {"0": "rubber", "1": "rubber", "2": "2"}, "11129": {"0": "rubber", "1": "yes", "2": "right"}, "11130": {"0": "no", "1": "purple", "2": "cylinder"}, "11131": {"0": "2", "1": "0", "2": "sphere"}, "11132": {"0": "brown", "1": "metal", "2": "error"}, "11133": {"0": "error", "1": "no", "2": "blue"}, "11134": {"0": "no", "1": "cylinder", "2": "0"}, "11135": {"0": "blue", "1": "1", "2": "1"}, "11136": {"0": "green", "1": "cube", "2": "0"}, "11137": {"0": "1", "1": "metal", "2": "rubber"}, "11138": {"0": "brown", "1": "purple"}, "11139": {"0": "gray", "1": "up", "2": "cylinder"}, "11140": {"0": "gray", "1": "no", "2": "error"}, "11141": {"0": "rubber", "1": "yes", "2": "rubber"}, "11142": {"0": "green", "1": "sphere", "2": "brown"}, "11143": {"0": "error", "1": "error", "2": "0"}, "11144": {"0": "0", "1": "cylinder", "2": "red"}, "11145": {"0": "1", "1": "0", "2": "error"}, "11146": {"0": "yes", "1": "no", "2": "0"}, "11147": {"0": "yes", "1": "yes", "2": "green"}, "11148": {"0": "no", "1": "right", "2": "no"}, "11149": {"0": "metal", "1": "0", "2": "metal"}, "11150": {"0": "metal", "1": "rubber", "2": "2"}, "11151": {"0": "1", "1": "0", "2": "yes"}, "11152": {"0": "0", "1": "yes", "2": "cube"}, "11153": {"0": "error", "1": "1", "2": "cylinder"}, "11154": {"0": "rubber", "1": "yellow", "2": "cyan"}, "11155": {"0": "no", "1": "rubber", "2": "cylinder"}, "11156": {"0": "1", "1": "1", "2": "cyan"}, "11157": {"0": "no", "1": "no", "2": "red"}, "11158": {"0": "red", "1": "green", "2": "0"}, "11159": {"0": "cyan", "1": "sphere", "2": "error"}, "11160": {"0": "blue", "1": "yes", "2": "sphere"}, "11161": {"0": "1", "1": "down", "2": "yes"}, "11162": {"0": "0", "1": "yellow", "2": "2"}, "11163": {"0": "2", "1": "yes", "2": "rubber"}, "11164": {"0": "metal", "1": "2", "2": "cube"}, "11165": {"0": "4", "1": "brown", "2": "no"}, "11166": {"0": "rubber", "1": "gray", "2": "4"}, "11167": {"0": "1", "1": "no", "2": "cube"}, "11168": {"0": "yes", "1": "no", "2": "2"}, "11169": {"0": "3", "1": "blue", "2": "yes"}, "11170": {"0": "sphere", "1": "error", "2": "no", "3": "cube", "4": "sphere", "5": "3", "6": "sphere", "7": "sphere", "8": "1", "9": "yes"}, "11171": {"0": "metal", "1": "brown", "2": "1"}, "11172": {"0": "green", "1": "2", "2": "0"}, "11173": {"0": "rubber", "1": "yes", "2": "no"}, "11174": {"0": "yes", "1": "cylinder", "2": "0"}, "11175": {"0": "cyan", "1": "error", "2": "no", "3": "error", "4": "error", "5": "1", "6": "cylinder", "7": "error", "8": "4", "9": "2"}, "11176": {"0": "up", "1": "cube", "2": "gray"}, "11177": {"0": "cylinder", "1": "yes", "2": "no"}, "11178": {"0": "gray", "1": "red", "2": "1"}, "11179": {"0": "gray", "1": "1", "2": "purple"}, "11180": {"0": "cube", "1": "metal", "2": "no"}, "11181": {"0": "left", "1": "blue", "2": "yes"}, "11182": {"0": "1", "1": "sphere", "2": "error"}, "11183": {"0": "yes", "1": "1", "2": "error"}, "11184": {"0": "cylinder", "1": "cube", "2": "cube"}, "11185": {"0": "cylinder", "1": "purple", "2": "1"}, "11186": {"0": "sphere", "1": "blue", "2": "yes"}, "11187": {"0": "yes", "1": "no"}, "11188": {"0": "yes", "1": "4", "2": "left"}, "11189": {"0": "purple", "1": "purple", "2": "cyan"}, "11190": {"0": "no", "1": "yes", "2": "metal"}, "11191": {"0": "yellow", "1": "0", "2": "2"}, "11192": {"0": "blue", "1": "0", "2": "yes"}, "11193": {"0": "purple", "1": "error", "2": "no", "3": "error", "4": "error", "5": "0", "6": "right", "7": "cylinder", "8": "yes", "9": "2"}, "11194": {"0": "cube", "1": "down", "2": "no"}, "11195": {"0": "error", "1": "1", "2": "no"}, "11196": {"0": "0", "1": "cylinder"}, "11197": {"0": "left", "1": "sphere", "2": "down"}, "11198": {"0": "rubber", "1": "sphere", "2": "no"}, "11199": {"0": "cylinder", "1": "sphere", "2": "yellow"}, "11200": {"0": "0", "1": "no", "2": "0"}, "11201": {"0": "cylinder", "1": "metal", "2": "no"}, "11202": {"0": "blue", "1": "rubber", "2": "cube"}, "11203": {"0": "red", "1": "cyan", "2": "cylinder"}, "11204": {"0": "0", "1": "yes"}, "11205": {"0": "gray", "1": "error", "2": "cylinder"}, "11206": {"0": "yes", "1": "1", "2": "yes"}, "11207": {"0": "error", "1": "2", "2": "no"}, "11208": {"0": "cube", "1": "metal", "2": "gray"}, "11209": {"0": "0", "1": "down", "2": "rubber"}, "11210": {"0": "3", "1": "yellow", "2": "cube"}, "11211": {"0": "sphere", "1": "2", "2": "blue"}, "11212": {"0": "no", "1": "no", "2": "blue"}, "11213": {"0": "cylinder", "1": "no", "2": "metal"}, "11214": {"0": "yellow", "1": "cylinder", "2": "0"}, "11215": {"0": "3", "1": "rubber", "2": "left"}, "11216": {"0": "1", "1": "no", "2": "yellow"}, "11217": {"0": "rubber", "1": "cube", "2": "no"}, "11218": {"0": "sphere", "1": "cube", "2": "0"}, "11219": {"0": "rubber", "1": "sphere", "2": "0"}, "11220": {"0": "yes", "1": "yes", "2": "yes"}, "11221": {"0": "sphere", "1": "cube", "2": "no"}, "11222": {"0": "rubber", "1": "rubber", "2": "no"}, "11223": {"0": "metal", "1": "error", "2": "error"}, "11224": {"0": "0", "1": "error", "2": "brown"}, "11225": {"0": "yellow", "1": "cylinder", "2": "cylinder"}, "11226": {"0": "yes", "1": "cyan", "2": "yes"}, "11227": {"0": "blue", "1": "sphere", "2": "gray"}, "11228": {"0": "cyan", "1": "metal", "2": "yes"}, "11229": {"0": "0", "1": "cylinder", "2": "cylinder"}, "11230": {"0": "metal", "1": "1", "2": "green"}, "11231": {"0": "cube", "1": "error"}, "11232": {"0": "rubber", "1": "yes", "2": "brown"}, "11233": {"0": "metal", "1": "brown", "2": "0"}, "11234": {"0": "cylinder", "1": "no", "2": "rubber"}, "11235": {"0": "left", "1": "1", "2": "yes"}, "11236": {"0": "error", "1": "error", "2": "sphere"}, "11237": {"0": "0", "1": "rubber", "2": "0"}, "11238": {"0": "up", "1": "down", "2": "sphere"}, "11239": {"0": "no", "1": "cylinder"}, "11240": {"0": "no", "1": "no", "2": "cylinder"}, "11241": {"0": "brown", "1": "2", "2": "metal"}, "11242": {"0": "green", "1": "0", "2": "no"}, "11243": {"0": "error", "1": "rubber"}, "11244": {"0": "error", "1": "0", "2": "metal"}, "11245": {"0": "cube", "1": "cube", "2": "rubber"}, "11246": {"0": "yes", "1": "blue", "2": "no"}, "11247": {"0": "yes", "1": "0", "2": "error"}, "11248": {"0": "error", "1": "rubber", "2": "rubber"}, "11249": {"0": "error", "1": "green", "2": "0"}, "11250": {"0": "sphere", "1": "sphere", "2": "no"}, "11251": {"0": "3", "1": "cylinder", "2": "metal"}, "11252": {"0": "metal", "1": "cube", "2": "sphere"}, "11253": {"0": "gray", "1": "cylinder", "2": "no"}, "11254": {"0": "metal", "1": "no", "2": "cube"}, "11255": {"0": "yes", "1": "purple", "2": "2"}, "11256": {"0": "sphere", "1": "metal", "2": "0"}, "11257": {"0": "1", "1": "rubber", "2": "green"}, "11258": {"0": "yellow", "1": "0", "2": "metal"}, "11259": {"0": "gray", "1": "gray", "2": "2"}, "11260": {"0": "0", "1": "sphere", "2": "no"}, "11261": {"0": "no", "1": "rubber", "2": "metal"}, "11262": {"0": "no", "1": "yellow", "2": "yellow"}, "11263": {"0": "yes", "1": "2", "2": "no"}, "11264": {"0": "metal", "1": "sphere", "2": "down"}, "11265": {"0": "1", "1": "cylinder"}, "11266": {"0": "1", "1": "no"}, "11267": {"0": "rubber", "1": "brown", "2": "metal"}, "11268": {"0": "purple", "1": "brown", "2": "error"}, "11269": {"0": "red", "1": "blue", "2": "1", "3": "1", "4": "error", "5": "right", "6": "yes", "7": "no"}, "11270": {"0": "cube", "1": "cylinder", "2": "yes"}, "11271": {"0": "rubber", "1": "cylinder", "2": "no"}, "11272": {"0": "blue", "1": "no", "2": "metal"}, "11273": {"0": "no", "1": "0", "2": "metal"}, "11274": {"0": "cube", "1": "error", "2": "0"}, "11275": {"0": "green", "1": "yellow", "2": "no"}, "11276": {"0": "0", "1": "cylinder", "2": "cube"}, "11277": {"0": "metal", "1": "cyan", "2": "1"}, "11278": {"0": "yes", "1": "yes", "2": "0"}, "11279": {"0": "error", "1": "left", "2": "left"}, "11280": {"0": "brown", "1": "sphere", "2": "yes"}, "11281": {"0": "no", "1": "error", "2": "cylinder"}, "11282": {"0": "cube", "1": "cylinder"}, "11283": {"0": "gray", "1": "metal", "2": "cyan"}, "11284": {"0": "yes", "1": "metal", "2": "no"}, "11285": {"0": "yes", "1": "no"}, "11286": {"0": "cylinder", "1": "cylinder", "2": "purple"}, "11287": {"0": "1", "1": "yes", "2": "0"}, "11288": {"0": "1", "1": "cyan", "2": "error"}, "11289": {"0": "purple", "1": "yes", "2": "error"}, "11290": {"0": "rubber", "1": "cube", "2": "no"}, "11291": {"0": "cyan", "1": "rubber", "2": "red"}, "11292": {"0": "0", "1": "rubber", "2": "red"}, "11293": {"0": "no", "1": "0", "2": "blue"}, "11294": {"0": "cylinder", "1": "1", "2": "purple"}, "11295": {"0": "rubber", "1": "red", "2": "rubber"}, "11296": {"0": "0", "1": "error", "2": "yes"}, "11297": {"0": "sphere", "1": "rubber", "2": "sphere"}, "11298": {"0": "error", "1": "error", "2": "error"}, "11299": {"0": "yes", "1": "metal", "2": "error"}, "11300": {"0": "red", "1": "red", "2": "3"}, "11301": {"0": "no", "1": "1", "2": "cylinder"}, "11302": {"0": "no", "1": "error", "2": "yes"}, "11303": {"0": "0", "1": "yes", "2": "error"}, "11304": {"0": "1", "1": "2", "2": "no"}, "11305": {"0": "rubber", "1": "yes", "2": "sphere"}, "11306": {"0": "yes", "1": "no", "2": "cube"}, "11307": {"0": "blue", "1": "no", "2": "metal"}, "11308": {"0": "1", "1": "1", "2": "rubber"}, "11309": {"0": "1", "1": "rubber", "2": "brown"}, "11310": {"0": "cylinder", "1": "cyan", "2": "no", "3": "yellow", "4": "sphere", "5": "2", "6": "metal", "7": "metal", "8": "no", "9": "yes"}, "11311": {"0": "yellow", "1": "red", "2": "metal"}, "11312": {"0": "red", "1": "no", "2": "purple"}, "11313": {"0": "0", "1": "rubber", "2": "metal"}, "11314": {"0": "1", "1": "sphere", "2": "3"}, "11315": {"0": "yes", "1": "error", "2": "rubber"}, "11316": {"0": "blue", "1": "1", "2": "no"}, "11317": {"0": "brown", "1": "cylinder", "2": "no"}, "11318": {"0": "yes", "1": "down", "2": "cylinder"}, "11319": {"0": "no", "1": "gray", "2": "0"}, "11320": {"0": "yes", "1": "no", "2": "rubber"}, "11321": {"0": "error", "1": "cube"}, "11322": {"0": "1", "1": "red", "2": "green"}, "11323": {"0": "cylinder", "1": "down", "2": "blue"}, "11324": {"0": "0", "1": "2", "2": "0"}, "11325": {"0": "error", "1": "yes", "2": "cube"}, "11326": {"0": "yes", "1": "cylinder", "2": "down"}, "11327": {"0": "yes", "1": "metal", "2": "rubber"}, "11328": {"0": "sphere", "1": "gray", "2": "gray"}, "11329": {"0": "sphere", "1": "3", "2": "2"}, "11330": {"0": "green", "1": "rubber", "2": "3"}, "11331": {"0": "0", "1": "rubber", "2": "red"}, "11332": {"0": "rubber", "1": "metal", "2": "metal"}, "11333": {"0": "yes", "1": "1", "2": "error"}, "11334": {"0": "rubber", "1": "sphere", "2": "1"}, "11335": {"0": "down", "1": "yes", "2": "right"}, "11336": {"0": "yellow", "1": "yellow", "2": "yes"}, "11337": {"0": "cylinder", "1": "error", "2": "right"}, "11338": {"0": "sphere", "1": "error", "2": "2"}, "11339": {"0": "error", "1": "0", "2": "cyan"}, "11340": {"0": "cyan", "1": "2", "2": "green"}, "11341": {"0": "green", "1": "rubber", "2": "yes", "3": "1", "4": "cube", "5": "blue", "6": "no", "7": "yes"}, "11342": {"0": "yes", "1": "rubber", "2": "no"}, "11343": {"0": "1", "1": "rubber", "2": "cylinder"}, "11344": {"0": "brown", "1": "gray", "2": "gray"}, "11345": {"0": "yes", "1": "green", "2": "gray"}, "11346": {"0": "0", "1": "rubber", "2": "blue"}, "11347": {"0": "yes", "1": "error", "2": "sphere"}, "11348": {"0": "no", "1": "purple", "2": "green"}, "11349": {"0": "blue", "1": "sphere", "2": "0"}, "11350": {"0": "cube", "1": "0", "2": "metal"}, "11351": {"0": "blue", "1": "error", "2": "cube"}, "11352": {"0": "green", "1": "error", "2": "0"}, "11353": {"0": "green", "1": "cylinder", "2": "yes", "3": "sphere", "4": "blue", "5": "0", "6": "error", "7": "error", "8": "yes", "9": "0"}, "11354": {"0": "2", "1": "purple", "2": "2"}, "11355": {"0": "0", "1": "rubber", "2": "cylinder"}, "11356": {"0": "yes", "1": "0", "2": "no"}, "11357": {"0": "0", "1": "cyan", "2": "no"}, "11358": {"0": "1", "1": "purple", "2": "1"}, "11359": {"0": "cylinder", "1": "gray", "2": "1"}, "11360": {"0": "brown", "1": "1"}, "11361": {"0": "error", "1": "yes", "2": "metal"}, "11362": {"0": "left", "1": "1", "2": "no"}, "11363": {"0": "cube", "1": "metal", "2": "2"}, "11364": {"0": "gray", "1": "sphere", "2": "rubber"}, "11365": {"0": "1", "1": "1", "2": "no"}, "11366": {"0": "2", "1": "yes", "2": "rubber"}, "11367": {"0": "green", "1": "green", "2": "no"}, "11368": {"0": "cylinder", "1": "left", "2": "cube"}, "11369": {"0": "cube", "1": "0", "2": "blue"}, "11370": {"0": "error", "1": "error"}, "11371": {"0": "purple", "1": "yes", "2": "yellow"}, "11372": {"0": "gray", "1": "brown", "2": "cylinder"}, "11373": {"0": "1", "1": "green", "2": "metal"}, "11374": {"0": "1", "1": "brown", "2": "no"}, "11375": {"0": "rubber", "1": "1", "2": "right"}, "11376": {"0": "no", "1": "yellow", "2": "0"}, "11377": {"0": "cylinder", "1": "cylinder", "2": "sphere"}, "11378": {"0": "metal", "1": "no", "2": "no"}, "11379": {"0": "sphere", "1": "0", "2": "2"}, "11380": {"0": "no", "1": "2", "2": "blue"}, "11381": {"0": "right", "1": "3", "2": "2"}, "11382": {"0": "green", "1": "error", "2": "cylinder"}, "11383": {"0": "cube", "1": "0", "2": "rubber"}, "11384": {"0": "no", "1": "cyan", "2": "0"}, "11385": {"0": "4", "1": "rubber", "2": "metal"}, "11386": {"0": "rubber", "1": "0", "2": "sphere"}, "11387": {"0": "2", "1": "yellow", "2": "metal"}, "11388": {"0": "no", "1": "rubber", "2": "1"}, "11389": {"0": "blue", "1": "cube", "2": "sphere"}, "11390": {"0": "brown", "1": "yes"}, "11391": {"0": "gray", "1": "0", "2": "purple"}, "11392": {"0": "0", "1": "sphere", "2": "1"}, "11393": {"0": "gray", "1": "error", "2": "rubber"}, "11394": {"0": "brown", "1": "cube", "2": "no"}, "11395": {"0": "no", "1": "error", "2": "error"}, "11396": {"0": "blue", "1": "2", "2": "up"}, "11397": {"0": "cyan", "1": "sphere", "2": "yes", "3": "cylinder", "4": "sphere", "5": "0", "6": "error", "7": "error", "8": "no", "9": "yes"}, "11398": {"0": "yellow", "1": "error", "2": "0"}, "11399": {"0": "green", "1": "cube", "2": "yes", "3": "error", "4": "error", "5": "0", "6": "error", "7": "blue", "8": "0", "9": "0"}, "11400": {"0": "right", "1": "yes", "2": "sphere"}, "11401": {"0": "sphere", "1": "no"}, "11402": {"0": "cube", "1": "blue", "2": "cyan"}, "11403": {"0": "1", "1": "metal", "2": "error"}, "11404": {"0": "metal", "1": "green"}, "11405": {"0": "purple", "1": "no", "2": "red"}, "11406": {"0": "rubber", "1": "0", "2": "error"}, "11407": {"0": "yellow", "1": "error", "2": "1"}, "11408": {"0": "brown", "1": "1", "2": "metal"}, "11409": {"0": "0", "1": "no", "2": "error"}, "11410": {"0": "red", "1": "down", "2": "blue"}, "11411": {"0": "1", "1": "cube", "2": "2"}, "11412": {"0": "no", "1": "sphere", "2": "brown"}, "11413": {"0": "1", "1": "cube", "2": "cyan"}, "11414": {"0": "yellow", "1": "rubber", "2": "metal"}, "11415": {"0": "metal", "1": "sphere", "2": "no"}, "11416": {"0": "metal", "1": "gray", "2": "purple"}, "11417": {"0": "no", "1": "brown", "2": "yellow"}, "11418": {"0": "error", "1": "error", "2": "no"}, "11419": {"0": "0", "1": "brown", "2": "brown"}, "11420": {"0": "rubber", "1": "blue"}, "11421": {"0": "0", "1": "no", "2": "cube"}, "11422": {"0": "metal", "1": "yes", "2": "error"}, "11423": {"0": "rubber", "1": "error", "2": "3"}, "11424": {"0": "3", "1": "no", "2": "yes"}, "11425": {"0": "sphere", "1": "rubber", "2": "rubber"}, "11426": {"0": "rubber", "1": "0", "2": "purple"}, "11427": {"0": "left", "1": "brown", "2": "sphere"}, "11428": {"0": "no", "1": "sphere", "2": "green"}, "11429": {"0": "rubber", "1": "error", "2": "1"}, "11430": {"0": "yes", "1": "cube", "2": "1"}, "11431": {"0": "2", "1": "metal", "2": "1"}, "11432": {"0": "purple", "1": "sphere", "2": "cube"}, "11433": {"0": "metal", "1": "1", "2": "error"}, "11434": {"0": "no", "1": "sphere", "2": "green"}, "11435": {"0": "gray", "1": "rubber", "2": "gray"}, "11436": {"0": "brown", "1": "rubber", "2": "yes"}, "11437": {"0": "yes", "1": "green"}, "11438": {"0": "rubber", "1": "error", "2": "0"}, "11439": {"0": "cube", "1": "3", "2": "metal"}, "11440": {"0": "error", "1": "metal", "2": "error"}, "11441": {"0": "1", "1": "yellow", "2": "brown"}, "11442": {"0": "0", "1": "brown", "2": "brown"}, "11443": {"0": "cube", "1": "0", "2": "cube"}, "11444": {"0": "brown", "1": "2", "2": "rubber"}, "11445": {"0": "rubber", "1": "0", "2": "cube"}, "11446": {"0": "brown", "1": "metal", "2": "no"}, "11447": {"0": "cube", "1": "cube", "2": "yes"}, "11448": {"0": "cylinder", "1": "metal", "2": "gray"}, "11449": {"0": "cylinder", "1": "brown"}, "11450": {"0": "gray", "1": "yes", "2": "cyan"}, "11451": {"0": "down", "1": "no", "2": "yes"}, "11452": {"0": "metal", "1": "cube", "2": "yes", "3": "cylinder", "4": "sphere", "5": "0", "6": "down", "7": "blue", "8": "yes", "9": "yes"}, "11453": {"0": "green", "1": "1", "2": "cylinder"}, "11454": {"0": "cube", "1": "0", "2": "brown"}, "11455": {"0": "yes", "1": "green", "2": "no"}, "11456": {"0": "cylinder", "1": "cylinder", "2": "yes"}, "11457": {"0": "no", "1": "rubber", "2": "sphere"}, "11458": {"0": "error", "1": "error", "2": "no", "3": "2", "4": "error", "5": "yellow", "6": "2", "7": "1"}, "11459": {"0": "rubber", "1": "rubber", "2": "purple"}, "11460": {"0": "cylinder", "1": "brown", "2": "brown"}, "11461": {"0": "yes", "1": "1"}, "11462": {"0": "no", "1": "0", "2": "no"}, "11463": {"0": "0", "1": "cube", "2": "cylinder"}, "11464": {"0": "cylinder", "1": "yes", "2": "sphere"}, "11465": {"0": "error", "1": "yes", "2": "gray"}, "11466": {"0": "metal", "1": "yes", "2": "metal"}, "11467": {"0": "cylinder", "1": "1", "2": "0"}, "11468": {"0": "error", "1": "rubber", "2": "rubber"}, "11469": {"0": "1", "1": "metal", "2": "purple"}, "11470": {"0": "purple", "1": "yes", "2": "green"}, "11471": {"0": "1", "1": "metal", "2": "0"}, "11472": {"0": "rubber", "1": "metal", "2": "rubber"}, "11473": {"0": "cylinder", "1": "cube", "2": "no"}, "11474": {"0": "red", "1": "cube", "2": "no"}, "11475": {"0": "cylinder", "1": "0", "2": "2"}, "11476": {"0": "blue", "1": "0", "2": "cube"}, "11477": {"0": "yes", "1": "no", "2": "right"}, "11478": {"0": "error", "1": "cyan", "2": "no", "3": "error", "4": "error", "5": "1", "6": "sphere", "7": "metal", "8": "no", "9": "no"}, "11479": {"0": "yellow", "1": "rubber", "2": "cylinder"}, "11480": {"0": "yes", "1": "cylinder"}, "11481": {"0": "yes", "1": "cylinder", "2": "0"}, "11482": {"0": "4", "1": "metal", "2": "no"}, "11483": {"0": "cyan", "1": "sphere", "2": "cylinder"}, "11484": {"0": "metal", "1": "error", "2": "metal"}, "11485": {"0": "cube", "1": "2", "2": "cyan"}, "11486": {"0": "1", "1": "rubber", "2": "1"}, "11487": {"0": "yes", "1": "sphere", "2": "cylinder"}, "11488": {"0": "sphere", "1": "cylinder", "2": "no", "3": "cube", "4": "cylinder", "5": "0", "6": "brown", "7": "cube", "8": "yes", "9": "no"}, "11489": {"0": "no", "1": "green", "2": "blue"}, "11490": {"0": "rubber", "1": "metal", "2": "no"}, "11491": {"0": "rubber", "1": "0", "2": "purple"}, "11492": {"0": "yellow", "1": "yes", "2": "cyan"}, "11493": {"0": "brown", "1": "sphere", "2": "cyan"}, "11494": {"0": "cyan", "1": "green", "2": "0"}, "11495": {"0": "cube", "1": "no", "2": "brown"}, "11496": {"0": "error", "1": "no", "2": "metal"}, "11497": {"0": "no", "1": "cylinder", "2": "0"}, "11498": {"0": "rubber", "1": "blue", "2": "no"}, "11499": {"0": "cube", "1": "down", "2": "blue"}, "11500": {"0": "blue", "1": "2", "2": "blue"}, "11501": {"0": "yellow", "1": "2", "2": "error"}, "11502": {"0": "cube", "1": "yes", "2": "metal"}, "11503": {"0": "2", "1": "5", "2": "red"}, "11504": {"0": "down", "1": "no", "2": "rubber"}, "11505": {"0": "cylinder", "1": "no", "2": "cylinder"}, "11506": {"0": "0", "1": "yes"}, "11507": {"0": "up", "1": "yes", "2": "purple"}, "11508": {"0": "1", "1": "sphere", "2": "2"}, "11509": {"0": "error", "1": "error", "2": "gray"}, "11510": {"0": "0", "1": "yes", "2": "error"}, "11511": {"0": "yes", "1": "left", "2": "1"}, "11512": {"0": "cyan", "1": "yes", "2": "1"}, "11513": {"0": "error", "1": "cube", "2": "purple"}, "11514": {"0": "no", "1": "brown"}, "11515": {"0": "cylinder", "1": "down", "2": "cyan"}, "11516": {"0": "error", "1": "rubber", "2": "cyan"}, "11517": {"0": "rubber", "1": "no", "2": "0"}, "11518": {"0": "cube", "1": "0", "2": "sphere"}, "11519": {"0": "metal", "1": "no", "2": "no"}, "11520": {"0": "metal", "1": "left", "2": "green"}, "11521": {"0": "metal", "1": "3", "2": "1"}, "11522": {"0": "0", "1": "error", "2": "red"}, "11523": {"0": "rubber", "1": "metal", "2": "sphere"}, "11524": {"0": "0", "1": "3", "2": "brown"}, "11525": {"0": "no", "1": "blue", "2": "0"}, "11526": {"0": "yes", "1": "0", "2": "cyan"}, "11527": {"0": "right", "1": "yes", "2": "gray"}, "11528": {"0": "cyan", "1": "red", "2": "yes"}, "11529": {"0": "brown", "1": "cylinder", "2": "gray"}, "11530": {"0": "error", "1": "blue", "2": "brown"}, "11531": {"0": "cube", "1": "4", "2": "error"}, "11532": {"0": "cyan", "1": "green", "2": "1"}, "11533": {"0": "cube", "1": "0", "2": "left"}, "11534": {"0": "gray", "1": "green", "2": "green"}, "11535": {"0": "yes", "1": "error", "2": "metal"}, "11536": {"0": "2", "1": "cube", "2": "sphere"}, "11537": {"0": "metal", "1": "no", "2": "sphere"}, "11538": {"0": "2", "1": "red", "2": "red"}, "11539": {"0": "metal", "1": "green", "2": "cyan"}, "11540": {"0": "cylinder", "1": "error", "2": "1"}, "11541": {"0": "yellow", "1": "yes", "2": "sphere"}, "11542": {"0": "purple", "1": "yes", "2": "0"}, "11543": {"0": "yellow", "1": "blue", "2": "sphere"}, "11544": {"0": "red", "1": "0", "2": "no"}, "11545": {"0": "rubber", "1": "0", "2": "red"}, "11546": {"0": "gray", "1": "sphere", "2": "blue"}, "11547": {"0": "3", "1": "cube", "2": "0"}, "11548": {"0": "no", "1": "no", "2": "yellow"}, "11549": {"0": "yes", "1": "metal", "2": "0"}, "11550": {"0": "2", "1": "right", "2": "yes"}, "11551": {"0": "no", "1": "cube", "2": "error"}, "11552": {"0": "2", "1": "no"}, "11553": {"0": "metal", "1": "no", "2": "error"}, "11554": {"0": "cube", "1": "cyan", "2": "sphere"}, "11555": {"0": "no", "1": "cube", "2": "metal"}, "11556": {"0": "left", "1": "no", "2": "1"}, "11557": {"0": "1", "1": "right", "2": "no"}, "11558": {"0": "cyan", "1": "2", "2": "1"}, "11559": {"0": "down", "1": "no", "2": "cylinder"}, "11560": {"0": "red", "1": "metal", "2": "blue"}, "11561": {"0": "yes", "1": "sphere", "2": "green"}, "11562": {"0": "metal", "1": "metal", "2": "1"}, "11563": {"0": "cyan", "1": "rubber", "2": "sphere"}, "11564": {"0": "purple", "1": "rubber", "2": "metal"}, "11565": {"0": "no", "1": "sphere", "2": "yes"}, "11566": {"0": "no", "1": "cube", "2": "error"}, "11567": {"0": "no", "1": "purple", "2": "cube"}, "11568": {"0": "rubber", "1": "red", "2": "yes"}, "11569": {"0": "metal", "1": "no", "2": "no"}, "11570": {"0": "yes", "1": "gray", "2": "error"}, "11571": {"0": "rubber", "1": "2", "2": "metal"}, "11572": {"0": "no", "1": "2", "2": "sphere"}, "11573": {"0": "error", "1": "metal", "2": "no"}, "11574": {"0": "error", "1": "1", "2": "error"}, "11575": {"0": "rubber", "1": "rubber", "2": "0"}, "11576": {"0": "rubber", "1": "1", "2": "no"}, "11577": {"0": "no", "1": "rubber"}, "11578": {"0": "gray", "1": "error", "2": "metal"}, "11579": {"0": "0", "1": "cylinder", "2": "rubber"}, "11580": {"0": "cylinder", "1": "sphere", "2": "0"}, "11581": {"0": "0", "1": "error", "2": "green"}, "11582": {"0": "sphere", "1": "green", "2": "2"}, "11583": {"0": "cylinder", "1": "brown", "2": "yes", "3": "2", "4": "cylinder", "5": "error", "6": "no", "7": "yes"}, "11584": {"0": "yellow", "1": "rubber", "2": "yes"}, "11585": {"0": "1", "1": "no", "2": "error"}, "11586": {"0": "no", "1": "green", "2": "0"}, "11587": {"0": "rubber", "1": "cylinder", "2": "metal"}, "11588": {"0": "gray", "1": "0", "2": "no"}, "11589": {"0": "1", "1": "error", "2": "error"}, "11590": {"0": "metal", "1": "gray", "2": "0"}, "11591": {"0": "yes", "1": "cylinder", "2": "1"}, "11592": {"0": "0", "1": "sphere", "2": "rubber"}, "11593": {"0": "0", "1": "1", "2": "rubber"}, "11594": {"0": "no", "1": "cylinder", "2": "green"}, "11595": {"0": "error", "1": "metal", "2": "error"}, "11596": {"0": "yellow", "1": "0", "2": "yes"}, "11597": {"0": "yes", "1": "0", "2": "cube"}, "11598": {"0": "yes", "1": "metal", "2": "yes"}, "11599": {"0": "cyan", "1": "2", "2": "green"}, "11600": {"0": "yes", "1": "0", "2": "metal"}, "11601": {"0": "gray", "1": "1", "2": "yes"}, "11602": {"0": "1", "1": "cube"}, "11603": {"0": "cylinder", "1": "sphere", "2": "1", "3": "error", "4": "error", "5": "0", "6": "green", "7": "blue", "8": "0", "9": "1"}, "11604": {"0": "cube", "1": "metal", "2": "1"}, "11605": {"0": "green", "1": "yes", "2": "cylinder"}, "11606": {"0": "cube", "1": "metal", "2": "red"}, "11607": {"0": "sphere", "1": "up", "2": "green"}, "11608": {"0": "cube", "1": "brown", "2": "2"}, "11609": {"0": "no", "1": "yellow", "2": "error"}, "11610": {"0": "red", "1": "yellow", "2": "1"}, "11611": {"0": "yellow", "1": "yes", "2": "cylinder"}, "11612": {"0": "sphere", "1": "1", "2": "0"}, "11613": {"0": "0", "1": "yellow", "2": "cube"}, "11614": {"0": "yellow", "1": "rubber", "2": "rubber"}, "11615": {"0": "cyan", "1": "rubber", "2": "error"}, "11616": {"0": "yellow", "1": "no", "2": "red"}, "11617": {"0": "cyan", "1": "metal", "2": "rubber"}, "11618": {"0": "gray", "1": "error", "2": "cyan"}, "11619": {"0": "metal", "1": "no", "2": "rubber"}, "11620": {"0": "0", "1": "metal", "2": "0"}, "11621": {"0": "cylinder", "1": "rubber"}, "11622": {"0": "yellow", "1": "3", "2": "cylinder"}, "11623": {"0": "gray", "1": "rubber", "2": "yes"}, "11624": {"0": "down", "1": "metal", "2": "rubber"}, "11625": {"0": "cube", "1": "metal", "2": "down"}, "11626": {"0": "yellow", "1": "2", "2": "no"}, "11627": {"0": "sphere", "1": "yes", "2": "gray"}, "11628": {"0": "sphere", "1": "error", "2": "no", "3": "1", "4": "right", "5": "red", "6": "1", "7": "1"}, "11629": {"0": "error", "1": "metal", "2": "rubber"}, "11630": {"0": "purple", "1": "brown", "2": "yes", "3": "error", "4": "error", "5": "2", "6": "cylinder", "7": "metal", "8": "yes", "9": "1"}, "11631": {"0": "4", "1": "gray", "2": "yellow"}, "11632": {"0": "0", "1": "cylinder", "2": "cube"}, "11633": {"0": "error", "1": "blue", "2": "metal"}, "11634": {"0": "0", "1": "error", "2": "0"}, "11635": {"0": "yes", "1": "1", "2": "rubber"}, "11636": {"0": "blue", "1": "red", "2": "no"}, "11637": {"0": "blue", "1": "right", "2": "4"}, "11638": {"0": "red", "1": "red", "2": "0", "3": "0", "4": "no", "5": "yes"}, "11639": {"0": "0", "1": "yes", "2": "green"}, "11640": {"0": "blue", "1": "brown", "2": "sphere"}, "11641": {"0": "error", "1": "cylinder", "2": "error"}, "11642": {"0": "red", "1": "yes", "2": "brown"}, "11643": {"0": "no", "1": "error", "2": "rubber"}, "11644": {"0": "metal", "1": "purple", "2": "gray"}, "11645": {"0": "rubber", "1": "no", "2": "1"}, "11646": {"0": "rubber", "1": "rubber", "2": "1"}, "11647": {"0": "gray", "1": "cube", "2": "brown"}, "11648": {"0": "error", "1": "purple", "2": "yes"}, "11649": {"0": "red", "1": "1", "2": "error"}, "11650": {"0": "cyan", "1": "cylinder", "2": "yes"}, "11651": {"0": "metal", "1": "cube", "2": "red"}, "11652": {"0": "yes", "1": "no", "2": "metal"}, "11653": {"0": "sphere", "1": "blue"}, "11654": {"0": "cube", "1": "0", "2": "no"}, "11655": {"0": "cylinder", "1": "cube", "2": "sphere"}, "11656": {"0": "down", "1": "1", "2": "no"}, "11657": {"0": "metal", "1": "yellow", "2": "error", "3": "error", "4": "metal", "5": "0", "6": "cylinder", "7": "error", "8": "error", "9": "error"}, "11658": {"0": "gray", "1": "yes", "2": "1"}, "11659": {"0": "0", "1": "cyan", "2": "sphere"}, "11660": {"0": "brown", "1": "error"}, "11661": {"0": "0", "1": "1", "2": "no"}, "11662": {"0": "error", "1": "blue", "2": "rubber"}, "11663": {"0": "yes", "1": "metal", "2": "cube"}, "11664": {"0": "3", "1": "metal", "2": "no"}, "11665": {"0": "yes", "1": "no", "2": "blue"}, "11666": {"0": "error", "1": "3", "2": "0"}, "11667": {"0": "yes", "1": "error", "2": "yes"}, "11668": {"0": "cylinder", "1": "no", "2": "yellow"}, "11669": {"0": "gray", "1": "yes"}, "11670": {"0": "0", "1": "sphere", "2": "sphere"}, "11671": {"0": "metal", "1": "no", "2": "metal"}, "11672": {"0": "cylinder", "1": "cube", "2": "yellow"}, "11673": {"0": "rubber", "1": "yes", "2": "yes"}, "11674": {"0": "brown", "1": "no", "2": "brown"}, "11675": {"0": "no", "1": "cyan", "2": "brown"}, "11676": {"0": "yes", "1": "no", "2": "metal"}, "11677": {"0": "purple", "1": "rubber", "2": "purple"}, "11678": {"0": "2", "1": "1", "2": "cube"}, "11679": {"0": "1", "1": "rubber", "2": "yes"}, "11680": {"0": "0", "1": "brown", "2": "blue"}, "11681": {"0": "cylinder", "1": "sphere", "2": "yes"}, "11682": {"0": "no", "1": "no"}, "11683": {"0": "cube", "1": "yes", "2": "no"}, "11684": {"0": "1", "1": "no", "2": "cube"}, "11685": {"0": "sphere", "1": "yellow", "2": "0"}, "11686": {"0": "metal", "1": "brown", "2": "rubber"}, "11687": {"0": "cylinder", "1": "2", "2": "cylinder"}, "11688": {"0": "purple", "1": "purple", "2": "gray"}, "11689": {"0": "3", "1": "error", "2": "metal"}, "11690": {"0": "no", "1": "0", "2": "rubber"}, "11691": {"0": "0", "1": "rubber", "2": "no"}, "11692": {"0": "rubber", "1": "rubber", "2": "cube"}, "11693": {"0": "error", "1": "gray", "2": "red"}, "11694": {"0": "rubber", "1": "no", "2": "gray"}, "11695": {"0": "yes", "1": "no", "2": "brown"}, "11696": {"0": "sphere", "1": "green", "2": "yes"}, "11697": {"0": "gray", "1": "cylinder", "2": "sphere"}, "11698": {"0": "cylinder", "1": "green", "2": "2"}, "11699": {"0": "2", "1": "left", "2": "cylinder"}, "11700": {"0": "rubber", "1": "purple", "2": "error"}, "11701": {"0": "blue", "1": "brown", "2": "3"}, "11702": {"0": "0", "1": "green", "2": "yes"}, "11703": {"0": "rubber", "1": "1", "2": "red"}, "11704": {"0": "cylinder", "1": "rubber", "2": "metal"}, "11705": {"0": "yellow", "1": "yes", "2": "green"}, "11706": {"0": "sphere", "1": "2", "2": "cylinder"}, "11707": {"0": "blue", "1": "right", "2": "right"}, "11708": {"0": "metal", "1": "blue", "2": "metal"}, "11709": {"0": "purple", "1": "yes", "2": "purple"}, "11710": {"0": "1", "1": "2"}, "11711": {"0": "0", "1": "cylinder"}, "11712": {"0": "yes", "1": "error", "2": "2"}, "11713": {"0": "3", "1": "cylinder", "2": "yellow"}, "11714": {"0": "1", "1": "sphere", "2": "yes"}, "11715": {"0": "metal", "1": "0", "2": "metal"}, "11716": {"0": "3", "1": "cube", "2": "rubber"}, "11717": {"0": "purple", "1": "yes", "2": "rubber"}, "11718": {"0": "blue", "1": "red"}, "11719": {"0": "no", "1": "metal", "2": "error"}, "11720": {"0": "1", "1": "gray", "2": "up"}, "11721": {"0": "cylinder", "1": "metal", "2": "yes", "3": "gray", "4": "gray", "5": "2", "6": "up", "7": "error", "8": "1", "9": "4"}, "11722": {"0": "1", "1": "sphere", "2": "yes"}, "11723": {"0": "cylinder", "1": "0", "2": "sphere"}, "11724": {"0": "down", "1": "error", "2": "0"}, "11725": {"0": "2", "1": "cyan", "2": "purple"}, "11726": {"0": "cyan", "1": "2", "2": "rubber"}, "11727": {"0": "metal", "1": "rubber", "2": "cyan"}, "11728": {"0": "sphere", "1": "cyan"}, "11729": {"0": "rubber", "1": "gray", "2": "yellow"}, "11730": {"0": "rubber", "1": "4", "2": "cylinder"}, "11731": {"0": "cube", "1": "yellow", "2": "yellow"}, "11732": {"0": "blue", "1": "2", "2": "1"}, "11733": {"0": "rubber", "1": "1", "2": "purple"}, "11734": {"0": "yes", "1": "0", "2": "gray"}, "11735": {"0": "sphere", "1": "gray", "2": "cylinder"}, "11736": {"0": "purple", "1": "no", "2": "yellow"}, "11737": {"0": "2", "1": "yes", "2": "3"}, "11738": {"0": "down", "1": "yes", "2": "cyan"}, "11739": {"0": "rubber", "1": "0", "2": "metal"}, "11740": {"0": "sphere", "1": "sphere", "2": "2"}, "11741": {"0": "right", "1": "rubber", "2": "sphere"}, "11742": {"0": "0", "1": "brown", "2": "sphere"}, "11743": {"0": "blue", "1": "0", "2": "blue"}, "11744": {"0": "0", "1": "no", "2": "rubber"}, "11745": {"0": "red", "1": "cube", "2": "up"}, "11746": {"0": "3", "1": "yellow", "2": "3"}, "11747": {"0": "sphere", "1": "cube", "2": "metal"}, "11748": {"0": "rubber", "1": "no", "2": "blue"}, "11749": {"0": "yes", "1": "cylinder"}, "11750": {"0": "0", "1": "2", "2": "metal"}, "11751": {"0": "1", "1": "1", "2": "1"}, "11752": {"0": "blue", "1": "blue", "2": "purple"}, "11753": {"0": "metal", "1": "no"}, "11754": {"0": "1", "1": "no", "2": "yellow"}, "11755": {"0": "0", "1": "yes", "2": "error"}, "11756": {"0": "purple", "1": "cylinder", "2": "blue"}, "11757": {"0": "yes", "1": "2", "2": "purple"}, "11758": {"0": "2", "1": "blue", "2": "0"}, "11759": {"0": "no", "1": "cyan", "2": "0"}, "11760": {"0": "no", "1": "metal", "2": "0"}, "11761": {"0": "metal", "1": "yes", "2": "yellow"}, "11762": {"0": "right", "1": "no", "2": "cylinder"}, "11763": {"0": "0", "1": "sphere", "2": "yellow"}, "11764": {"0": "metal", "1": "yellow", "2": "gray"}, "11765": {"0": "no", "1": "cyan", "2": "no"}, "11766": {"0": "4", "1": "cylinder", "2": "rubber"}, "11767": {"0": "rubber", "1": "no", "2": "3"}, "11768": {"0": "error", "1": "error", "2": "error"}, "11769": {"0": "cylinder", "1": "error", "2": "red"}, "11770": {"0": "1", "1": "0", "2": "sphere"}, "11771": {"0": "blue", "1": "1", "2": "1"}, "11772": {"0": "no", "1": "error", "2": "cube"}, "11773": {"0": "cylinder", "1": "blue", "2": "green"}, "11774": {"0": "2", "1": "error", "2": "left"}, "11775": {"0": "blue", "1": "rubber", "2": "cube"}, "11776": {"0": "no", "1": "blue", "2": "error"}, "11777": {"0": "no", "1": "error"}, "11778": {"0": "1", "1": "green", "2": "error"}, "11779": {"0": "brown", "1": "cylinder", "2": "red"}, "11780": {"0": "brown", "1": "1", "2": "sphere"}, "11781": {"0": "green", "1": "no", "2": "no"}, "11782": {"0": "purple", "1": "yellow", "2": "yes"}, "11783": {"0": "left", "1": "metal", "2": "green"}, "11784": {"0": "1", "1": "1"}, "11785": {"0": "up", "1": "red", "2": "yellow"}, "11786": {"0": "no", "1": "1", "2": "cube"}, "11787": {"0": "brown", "1": "error", "2": "error"}, "11788": {"0": "sphere", "1": "cube"}, "11789": {"0": "gray", "1": "yes"}, "11790": {"0": "red", "1": "yes", "2": "blue"}, "11791": {"0": "metal", "1": "brown", "2": "0"}, "11792": {"0": "2", "1": "cyan"}, "11793": {"0": "no", "1": "metal", "2": "sphere"}, "11794": {"0": "blue", "1": "1", "2": "gray"}, "11795": {"0": "metal", "1": "no", "2": "cylinder"}, "11796": {"0": "cube", "1": "yes", "2": "cyan"}, "11797": {"0": "purple", "1": "error", "2": "error"}, "11798": {"0": "yellow", "1": "cube", "2": "rubber"}, "11799": {"0": "cylinder", "1": "1", "2": "cyan"}, "11800": {"0": "yes", "1": "no"}, "11801": {"0": "error", "1": "1", "2": "error"}, "11802": {"0": "brown", "1": "brown", "2": "sphere"}, "11803": {"0": "1", "1": "cylinder", "2": "yes"}, "11804": {"0": "0", "1": "yellow", "2": "no"}, "11805": {"0": "no", "1": "1"}, "11806": {"0": "blue", "1": "purple", "2": "no"}, "11807": {"0": "cube", "1": "rubber", "2": "metal"}, "11808": {"0": "gray", "1": "2", "2": "yes"}, "11809": {"0": "error", "1": "rubber", "2": "brown"}, "11810": {"0": "no", "1": "no"}, "11811": {"0": "purple", "1": "1"}, "11812": {"0": "2", "1": "sphere", "2": "sphere"}, "11813": {"0": "cyan", "1": "no", "2": "0"}, "11814": {"0": "error", "1": "yellow"}, "11815": {"0": "metal", "1": "metal"}, "11816": {"0": "error", "1": "cube", "2": "right"}, "11817": {"0": "left", "1": "yes", "2": "rubber"}, "11818": {"0": "no", "1": "right"}, "11819": {"0": "rubber", "1": "cube", "2": "cylinder"}, "11820": {"0": "error", "1": "no", "2": "no"}, "11821": {"0": "error", "1": "0", "2": "no"}, "11822": {"0": "cube", "1": "no", "2": "rubber"}, "11823": {"0": "yellow", "1": "yes", "2": "cylinder"}, "11824": {"0": "0", "1": "sphere", "2": "cylinder"}, "11825": {"0": "yes", "1": "1", "2": "cube"}, "11826": {"0": "no", "1": "cube", "2": "1"}, "11827": {"0": "blue", "1": "no", "2": "down"}, "11828": {"0": "yellow", "1": "error", "2": "yellow"}, "11829": {"0": "1", "1": "yellow", "2": "yes"}, "11830": {"0": "yes", "1": "metal"}, "11831": {"0": "cube", "1": "yellow", "2": "up"}, "11832": {"0": "yellow", "1": "cyan", "2": "3"}, "11833": {"0": "down", "1": "0", "2": "metal"}, "11834": {"0": "no", "1": "2", "2": "0"}, "11835": {"0": "yes", "1": "left", "2": "error"}, "11836": {"0": "blue", "1": "purple", "2": "1"}, "11837": {"0": "green", "1": "0", "2": "metal"}, "11838": {"0": "error", "1": "brown", "2": "no"}, "11839": {"0": "cylinder", "1": "2", "2": "yellow"}, "11840": {"0": "gray", "1": "no", "2": "error"}, "11841": {"0": "rubber", "1": "metal", "2": "yes", "3": "1", "4": "rubber", "5": "up", "6": "yes", "7": "yes"}, "11842": {"0": "no", "1": "down", "2": "blue"}, "11843": {"0": "no", "1": "0", "2": "brown"}, "11844": {"0": "yellow", "1": "sphere", "2": "yes", "3": "blue", "4": "green", "5": "3", "6": "up", "7": "down", "8": "1", "9": "2"}, "11845": {"0": "0", "1": "metal"}, "11846": {"0": "brown", "1": "no", "2": "brown"}, "11847": {"0": "cube", "1": "gray", "2": "1"}, "11848": {"0": "no", "1": "cyan", "2": "no"}, "11849": {"0": "purple", "1": "1", "2": "purple"}, "11850": {"0": "no", "1": "yes", "2": "cube"}, "11851": {"0": "gray", "1": "green", "2": "error"}, "11852": {"0": "rubber", "1": "sphere", "2": "yes", "3": "error", "4": "green", "5": "0", "6": "cyan", "7": "cylinder", "8": "yes", "9": "yes"}, "11853": {"0": "right", "1": "sphere", "2": "purple"}, "11854": {"0": "cube", "1": "no", "2": "brown"}, "11855": {"0": "brown", "1": "yes", "2": "brown"}, "11856": {"0": "cylinder", "1": "1", "2": "1"}, "11857": {"0": "cube", "1": "cylinder", "2": "yes"}, "11858": {"0": "rubber", "1": "yes", "2": "brown"}, "11859": {"0": "cube", "1": "metal", "2": "purple"}, "11860": {"0": "gray", "1": "yes", "2": "2"}, "11861": {"0": "green", "1": "yes", "2": "1"}, "11862": {"0": "gray", "1": "yes", "2": "gray"}, "11863": {"0": "yellow", "1": "green", "2": "1"}, "11864": {"0": "error", "1": "yes", "2": "red"}, "11865": {"0": "red", "1": "no", "2": "0"}, "11866": {"0": "rubber", "1": "up", "2": "3"}, "11867": {"0": "left", "1": "0", "2": "gray"}, "11868": {"0": "cube", "1": "yes", "2": "2"}, "11869": {"0": "no", "1": "cylinder", "2": "no"}, "11870": {"0": "error", "1": "yes", "2": "error"}, "11871": {"0": "error", "1": "error", "2": "no"}, "11872": {"0": "1", "1": "no"}, "11873": {"0": "cube", "1": "sphere", "2": "0"}, "11874": {"0": "3", "1": "3", "2": "error"}, "11875": {"0": "2", "1": "blue", "2": "metal"}, "11876": {"0": "error", "1": "no", "2": "yes"}, "11877": {"0": "green", "1": "cylinder", "2": "0", "3": "green", "4": "brown", "5": "1", "6": "1", "7": "0"}, "11878": {"0": "cube", "1": "0", "2": "yes"}, "11879": {"0": "rubber", "1": "error", "2": "gray"}, "11880": {"0": "red", "1": "blue", "2": "left"}, "11881": {"0": "cube", "1": "rubber", "2": "no"}, "11882": {"0": "sphere", "1": "no", "2": "no"}, "11883": {"0": "right", "1": "yes", "2": "brown"}, "11884": {"0": "rubber", "1": "metal", "2": "metal"}, "11885": {"0": "error", "1": "rubber", "2": "red"}, "11886": {"0": "yes", "1": "gray", "2": "0"}, "11887": {"0": "yes", "1": "brown", "2": "0"}, "11888": {"0": "0", "1": "no", "2": "gray"}, "11889": {"0": "cube", "1": "red", "2": "3"}, "11890": {"0": "yellow", "1": "no"}, "11891": {"0": "yes", "1": "gray", "2": "no"}, "11892": {"0": "metal", "1": "3", "2": "1"}, "11893": {"0": "0", "1": "rubber", "2": "error"}, "11894": {"0": "0", "1": "gray", "2": "green"}, "11895": {"0": "2", "1": "rubber", "2": "yes"}, "11896": {"0": "no", "1": "green", "2": "cube"}, "11897": {"0": "gray", "1": "brown", "2": "cylinder"}, "11898": {"0": "rubber", "1": "1", "2": "metal"}, "11899": {"0": "rubber", "1": "metal", "2": "cube"}, "11900": {"0": "0", "1": "green", "2": "yes"}, "11901": {"0": "sphere", "1": "up", "2": "rubber"}, "11902": {"0": "no", "1": "cyan", "2": "1"}, "11903": {"0": "0", "1": "3", "2": "sphere"}, "11904": {"0": "red", "1": "0", "2": "up"}, "11905": {"0": "blue", "1": "cyan", "2": "green"}, "11906": {"0": "cyan", "1": "gray", "2": "metal"}, "11907": {"0": "right", "1": "2"}, "11908": {"0": "1", "1": "2"}, "11909": {"0": "red", "1": "green", "2": "0"}, "11910": {"0": "green", "1": "error", "2": "3", "3": "rubber", "4": "rubber", "5": "0", "6": "green", "7": "cyan", "8": "no", "9": "no"}, "11911": {"0": "brown", "1": "yellow", "2": "0"}, "11912": {"0": "0", "1": "rubber", "2": "cube"}, "11913": {"0": "purple", "1": "3", "2": "rubber"}, "11914": {"0": "rubber", "1": "1", "2": "brown"}, "11915": {"0": "yes", "1": "metal", "2": "0"}, "11916": {"0": "2", "1": "rubber", "2": "cube"}, "11917": {"0": "blue", "1": "cylinder", "2": "0"}, "11918": {"0": "no", "1": "no", "2": "rubber"}, "11919": {"0": "1", "1": "sphere", "2": "metal"}, "11920": {"0": "cylinder", "1": "sphere", "2": "0"}, "11921": {"0": "1", "1": "cube", "2": "cylinder"}, "11922": {"0": "0", "1": "gray", "2": "metal"}, "11923": {"0": "1", "1": "rubber"}, "11924": {"0": "yellow", "1": "1", "2": "purple"}, "11925": {"0": "no", "1": "green", "2": "yes"}, "11926": {"0": "no", "1": "no", "2": "cylinder"}, "11927": {"0": "rubber", "1": "2", "2": "cyan"}, "11928": {"0": "cube", "1": "cylinder", "2": "no"}, "11929": {"0": "rubber", "1": "2", "2": "no"}, "11930": {"0": "1", "1": "right", "2": "error"}, "11931": {"0": "cyan", "1": "1", "2": "no"}, "11932": {"0": "error", "1": "error", "2": "1"}, "11933": {"0": "0", "1": "1", "2": "purple"}, "11934": {"0": "cube", "1": "error", "2": "green"}, "11935": {"0": "no", "1": "right", "2": "no"}, "11936": {"0": "error", "1": "3", "2": "no"}, "11937": {"0": "cylinder", "1": "no", "2": "sphere"}, "11938": {"0": "cylinder", "1": "cube", "2": "3"}, "11939": {"0": "blue", "1": "1", "2": "no"}, "11940": {"0": "rubber", "1": "metal", "2": "metal"}, "11941": {"0": "2", "1": "metal", "2": "error"}, "11942": {"0": "red", "1": "purple", "2": "sphere"}, "11943": {"0": "cylinder", "1": "left", "2": "yes"}, "11944": {"0": "metal", "1": "green", "2": "cylinder"}, "11945": {"0": "metal", "1": "cube", "2": "no"}, "11946": {"0": "rubber", "1": "red"}, "11947": {"0": "yes", "1": "cube", "2": "yes"}, "11948": {"0": "metal", "1": "purple", "2": "metal"}, "11949": {"0": "cube", "1": "metal", "2": "0"}, "11950": {"0": "rubber", "1": "purple", "2": "3"}, "11951": {"0": "cyan", "1": "blue"}, "11952": {"0": "error", "1": "gray", "2": "no"}, "11953": {"0": "no", "1": "rubber", "2": "1"}, "11954": {"0": "1", "1": "red", "2": "2"}, "11955": {"0": "red", "1": "error", "2": "3"}, "11956": {"0": "error", "1": "purple", "2": "2"}, "11957": {"0": "metal", "1": "error", "2": "0"}, "11958": {"0": "no", "1": "rubber", "2": "gray"}, "11959": {"0": "purple", "1": "error", "2": "no"}, "11960": {"0": "red", "1": "4", "2": "gray"}, "11961": {"0": "1", "1": "2", "2": "error"}, "11962": {"0": "0", "1": "1", "2": "cyan"}, "11963": {"0": "1", "1": "sphere", "2": "yes"}, "11964": {"0": "sphere", "1": "blue"}, "11965": {"0": "blue", "1": "cube"}, "11966": {"0": "0", "1": "brown", "2": "purple"}, "11967": {"0": "1", "1": "no", "2": "gray"}, "11968": {"0": "red", "1": "yes", "2": "0"}, "11969": {"0": "cube", "1": "cube", "2": "no"}, "11970": {"0": "metal", "1": "sphere", "2": "metal"}, "11971": {"0": "metal", "1": "red", "2": "red"}, "11972": {"0": "red", "1": "cylinder", "2": "rubber"}, "11973": {"0": "yes", "1": "green", "2": "sphere"}, "11974": {"0": "0", "1": "cube", "2": "cylinder"}, "11975": {"0": "error", "1": "rubber", "2": "no"}, "11976": {"0": "2", "1": "sphere", "2": "metal"}, "11977": {"0": "no", "1": "sphere", "2": "brown"}, "11978": {"0": "2", "1": "cylinder", "2": "green"}, "11979": {"0": "sphere", "1": "2", "2": "yes"}, "11980": {"0": "1", "1": "0", "2": "sphere"}, "11981": {"0": "2", "1": "yellow", "2": "2"}, "11982": {"0": "1", "1": "cube", "2": "green"}, "11983": {"0": "green", "1": "red", "2": "sphere"}, "11984": {"0": "no", "1": "error", "2": "cylinder"}, "11985": {"0": "yes", "1": "gray", "2": "gray"}, "11986": {"0": "cube", "1": "sphere", "2": "green"}, "11987": {"0": "cyan", "1": "no", "2": "blue"}, "11988": {"0": "brown", "1": "yellow", "2": "0"}, "11989": {"0": "cube", "1": "sphere", "2": "no"}, "11990": {"0": "rubber", "1": "brown", "2": "brown"}, "11991": {"0": "0", "1": "error", "2": "no"}, "11992": {"0": "rubber", "1": "1", "2": "sphere"}, "11993": {"0": "cylinder", "1": "error", "2": "error"}, "11994": {"0": "sphere", "1": "sphere", "2": "no"}, "11995": {"0": "yes", "1": "yellow", "2": "red"}, "11996": {"0": "2", "1": "yes", "2": "red"}, "11997": {"0": "red", "1": "2", "2": "yes"}, "11998": {"0": "brown", "1": "1", "2": "yes"}, "11999": {"0": "cube", "1": "green", "2": "1"}}


--------------------------------------------------------------------------------
/tools/transform_anno.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import pdb
 3 | import argparse
 4 | import json
 5 | 
 6 | parser = argparse.ArgumentParser()
 7 | parser.add_argument('--mode', default=0, type=int, help='0 for GT parsing' )
 8 | parser.add_argument('--st_id', default=0, type=int, help='start id' )
 9 | parser.add_argument('--ed_id', default=1000, type=int, help='end id' )
10 | parser.add_argument('--gt_ques_ann_dir', default='/home/tmp_user/code/clevrer_dataset_generation_v2/clevrer_question_generation/output/questions_v13')
11 | 
12 | def prepare_gt_programs(args):
13 |     for vid in range(args.st_id, args.ed_id):
14 |         sim_str = 'sim_%05d'%vid
15 |         gt_oe_ques_path = os.path.join(args.gt_ques_ann_dir, 'open_end_questions.json')
16 |         with open(gt_oe_ques_path, 'r') as fh:
17 |             gt_oe_ques_info = json.load(fh)
18 |         gt_mc_ques_path = os.path.join(args.gt_ques_ann_dir, 'multiple_choice_questions.json')
19 |         with open(gt_mc_ques_path, 'r') as fh:
20 |             gt_mc_ques_info = json.load(fh)
21 |         assert gt_oe_ques_info['scene_index'] == gt_mc_ques_info['scene_index']
22 |         out_dict = {"scene_index": gt_oe_ques_info['scene_index'],
23 |                 "video_filename": gt_oe_ques_info['video_filename']}
24 | 
25 | if __name__=='__main__':
26 |     args = parser.parse_args()
27 |     if args.mode==0:
28 |         prepare_gt_programs(args)
29 | 
30 | 


--------------------------------------------------------------------------------
/utils/utils.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import pdb
  3 | import argparse
  4 | import json
  5 | import pycocotools.mask as coco_mask
  6 | import numpy as np
  7 | 
  8 | def get_obj_id_by_attr(obj_list, obj):
  9 |     for obj_info in obj_list:
 10 |         if obj_info['color'] == obj['color'] and obj_info['shape']==obj['shape'] \
 11 |                 and obj_info['material'] == obj['material']:
 12 |             return obj_info['id']
 13 |     return -1
 14 | 
 15 | def get_centre_from_mask(mask):
 16 |     [x, y, w, h] = coco_mask.toBbox(mask)
 17 |     return x+w*0.5, y+h*0.5
 18 | 
 19 | def load_gt_ann(sim_id, args):
 20 |     sim_str = 'sim_%05d'%sim_id
 21 |     ann_path = os.path.join(args.ann_dir, sim_str, 'annotations', 'annotation.json')
 22 |     with open(ann_path, 'r') as fh:
 23 |         ann = json.load(fh)
 24 |     objs = []
 25 |     for obj_id, obj_info in enumerate(ann['config']):
 26 |         tmp_obj = {'color': obj_info['color'],
 27 |                 'material': obj_info['material'],
 28 |                 'shape': obj_info['shape'],
 29 |                 'mass': obj_info['mass'],
 30 |                 'charge': obj_info['charge'],
 31 |                 'id': obj_id}
 32 |         objs.append(tmp_obj)
 33 |     ann_mask_path  = os.path.join(args.track_dir, sim_str+'.json')
 34 |     with open(ann_mask_path, 'r') as fh:
 35 |         ann_mask = json.load(fh)['scenes']
 36 |     scene_num = len(ann_mask)
 37 |     preds = []
 38 |     tmp_pred = []
 39 |     for frm_id  in range(0, scene_num, args.frame_diff):
 40 |         frm_obj_info = ann_mask[frm_id]['objects'] 
 41 |         out_frm_info = {'frame_index': frm_id, 'objects':[]}
 42 |         obj_list = []
 43 |         for obj_id, obj_info in enumerate(frm_obj_info): 
 44 |             tmp_obj = {'color': obj_info['color'],
 45 |                     'material': obj_info['material'],
 46 |                     'shape': obj_info['shape'],
 47 |                     'frame': frm_id}
 48 |             obj_id = get_obj_id_by_attr(objs, obj_info) 
 49 |             tmp_obj['id'] = obj_id
 50 |             x, y = get_centre_from_mask( obj_info['mask'])
 51 |             tmp_obj['x'] = x
 52 |             tmp_obj['y'] = y
 53 |             obj_list.append(tmp_obj)
 54 |         out_frm_info['objects'] = obj_list
 55 |         tmp_pred.append(out_frm_info)
 56 |     preds = [{'what_if': -1, 'trajectory': tmp_pred, 'collisions': ann['collisions'] }]
 57 |     return objs, preds
 58 | 
 59 | def load_mc_ann(sim_id, args):
 60 |     IMG_H, IMG_W = 320, 480
 61 |     sim_str = 'sim_%05d'%sim_id
 62 |     if args.gt_flag:
 63 |         ann_path = os.path.join(args.ann_dir, sim_str, 'annotations', 'annotation.json')
 64 |     else:
 65 |         ann_path = os.path.join(args.ann_dir, sim_str + '.json')
 66 |         
 67 |     with open(ann_path, 'r') as fh:
 68 |         ann = json.load(fh)
 69 |     objs = []
 70 |     motion_path = os.path.join(args.raw_motion_prediction_dir, sim_str+'.json') 
 71 |     with open(motion_path, 'r') as fh:
 72 |         raw_pred = json.load(fh) 
 73 |     for obj_id, obj_info in enumerate(ann['config']):
 74 |         tmp_obj = {'color': obj_info['color'],
 75 |                 'material': obj_info['material'],
 76 |                 'shape': obj_info['shape'],
 77 |                 'id': obj_id}
 78 |         if 'mass' in obj_info:        
 79 |             tmp_obj['mass'] = obj_info['mass']
 80 |         else:
 81 |             tmp_obj['mass'] = 1
 82 |         if 'charge' in obj_info:
 83 |             tmp_obj['charge'] =  obj_info['charge']
 84 |         else:
 85 |             tmp_obj['charge'] =  0
 86 |         objs.append(tmp_obj)
 87 |     raw_pred_list = []
 88 |     for key_id in ['future', 'mass', 'charge']:
 89 |         pred_tmp  = raw_pred[key_id]
 90 |         if isinstance(pred_tmp, dict):
 91 |             raw_pred_list.append(pred_tmp)
 92 |         elif isinstance(pred_tmp, list):
 93 |             raw_pred_list += pred_tmp 
 94 |     preds = []
 95 |     for pred_id, pred_tmp in enumerate(raw_pred_list):
 96 |         tmp_output = {"what_if": pred_tmp["what_if"]}
 97 |         if "mass" in pred_tmp:
 98 |             tmp_output['mass'] = pred_tmp['mass']
 99 |         if "charge" in pred_tmp:
100 |             tmp_output['charge'] = pred_tmp['charge']
101 |         track = pred_tmp['trajectories']
102 |         assert len(track) == len(ann['config'])
103 |         obj_num, frm_num = len(track), len(track[0])
104 |         track_list = [] 
105 |         for frm_id in range(frm_num):
106 |             frm = frm_id * args.frame_diff
107 |             frm_info = {'frame_index': frm, "objects": []}
108 |             objs_list = []
109 |             for obj_id in range(obj_num):
110 |                 x_c, y_c, w, h = track[obj_id][frm_id]
111 |                 if x_c <0 or x_c>1 or y_c<0 or y_c>1:
112 |                     continue
113 |                 obj_info = {"color": ann['config'][obj_id]["color"],
114 |                         "shape": ann['config'][obj_id]["shape"],
115 |                         "material": ann['config'][obj_id]["material"],
116 |                         "frame": frm, 
117 |                         "x": x_c * IMG_W, 
118 |                         "y": y_c * IMG_H,
119 |                         "w": w * IMG_W,
120 |                         "h": h * IMG_H,
121 |                         "id": obj_id}
122 |                 objs_list.append(obj_info)
123 |             frm_info["objects"] = objs_list
124 |             track_list.append(frm_info)
125 |         tmp_output["trajectory"] = track_list
126 |         preds.append(tmp_output)
127 |     if not args.gt_flag and 'edegs' in ann:
128 |         edges = np.array(ann['edges'])
129 |         assert len(objs)==edges.shape[0], 'Shape inconsistent'
130 |     else:
131 |         edges = None
132 |     return objs, preds, edges
133 | 
134 | def load_ann(sim_id, args):
135 |     return load_mc_ann(sim_id, args)
136 | 
137 | def print_monitor(monitor):
138 |     for key_id, acc_num in monitor.items():
139 |         if key_id.endswith('total'):
140 |             continue
141 |         q_type = key_id.rsplit('_', 1)[0]
142 |         acc = acc_num /( 1.0 * monitor[q_type +'_total'] )
143 |         print('%s: acc: %f, %d/%d\n'%(q_type, acc, acc_num, monitor[q_type+'_total']))
144 | if __name__=='__main__':
145 |     pdb.set_trace()
146 | 


--------------------------------------------------------------------------------