33 |
40 |
41 |
42 |
43 |
Example videos
44 |
45 |
Below a few short excerpts. To watch the full video, hover on the frame and click on the YouTube icon.
46 |
47 |
Request to download dev and test set (366 videos for 14 hours) here.
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
--------------------------------------------------------------------------------
/examples/opencv/opencv.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -x
4 |
5 | # Change into directory of script.
6 | cd "$( dirname "${BASH_SOURCE[0]}" )"
7 |
8 | python track.py -v ../../dataset/ ../../predictions/ --data=dev --tracker=MEDIANFLOW
9 |
--------------------------------------------------------------------------------
/examples/opencv/requirements.txt:
--------------------------------------------------------------------------------
1 | opencv-python
2 | opencv-contrib-python
3 |
--------------------------------------------------------------------------------
/examples/opencv/track.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | import argparse
6 | import cv2
7 | import os
8 | import time
9 |
10 | import oxuva
11 |
12 |
13 | def main():
14 | parser = argparse.ArgumentParser()
15 | parser.add_argument('data_dir')
16 | parser.add_argument('predictions_dir')
17 | parser.add_argument('--data', default='dev')
18 | parser.add_argument('--verbose', '-v', action='store_true')
19 | parser.add_argument('--tracker', default='TLD')
20 | global args
21 | args = parser.parse_args()
22 |
23 | tracker_id = 'cv' + args.tracker
24 | tracker_preds_dir = os.path.join(args.predictions_dir, args.data, tracker_id)
25 | if not os.path.exists(tracker_preds_dir):
26 | os.makedirs(tracker_preds_dir, 0755)
27 |
28 | tasks_file = os.path.join(args.data_dir, 'tasks', args.data + '.csv')
29 | with open(tasks_file, 'r') as fp:
30 | tasks = oxuva.load_dataset_tasks_csv(fp)
31 | # tracks_file = os.path.join(args.data_dir, 'annotations', args.data + '.csv')
32 | # with open(tracks_file, 'r') as fp:
33 | # tracks = oxuva.load_annotations_csv(fp)
34 | # tasks = {key: oxuva.make_task_from_track(track) for key, track in tracks.items()}
35 |
36 | imfile = lambda vid, t: os.path.join(
37 | args.data_dir, 'images', args.data, vid, '{:06d}.jpeg'.format(t))
38 |
39 | for key, task in tasks.items():
40 | vid, obj = key
41 | if args.verbose:
42 | print(vid, obj)
43 | preds_file = os.path.join(tracker_preds_dir, '{}_{}.csv'.format(vid, obj))
44 | if os.path.exists(preds_file):
45 | continue
46 |
47 | tracker = Tracker(tracker_type=args.tracker)
48 | tracker.init(imfile(vid, task.init_time), task.init_rect)
49 | preds = oxuva.SparseTimeSeries()
50 | start = time.time()
51 | for t in range(task.init_time + 1, task.last_time + 1):
52 | preds[t] = tracker.next(imfile(vid, t))
53 | dur_sec = time.time() - start
54 | if args.verbose:
55 | fps = (task.last_time - task.init_time + 1) / dur_sec
56 | print('fps {:.3g}'.format(fps))
57 |
58 | tmp_preds_file = os.path.join(tracker_preds_dir, '{}_{}.csv.tmp'.format(vid, obj))
59 | with open(tmp_preds_file, 'w') as fp:
60 | oxuva.dump_predictions_csv(vid, obj, preds, fp)
61 | os.rename(tmp_preds_file, preds_file)
62 |
63 |
64 | class Tracker:
65 |
66 | def __init__(self, tracker_type):
67 | self._tracker = create_tracker(tracker_type)
68 |
69 | def init(self, imfile, rect):
70 | im = cv2.imread(imfile, cv2.IMREAD_COLOR)
71 | imheight, imwidth, _ = im.shape
72 | if args.verbose:
73 | print('image size', '{}x{}'.format(imwidth, imheight))
74 | cvrect = rect_to_opencv(rect, imsize_hw=(imheight, imwidth))
75 | ok = self._tracker.init(im, cvrect)
76 | assert ok
77 |
78 | def next(self, imfile):
79 | im = cv2.imread(imfile, cv2.IMREAD_COLOR)
80 | imheight, imwidth, _ = im.shape
81 | ok, cvrect = self._tracker.update(im)
82 | if not ok:
83 | return oxuva.make_prediction(present=False, score=0.0)
84 | else:
85 | rect = rect_from_opencv(cvrect, imsize_hw=(imheight, imwidth))
86 | return oxuva.make_prediction(present=True, score=1.0, **rect)
87 |
88 |
89 | def create_tracker(tracker_type):
90 | # https://www.learnopencv.com/object-tracking-using-opencv-cpp-python/
91 | major_ver, minor_ver, subminor_ver = cv2.__version__.split('.')
92 | if int(minor_ver) < 3:
93 | tracker = cv2.Tracker_create(tracker_type)
94 | else:
95 | if tracker_type == 'BOOSTING':
96 | tracker = cv2.TrackerBoosting_create()
97 | elif tracker_type == 'MIL':
98 | tracker = cv2.TrackerMIL_create()
99 | elif tracker_type == 'KCF':
100 | tracker = cv2.TrackerKCF_create()
101 | elif tracker_type == 'TLD':
102 | tracker = cv2.TrackerTLD_create()
103 | elif tracker_type == 'MEDIANFLOW':
104 | tracker = cv2.TrackerMedianFlow_create()
105 | elif tracker_type == 'GOTURN':
106 | tracker = cv2.TrackerGOTURN_create()
107 | return tracker
108 |
109 |
110 | def rect_to_opencv(rect, imsize_hw):
111 | imheight, imwidth = imsize_hw
112 | xmin_abs = rect['xmin'] * imwidth
113 | ymin_abs = rect['ymin'] * imheight
114 | xmax_abs = rect['xmax'] * imwidth
115 | ymax_abs = rect['ymax'] * imheight
116 | return (xmin_abs, ymin_abs, xmax_abs - xmin_abs, ymax_abs - ymin_abs)
117 |
118 |
119 | def rect_from_opencv(rect, imsize_hw):
120 | imheight, imwidth = imsize_hw
121 | xmin_abs, ymin_abs, width_abs, height_abs = rect
122 | xmax_abs = xmin_abs + width_abs
123 | ymax_abs = ymin_abs + height_abs
124 | return {
125 | 'xmin': xmin_abs / imwidth,
126 | 'ymin': ymin_abs / imheight,
127 | 'xmax': xmax_abs / imwidth,
128 | 'ymax': ymax_abs / imheight,
129 | }
130 |
131 |
132 | if __name__ == '__main__':
133 | main()
134 |
--------------------------------------------------------------------------------
/python/oxuva/__init__.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | from oxuva.annot import *
6 | from oxuva.assess import *
7 | from oxuva.dataset import *
8 | from oxuva.io_annot import *
9 | from oxuva.io_pred import *
10 | from oxuva.io_task import *
11 | from oxuva.pred import *
12 | from oxuva.task import *
13 | from oxuva.util import *
14 |
--------------------------------------------------------------------------------
/python/oxuva/annot.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | from oxuva import util
6 |
7 |
8 | def make_track_label(category, frames, contains_cuts=None, always_visible=None):
9 | '''Creates a track annotation dictionary.'''
10 | return {
11 | 'frames': frames,
12 | 'category': category,
13 | 'contains_cuts': contains_cuts,
14 | 'always_visible': always_visible,
15 | }
16 |
17 |
18 | def make_frame_label(present=None, exemplar=None, xmin=None, ymin=None, xmax=None, ymax=None):
19 | '''Creates a frame annotation dictionary (for ground-truth).'''
20 | return {
21 | 'present': util.default_if_none(present, False),
22 | 'exemplar': util.default_if_none(exemplar, False),
23 | 'xmin': util.default_if_none(xmin, 0.0),
24 | 'xmax': util.default_if_none(xmax, 0.0),
25 | 'ymin': util.default_if_none(ymin, 0.0),
26 | 'ymax': util.default_if_none(ymax, 0.0),
27 | }
28 |
--------------------------------------------------------------------------------
/python/oxuva/assess.py:
--------------------------------------------------------------------------------
1 | '''
2 | Examples:
3 |
4 | To evaluate the prediction of a tracker for one tracking task:
5 |
6 | assessment = assess.assess_sequence(task.labels, prediction, iou_threshold=0.5)
7 |
8 | The function assess_sequence() calls subset_using_previous_if_missing() internally.
9 | This function may alternatively be called before assess_sequence().
10 | The result will be the same because subset_using_previous_if_missing() is idempotent.
11 |
12 | prediction_subset = assess.subset_using_previous_if_missing(
13 | prediction, task.labels.sorted_keys())
14 | assessment = assess.assess_sequence(
15 | task.labels, prediction_subset, iou_threshold=0.5)
16 |
17 | Since assessment is a SparseTimeSeries of frame assessments,
18 | we can consider a subset of frames:
19 |
20 | assessment_subset = util.select_interval(
21 | assessment, min_time, max_time, init_time=task.init_time)
22 |
23 | To accumulate per-frame assessments into a summary for the sequence:
24 |
25 | sequence_assessment = assess.assessment_sum(frame_assessments)
26 |
27 | This can also be used to accumulate sequence summaries for a dataset:
28 |
29 | dataset_assessment = assess.assessment_sum(sequence_assessments)
30 |
31 | To obtain the performance metrics from the summary:
32 |
33 | stats = assess.quality_metrics(dataset_assessment)
34 |
35 | Full example:
36 |
37 | assessments = {}
38 | for key in tasks:
39 | assessments[key] = assess.assess_sequence(
40 | tasks[key].labels, predictions[key], iou_threshold=0.5)
41 |
42 | sequence_assessments = {
43 | vid_obj: assess.assessment_sum(assessments[vid_obj].values())
44 | for vid_obj in assessments}
45 | dataset_assessment = assess.assessment_sum(sequence_assessments.values())
46 | return assess.quality_metrics(dataset_assessment)
47 | '''
48 |
49 | from __future__ import absolute_import
50 | from __future__ import division
51 | from __future__ import print_function
52 |
53 | import functools
54 | import itertools
55 | import json
56 | import math
57 | import numpy as np
58 | import os
59 |
60 | import logging
61 | logger = logging.getLogger(__name__)
62 |
63 | from oxuva import dataset
64 | from oxuva import io_pred
65 | from oxuva import util
66 |
67 |
68 | FRAME_RATE = 30
69 |
70 |
71 | def quality_metrics(assessment):
72 | '''Computes the TPR, TNR from TP, FP, etc.
73 |
74 | Args:
75 | assessment -- Dictionary with TP, FP, TN, FN.
76 | '''
77 | metrics = {}
78 | num_pos = assessment['TP'] + assessment['FN']
79 | num_neg = assessment['TN'] + assessment['FP']
80 | with np.errstate(invalid='ignore'):
81 | # Allow nan values in cases of 0 / 0.
82 | metrics['TPR'] = np.asfarray(assessment['TP']) / num_pos
83 | metrics['TNR'] = np.asfarray(assessment['TN']) / num_neg
84 | # TODO: Add some errorbars?
85 | metrics['GM'] = util.geometric_mean(metrics['TPR'], metrics['TNR'])
86 | metrics['MaxGM'] = max_geometric_mean_line(metrics['TNR'], metrics['TPR'], 1, 0)
87 | # Include the raw totals.
88 | metrics.update(assessment)
89 | return metrics
90 |
91 |
92 | def subset_using_previous_if_missing(series, times):
93 | '''Extracts a subset of values at the given times.
94 | If there is no data for a particular time, then the last value is used.
95 |
96 | Args:
97 | series: SparseTimeSeries of data.
98 | times: List of times.
99 |
100 | Returns:
101 | Time series sampled at specified times.
102 |
103 | Examples:
104 | >> subset_using_previous_if_missing([(2, 'hi'), (4, 'bye')], [2, 3, 4, 5])
105 | ['hi', 'hi', 'bye', 'bye']
106 |
107 | Raises an exception if asked for a time before the first element in series.
108 | '''
109 | assert isinstance(series, util.SparseTimeSeries)
110 | series = list(series.sorted_items())
111 | subset = [None for _ in times]
112 | t_curr, x_curr = None, None
113 | for i, t in enumerate(times):
114 | # Read from series until we have read all elements <= t.
115 | read_all = False
116 | while not read_all:
117 | if len(series) == 0:
118 | read_all = True
119 | else:
120 | # Peek at next element.
121 | t_next, x_next = series[0]
122 | if t_next > t:
123 | # We have gone past t.
124 | read_all = True
125 | else:
126 | # Keep going.
127 | t_curr, x_curr = t_next, x_next
128 | series = series[1:]
129 | if t_curr is None:
130 | raise ValueError('no value for time: {}'.format(t))
131 | if t_curr != t:
132 | logger.warning('no prediction for time %d: use prediction for time %s', t, t_curr)
133 | subset[i] = x_curr
134 | return util.SparseTimeSeries(zip(times, subset))
135 |
136 |
137 | def load_predictions_and_select_frames(tasks, tracker_pred_dir, permissive=False, log_prefix=''):
138 | '''Loads all predictions of a tracker and takes the subset of frames with ground truth.
139 |
140 | Args:
141 | tasks: VideoObjectDict of Tasks.
142 | tracker_pred_dir: Directory that contains files video_object.csv
143 |
144 | Returns:
145 | VideoObjectDict of SparseTimeSeries of frame assessments.
146 | '''
147 | logger.info('load predictions from "%s"', tracker_pred_dir)
148 | preds = dataset.VideoObjectDict()
149 | for track_num, vid_obj in enumerate(tasks.keys()):
150 | vid, obj = vid_obj
151 | track_name = vid + '_' + obj
152 | logger.debug(log_prefix + 'object {}/{} {}'.format(track_num + 1, len(tasks), track_name))
153 | pred_file = os.path.join(tracker_pred_dir, '{}.csv'.format(track_name))
154 | try:
155 | with open(pred_file, 'r') as fp:
156 | pred = io_pred.load_predictions_csv(fp)
157 | except IOError as exc:
158 | if permissive:
159 | logger.warning('exclude track %s: %s', track_name, str(exc))
160 | else:
161 | raise
162 | pred = subset_using_previous_if_missing(pred, tasks[vid_obj].labels.sorted_keys())
163 | preds[vid_obj] = pred
164 | return preds
165 |
166 |
167 | def assess_sequence(gt, pred, iou_threshold):
168 | '''Evaluate predicted track against ground-truth annotations.
169 |
170 | Args:
171 | gt: SparseTimeSeries of annotation dicts.
172 | pred: SparseTimeSeries of prediction dicts.
173 | iou_threshold: Threshold for determining true positive.
174 |
175 | Returns:
176 | An assessment of each frame with ground-truth.
177 | This is a TimeSeries of per-frame assessment dicts.
178 | '''
179 | times = gt.sorted_keys()
180 | # if pred.sorted_keys() != times:
181 | pred = subset_using_previous_if_missing(pred, times)
182 | return util.SparseTimeSeries({t: assess_frame(gt[t], pred[t], iou_threshold) for t in times})
183 |
184 |
185 | def make_assessment(num_frames=0, tp=0, fp=0, tn=0, fn=0, num_present=0, num_absent=0):
186 | return {'num_frames': num_frames, 'TP': tp, 'TN': tn, 'FP': fp, 'FN': fn,
187 | 'num_present': num_present, 'num_absent': num_absent}
188 |
189 |
190 | def assessment_sum(assessments):
191 | return util.dict_sum_strict(assessments, make_assessment())
192 |
193 |
194 | def assess_frame(gt, pred, iou_threshold):
195 | '''Turn prediction into TP, FP, TN, FN.
196 |
197 | Args:
198 | gt, pred: Dictionaries with fields: present, xmin, xmax, ymin, ymax.
199 |
200 | Returns:
201 | Frame assessment dict with TP, FP, etc.
202 | '''
203 | # TODO: Some other metrics may require knowledge of aspect ratio.
204 | # (This is not an issue for now since IOU is invariant).
205 |
206 | # TP: gt "present" and pred "present" and box correct
207 | # FN: gt "present" and not (pred "present" and box correct)
208 | # TN: gt "absent" and pred "absent"
209 | # FP: gt "absent" and pred "present"
210 | result = make_assessment(num_frames=1,
211 | num_present=(1 if gt['present'] else 0),
212 | num_absent=(0 if gt['present'] else 1))
213 | if gt['present']:
214 | if pred['present'] and iou_clip(gt, pred) >= iou_threshold:
215 | result['TP'] += 1
216 | else:
217 | result['FN'] += 1
218 | else:
219 | if pred['present']:
220 | result['FP'] += 1
221 | else:
222 | result['TN'] += 1
223 | if 'score' in pred:
224 | result['score'] = pred['score']
225 | return result
226 |
227 |
228 | def iou_clip(a, b):
229 | bounds = unit_rect()
230 | a = intersect(a, bounds)
231 | b = intersect(b, bounds)
232 | return iou(a, b)
233 |
234 |
235 | def iou(a, b):
236 | i = vol(intersect(a, b))
237 | u = vol(a) + vol(b) - i
238 | return float(i) / float(u)
239 |
240 |
241 | def vol(r):
242 | # Any inverted rectangle is silently considered empty.
243 | # (Allows for empty intersection.)
244 | xsize = max(0, r['xmax'] - r['xmin'])
245 | ysize = max(0, r['ymax'] - r['ymin'])
246 | return xsize * ysize
247 |
248 |
249 | def intersect(a, b):
250 | return {
251 | 'xmin': max(a['xmin'], b['xmin']),
252 | 'ymin': max(a['ymin'], b['ymin']),
253 | 'xmax': min(a['xmax'], b['xmax']),
254 | 'ymax': min(a['ymax'], b['ymax']),
255 | }
256 |
257 |
258 | def unit_rect():
259 | return {'xmin': 0.0, 'ymin': 0.0, 'xmax': 1.0, 'ymax': 1.0}
260 |
261 |
262 | def posthoc_threshold(assessments):
263 | '''Trace curve of operating points by varying score threshold.
264 |
265 | Args:
266 | assessments: List of TimeSeries of per-frame assessments.
267 | '''
268 | # Group all "present" predictions (TP, FP) by score.
269 | by_score = {}
270 | for ass in assessments:
271 | if ass['TP'] or ass['FP']:
272 | by_score.setdefault(float(ass['score']), []).append(ass)
273 |
274 | # Trace threshold from min to max.
275 | # Initially everything is labelled absent (negative).
276 | num_present = sum(ass['TP'] + ass['FN'] for ass in assessments)
277 | num_absent = sum(ass['TN'] + ass['FP'] for ass in assessments)
278 | total = {'TP': 0, 'FP': 0, 'TN': num_absent, 'FN': num_present}
279 |
280 | # Start switching the highest scores from "absent" to "present".
281 | points = []
282 | points.append(dict(total))
283 | if float('nan') in by_score:
284 | raise ValueError('score is nan but prediction is "present"')
285 | for score in sorted(by_score.keys(), reverse=True):
286 | # Iterate through the next block of points with the same score.
287 | for assessment in by_score[score]:
288 | total['TP'] += assessment['TP']
289 | total['FN'] -= assessment['TP']
290 | total['FP'] += assessment['FP']
291 | total['TN'] -= assessment['FP']
292 | points.append(dict(total))
293 | return points
294 |
295 |
296 | def max_geometric_mean_line(x1, y1, x2, y2):
297 | # Obtained using Matlab symbolic toolbox.
298 | # >> syms x1 x2 y1 y2 th
299 | # >> x = (1-th)*x1 + th*x2
300 | # >> y = (1-th)*y1 + th*y2
301 | # >> f = x * y
302 | # >> coeffs(f, th)
303 | # [ x1*y1, - y1*(x1 - x2) - x1*(y1 - y2), (x1 - x2)*(y1 - y2)]
304 | a = (x1 - x2) * (y1 - y2)
305 | b = - y1 * (x1 - x2) - x1 * (y1 - y2)
306 | # Maximize the quadratic on [0, 1].
307 | # Check endpoints.
308 | candidates = [0.0, 1.0]
309 | if a < 0:
310 | # Concave quadratic. Check if peak is in bounds.
311 | th_star = -b / (2 * a)
312 | if 0 <= th_star <= 1:
313 | candidates.append(th_star)
314 | g = lambda x, y: math.sqrt(x * y)
315 | h = lambda th: g((1 - th) * x1 + th * x2, (1 - th) * y1 + th * y2)
316 | return max([h(th) for th in candidates])
317 |
318 |
319 | def make_dataset_assessment(totals, quantized_totals, frame_assessments=None):
320 | '''Sufficient to produce all plots.
321 |
322 | This is what will be returned to the user by the evaluation server.
323 | '''
324 | return {
325 | 'totals': totals,
326 | 'quantized_totals': quantized_totals,
327 | 'frame_assessments': frame_assessments, # Ignored by dump_xxx functions!
328 | }
329 |
330 |
331 | def union_dataset_assessment(x, y):
332 | '''Combines the tracks of two datasets.'''
333 | if y is None:
334 | return x
335 | if x is None:
336 | return y
337 | return {
338 | 'totals': dataset.VideoObjectDict(dict(itertools.chain(
339 | x['totals'].items(),
340 | y['totals'].items()))),
341 | 'quantized_totals': dataset.VideoObjectDict(dict(itertools.chain(
342 | x['quantized_totals'].items(),
343 | y['quantized_totals'].items()))),
344 | }
345 |
346 |
347 | def dump_dataset_assessment_json(x, f):
348 | data = {
349 | # Convert to list of items because JSON does not support tuple as keys.
350 | 'totals': sorted(x['totals'].items()),
351 | # Convert to list of items because JSON does not support tuple as keys.
352 | # Extract elements of each QuantizedAssessment.
353 | 'quantized_totals': [(vid_obj, value.elems)
354 | for vid_obj, value in sorted(x['quantized_totals'].items())],
355 | }
356 | json.dump(data, f, sort_keys=True)
357 |
358 |
359 | def load_dataset_assessment_json(f):
360 | data = json.load(f)
361 | return make_dataset_assessment(
362 | totals=dataset.VideoObjectDict({
363 | tuple(vid_obj): total for vid_obj, total in data['totals']}),
364 | quantized_totals=dataset.VideoObjectDict({
365 | tuple(vid_obj): QuantizedAssessment({
366 | tuple(interval): total for interval, total in quantized_totals})
367 | for vid_obj, quantized_totals in data['quantized_totals']}))
368 |
369 |
370 | def assess_dataset(tasks, predictions, iou_threshold, resolution_seconds=30):
371 | '''
372 | Args:
373 | tasks: VideoObjectDict of tasks. Each task must include annotations.
374 | predictions: VideoObjectDict of predictions.
375 |
376 | Returns:
377 | Enough information to produce the plots.
378 | '''
379 | frame_assessments = dataset.VideoObjectDict({
380 | key: assess_sequence(tasks[key].labels, predictions[key], iou_threshold)
381 | for key in tasks.keys()})
382 | return make_dataset_assessment(
383 | totals=dataset.VideoObjectDict({
384 | key: assessment_sum(frame_assessments[key].values())
385 | for key in frame_assessments.keys()}),
386 | quantized_totals=dataset.VideoObjectDict({
387 | key: quantize_sequence_assessment(frame_assessments[key],
388 | init_time=tasks[key].init_time,
389 | resolution=(FRAME_RATE * resolution_seconds))
390 | for key in frame_assessments.keys()}),
391 | frame_assessments=frame_assessments)
392 |
393 |
394 | class QuantizedAssessment(object):
395 | '''Describes the assessment of intervals of a sequence.
396 |
397 | This is sufficient to construct the temporal plots
398 | without revealing whether each individual prediction is correct or not.
399 | '''
400 |
401 | def __init__(self, elems):
402 | '''
403 | Args:
404 | elems: Map from (a, b) to assessment dict.
405 | '''
406 | if isinstance(elems, dict):
407 | elems = list(elems.items())
408 | elems = sorted(elems)
409 | self.elems = elems
410 |
411 | def get(self, min_time=None, max_time=None):
412 | '''Get cumulative assessment of interval [min_time, max_time].'''
413 | # Include all bins within [min_time, max_time].
414 | subset = []
415 | for interval, value in self.elems:
416 | u, v = interval
417 | # if min_time <= u <= v <= max_time:
418 | if (min_time is None or min_time <= u) and (max_time is None or v <= max_time):
419 | subset.append(value)
420 | elif (min_time < u < max_time) or (min_time < v < max_time):
421 | # If interval is not within [min_time, max_time],
422 | # then require that it is entirely outside [min_time, max_time].
423 | raise ValueError('interval {} straddles requested {}'.format(
424 | str((u, v)), str((min_time, max_time))))
425 | return assessment_sum(subset)
426 |
427 | # def get_vector(self, intervals):
428 | # return _to_vector_dict([self.get(min_time, max_time)
429 | # for min_time, max_time in intervals])
430 |
431 |
432 | def quantize_sequence_assessment(assessment, resolution, init_time):
433 | '''
434 | Args:
435 | assessment: SparseTimeSeries of assessment dicts.
436 | resolution: Integer specifying temporal resolution.
437 | init_time: Absolute time at which tracker was started.
438 |
439 | Returns:
440 | Ordered list of ((a, b), value) elements where a, b are integers.
441 | '''
442 | if int(resolution) != resolution:
443 | logger.warning('resolution is not integer: %g', resolution)
444 | resolution = int(resolution)
445 |
446 | subsets = {}
447 | for abs_time, frame in assessment.items():
448 | t = abs_time - init_time
449 | i = int(math.ceil(t / float(resolution)))
450 | interval = resolution * (i - 1), resolution * i
451 | subsets.setdefault(interval, []).append(frame)
452 | sums = {interval: assessment_sum(subsets[interval]) for interval in subsets.keys()}
453 | return QuantizedAssessment(sorted(sums.items()))
454 |
455 |
456 | def _to_vector_dict(list_of_dicts):
457 | keys = _get_keys_and_assert_equal(list_of_dicts)
458 | vector_dict = {}
459 | for key in keys:
460 | vector_dict[key] = np.array([x[key] for x in list_of_dicts])
461 | return vector_dict
462 |
463 |
464 | def dataset_quality(totals, enable_bootstrap=True, num_trials=None, base_seed=0):
465 | '''
466 | Args:
467 | totals: VideoObjectDict of per-sequence assessment dicts.
468 | '''
469 | quality = summarize(totals.values())
470 | if enable_bootstrap:
471 | if num_trials is None:
472 | raise ValueError('must specify number of trials for bootstrap sampling')
473 | quality.update(bootstrap(summarize, totals, num_trials, base_seed=base_seed))
474 | quality = {k: np.asarray(v).tolist() for k, v in quality.items()}
475 | return quality
476 |
477 |
478 | def dataset_quality_interval(quantized_assessments, min_time=None, max_time=None,
479 | enable_bootstrap=True, num_trials=None, base_seed=0):
480 | '''
481 | Args:
482 | totals: VideoObjectDict of per-sequence assessment dicts.
483 | '''
484 | interval_totals = dataset.VideoObjectDict({
485 | track: quantized_assessments[track].get(min_time, max_time)
486 | for track in quantized_assessments.keys()})
487 | quality = summarize(interval_totals.values())
488 | if enable_bootstrap:
489 | if num_trials is None:
490 | raise ValueError('must specify number of trials for bootstrap sampling')
491 | quality.update(bootstrap(summarize, interval_totals, num_trials, base_seed=base_seed))
492 | quality = {k: np.asarray(v).tolist() for k, v in quality.items()}
493 | return quality
494 |
495 |
496 | def dataset_quality_filter(totals, require_none_absent=False, require_some_absent=False,
497 | enable_bootstrap=True, num_trials=None, base_seed=0):
498 | # Apply filter after bootstrap sampling dataset.
499 | summarize_func = functools.partial(summarize_filter,
500 | require_none_absent=require_none_absent,
501 | require_some_absent=require_some_absent)
502 | quality = summarize_func(totals.values())
503 | if enable_bootstrap:
504 | if num_trials is None:
505 | raise ValueError('must specify number of trials for bootstrap sampling')
506 | quality.update(bootstrap(summarize_func, totals, num_trials, base_seed=base_seed))
507 | quality = {k: np.asarray(v).tolist() for k, v in quality.items()}
508 | return quality
509 |
510 |
511 | def summarize(totals):
512 | '''Obtain dataset quality from per-sequence assessments.
513 |
514 | Args:
515 | totals: List of assessment dicts.
516 | '''
517 | return quality_metrics(assessment_sum(totals))
518 |
519 |
520 | def summarize_filter(totals, require_none_absent=False, require_some_absent=False):
521 | totals = [x for x in totals if
522 | (not require_none_absent or x['num_absent'] == 0) and
523 | (not require_some_absent or x['num_absent'] > 0)]
524 | return summarize(totals)
525 |
526 |
527 | def bootstrap(func, data, num_trials, base_seed=0):
528 | '''
529 | Args:
530 | func: Maps list of per-track elements to a dictionary of metrics.
531 | This will be called num_trials times.
532 | data: VideoObjectDict of elements.
533 |
534 | The function will be called func(x) where x is a list of the values in data.
535 | It would normally be called func(data.values()).
536 |
537 | VideoObjectDict is required because sampling is performed on videos not tracks.
538 | '''
539 | metrics = []
540 | for i in range(num_trials):
541 | sample = _bootstrap_sample_by_video(data, seed=(base_seed + i))
542 | logger.debug('bootstrap trial %d: num sequences %d', i + 1, len(sample))
543 | metrics.append(func(sample))
544 | return _stats_from_repetitions(metrics)
545 |
546 |
547 | def _bootstrap_sample_by_video(tracks, seed):
548 | '''Samples videos with replacement and returns a list of all tracks.
549 |
550 | Args:
551 | tracks: VideoObjectDict
552 | '''
553 | assert isinstance(tracks, dataset.VideoObjectDict)
554 | by_video = tracks.to_nested_dict()
555 | rand = np.random.RandomState(seed)
556 | names = list(by_video.keys())
557 | names_sample = rand.choice(names, len(by_video), replace=True)
558 | return list(itertools.chain.from_iterable(by_video[name].values() for name in names_sample))
559 |
560 |
561 | def _stats_from_repetitions(xs):
562 | '''Maps a list of dictionaries to the mean and variance of the values.
563 |
564 | Appends '_mean' and '_var' to the original keys.
565 | '''
566 | # Check that all dictionaries have the same keys.
567 | fields = _get_keys_and_assert_equal(xs)
568 | stats = {}
569 | stats.update({field + '_mean': np.mean([x[field] for x in xs], axis=0) for field in fields})
570 | stats.update({field + '_var': np.var([x[field] for x in xs], axis=0) for field in fields})
571 | return stats
572 |
573 |
574 | def _get_keys_and_assert_equal(xs):
575 | '''Asserts that all dictionaries have the same keys and returns the set of keys.'''
576 | assert len(xs) > 0
577 | fields = None
578 | for x in xs:
579 | curr_fields = set(x.keys())
580 | if fields is None:
581 | fields = curr_fields
582 | else:
583 | if curr_fields != fields:
584 | raise ValueError('fields differ: {} and {}'.format(fields, curr_fields))
585 | return fields
586 |
--------------------------------------------------------------------------------
/python/oxuva/dataset.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | import logging
6 | logger = logging.getLogger(__name__)
7 |
8 | from oxuva import util
9 |
10 |
11 | class VideoObjectDict(object):
12 | '''Represents map video -> object -> element.
13 | Behaves as a dictionary with keys of (video, object) tuples.
14 |
15 | Example:
16 | for key in tracks.keys():
17 | print(tracks[key])
18 |
19 | tracks = VideoObjectDict()
20 | ...
21 | for vid in tracks.videos():
22 | for obj in tracks.objects(vid):
23 | print(tracks[(vid, obj)])
24 | '''
25 |
26 | def __init__(self, elems=None):
27 | if elems is None:
28 | self._elems = dict()
29 | elif isinstance(elems, VideoObjectDict):
30 | self._elems = dict(elems._elems)
31 | else:
32 | self._elems = dict(elems)
33 |
34 | def videos(self):
35 | return set([vid for vid, obj in self._elems.keys()])
36 |
37 | def objects(self, vid):
38 | # TODO: This is somewhat inefficient if called for all videos.
39 | return [obj_i for vid_i, obj_i in self._elems.keys() if vid_i == vid]
40 |
41 | def __len__(self):
42 | return len(self._elems)
43 |
44 | def __getitem__(self, key):
45 | return self._elems[key]
46 |
47 | def __setitem__(self, key, value):
48 | self._elems[key] = value
49 |
50 | def __delitem__(self, key):
51 | del self._elems[key]
52 |
53 | def keys(self):
54 | return self._elems.keys()
55 |
56 | def values(self):
57 | return self._elems.values()
58 |
59 | def items(self):
60 | return self._elems.items()
61 |
62 | def __iter__(self):
63 | for k in self._elems.keys():
64 | yield k
65 |
66 | def to_nested_dict(self):
67 | elems = {}
68 | for (vid, obj), elem in self._elems.items():
69 | elems.setdefault(vid, {})[obj] = elem
70 | return elems
71 |
72 | def update_from_nested_dict(self, elems):
73 | for vid, vid_elems in elems.items():
74 | for obj, elem in vid_elems.items():
75 | self._elems[(vid, obj)] = elem
76 |
--------------------------------------------------------------------------------
/python/oxuva/io_annot.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | import csv
6 |
7 | from oxuva.annot import make_frame_label
8 | from oxuva.annot import make_track_label
9 | from oxuva.dataset import VideoObjectDict
10 | from oxuva import util
11 |
12 |
13 | TRACK_FIELDS = [
14 | 'video_id', 'object_id',
15 | 'class_id', 'class_name', 'contains_cuts', 'always_visible',
16 | 'frame_num', 'object_presence', 'xmin', 'xmax', 'ymin', 'ymax',
17 | ]
18 |
19 |
20 | def load_dataset_annotations_csv(fp):
21 | '''Loads the annotations for an entire dataset from one CSV file.'''
22 | reader = csv.DictReader(fp, fieldnames=TRACK_FIELDS)
23 | rows = [row for row in reader]
24 |
25 | # Group rows by object.
26 | rows_by_track = {}
27 | for row in rows:
28 | vid_id = row['video_id']
29 | obj_id = row['object_id']
30 | rows_by_track.setdefault((vid_id, obj_id), []).append(row)
31 |
32 | tracks = VideoObjectDict()
33 | for vid_obj in rows_by_track.keys():
34 | # vid_id, obj_id = vid_obj
35 | frames = util.SparseTimeSeries()
36 | for row in rows_by_track[vid_obj]:
37 | present = _parse_is_present(row['object_presence'])
38 | # TODO: Support 'exemplar' field in CSV format?
39 | t = int(row['frame_num'])
40 | frames[t] = make_frame_label(
41 | present=present,
42 | xmin=float(row['xmin']) if present else None,
43 | xmax=float(row['xmax']) if present else None,
44 | ymin=float(row['ymin']) if present else None,
45 | ymax=float(row['ymax']) if present else None)
46 | assert len(frames) >= 2
47 | first_row = rows_by_track[vid_obj][0]
48 | tracks[vid_obj] = make_track_label(
49 | category=first_row['class_name'],
50 | frames=frames,
51 | contains_cuts=first_row['contains_cuts'],
52 | always_visible=first_row['always_visible'])
53 |
54 | return tracks
55 |
56 |
57 | def dump_dataset_annotations_csv(tracks, fp):
58 | '''Writes the annotations for an entire dataset to one CSV file.'''
59 | writer = csv.DictWriter(fp, fieldnames=TRACK_FIELDS)
60 | for vid in sorted(tracks.videos()):
61 | # Sort objects by their first frame.
62 | sort_key = lambda obj: (_start_time(tracks[vid][obj]), obj)
63 | for obj in sorted(tracks.objects(vid), key=sort_key):
64 | for frame_num, frame in track['frames']:
65 | assert frame_num == int(frame_num)
66 | frame_num = int(frame_num)
67 | assert frame_num % 30 == 0
68 | # timestamp_sec = frame_num // 30
69 | class_name = track.get('category', '')
70 | class_id = CLASS_ID_LOOKUP[class_name] if class_name else ''
71 | row = {
72 | 'video_id': vid,
73 | 'object_id': obj,
74 | 'class_id': class_id,
75 | 'class_name': class_name,
76 | 'contains_cuts': _str_contains_cuts(track.get('contains_cuts', None)),
77 | 'always_visible': _str_always_visible(track.get('always_visible', None)),
78 | 'frame_num': frame_num,
79 | 'object_presence': 'present' if frame['present'] else 'absent',
80 | 'xmin': frame['xmin'],
81 | 'xmax': frame['xmax'],
82 | 'ymin': frame['ymin'],
83 | 'ymax': frame['ymax'],
84 | }
85 | writer.writerow(row)
86 |
87 |
88 | def _start_time(track):
89 | frames = track['frames']
90 | return frames.sorted_keys()[0]
91 |
92 |
93 | def _parse_is_present(s):
94 | if s == 'present':
95 | return True
96 | elif s == 'absent':
97 | return False
98 | else:
99 | raise ValueError('unknown value for presence: {}'.format(s))
100 |
101 |
102 | def _str_is_present(present):
103 | if present:
104 | return 'present'
105 | else:
106 | return 'absent'
107 |
108 |
109 | def _str_contains_cuts(contains_cuts):
110 | if contains_cuts is True:
111 | return 'true'
112 | elif contains_cuts is False:
113 | return 'false'
114 | else:
115 | return 'unknown'
116 |
117 |
118 | def _parse_contains_cuts(s):
119 | return util.str2bool_or_none(s)
120 |
121 |
122 | def _str_always_visible(always_visible):
123 | if always_visible is True:
124 | return 'true'
125 | elif always_visible is False:
126 | return 'false'
127 | else:
128 | return 'unknown'
129 |
130 |
131 | def _parse_always_visible(s):
132 | return util.str2bool_or_none(s)
133 |
--------------------------------------------------------------------------------
/python/oxuva/io_pred.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | import csv
6 |
7 | from oxuva.pred import make_prediction
8 | from oxuva import util
9 |
10 |
11 | PREDICTION_FIELD_NAMES_V1 = [
12 | 'video', 'object', 'imwidth', 'imheight',
13 | 'frame', 'score', 'present', 'xmin_pix', 'ymin_pix', 'xmax_pix', 'ymax_pix',
14 | ]
15 | PREDICTION_FIELD_NAMES_V2 = [
16 | 'video', 'object', 'frame_num', 'present', 'score', 'xmin', 'xmax', 'ymin', 'ymax',
17 | ]
18 | PREDICTION_FIELD_NAMES = PREDICTION_FIELD_NAMES_V2
19 |
20 |
21 | def load_predictions_csv(fp):
22 | '''Loads output of tracker from CSV file.
23 |
24 | Args:
25 | fp: File-like object with read() and seek().
26 |
27 | Returns:
28 | List of (time, prediction-dict) pairs.
29 | '''
30 | # has_header = csv.Sniffer().has_header(fp.read(4<<10)) # 4 kB
31 | # fp.seek(0)
32 | reader = _dict_reader_optional_fieldnames(fp, PREDICTION_FIELD_NAMES)
33 |
34 | predictions = util.SparseTimeSeries()
35 | for row in reader:
36 | present = util.str2bool(row['present'])
37 | t = int(row['frame_num'])
38 | predictions[t] = make_prediction(
39 | present=present,
40 | score=float(row['score']),
41 | xmin=float(row['xmin']) if present else None,
42 | xmax=float(row['xmax']) if present else None,
43 | ymin=float(row['ymin']) if present else None,
44 | ymax=float(row['ymax']) if present else None)
45 | return predictions
46 |
47 |
48 | def _dict_reader_optional_fieldnames(fp, fieldnames):
49 | '''Creates a csv.DictReader with the given fieldnames.
50 |
51 | The file may or may not contain a header row.
52 | If it contains a header row, it must match fieldnames.
53 |
54 | This function exists because csv.Sniffer() is unreliable.
55 | For example, it may fail if one row uses int and one row uses float.
56 | '''
57 | # Try to read headers from file.
58 | reader = csv.DictReader(fp, fieldnames=None)
59 | if set(reader.fieldnames) != set(fieldnames):
60 | fp.seek(0)
61 | reader = csv.DictReader(fp, fieldnames=fieldnames)
62 | return reader
63 |
64 |
65 | def dump_predictions_csv(vid_id, obj_id, predictions, fp):
66 | '''Writes output of tracker for a single track to a CSV file.
67 |
68 | Args:
69 | vid_id: String.
70 | obj_id: String.
71 | predictions: SparseTimeSeries of prediction dicts.
72 | fp: File-like object with write().
73 | '''
74 | writer = csv.DictWriter(fp, fieldnames=PREDICTION_FIELD_NAMES)
75 | for t, prediction in predictions.items():
76 | row = {
77 | 'video': vid_id,
78 | 'object': obj_id,
79 | 'frame_num': t,
80 | 'present': util.bool2str(prediction['present']),
81 | 'score': prediction['score'],
82 | 'xmin': prediction['xmin'],
83 | 'xmax': prediction['xmax'],
84 | 'ymin': prediction['ymin'],
85 | 'ymax': prediction['ymax'],
86 | }
87 | writer.writerow(row)
88 |
--------------------------------------------------------------------------------
/python/oxuva/io_task.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | import csv
6 |
7 | from oxuva.dataset import VideoObjectDict
8 | from oxuva.task import Task
9 |
10 |
11 | TASK_FIELDS = [
12 | 'video_id', 'object_id',
13 | 'init_frame', 'last_frame', 'xmin', 'xmax', 'ymin', 'ymax',
14 | ]
15 |
16 |
17 | def load_dataset_tasks_csv(fp):
18 | '''Loads the problem definitions for an entire dataset from one CSV file.'''
19 | reader = csv.DictReader(fp, fieldnames=TASK_FIELDS)
20 | rows = [row for row in reader]
21 |
22 | tasks = VideoObjectDict()
23 | for row in rows:
24 | key = (row['video_id'], row['object_id'])
25 | tasks[key] = Task(
26 | init_time=int(row['init_frame']),
27 | last_time=int(row['last_frame']),
28 | init_rect={
29 | 'xmin': float(row['xmin']),
30 | 'xmax': float(row['xmax']),
31 | 'ymin': float(row['ymin']),
32 | 'ymax': float(row['ymax'])})
33 | return tasks
34 |
--------------------------------------------------------------------------------
/python/oxuva/pred.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | from oxuva import util
6 |
7 |
8 | def make_prediction(present=None, score=None, xmin=None, ymin=None, xmax=None, ymax=None):
9 | '''Describes the output of a tracker in one frame.'''
10 | return {
11 | 'present': util.default_if_none(present, True),
12 | 'score': util.default_if_none(score, 0.0),
13 | 'xmin': util.default_if_none(xmin, 0.0),
14 | 'xmax': util.default_if_none(xmax, 0.0),
15 | 'ymin': util.default_if_none(ymin, 0.0),
16 | 'ymax': util.default_if_none(ymax, 0.0),
17 | }
18 |
--------------------------------------------------------------------------------
/python/oxuva/task.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | import logging
6 | logger = logging.getLogger(__name__)
7 |
8 | from oxuva import util
9 |
10 |
11 | class Task(object):
12 | '''Describes a tracking task with optional ground-truth annotations.'''
13 |
14 | def __init__(self, init_time, init_rect, labels=None, last_time=None, attributes=None):
15 | '''Create a trasking task.
16 |
17 | Args:
18 | init_time -- Time of supervision (in frames).
19 | init_rect -- Rectangle dict.
20 | labels -- SparseTimeSeries of frame annotation dicts.
21 | Does not include first frame.
22 | last_time -- Time of last frame of interest, inclusive (optional).
23 | Consider frames init_time <= t <= last_time.
24 | attributes -- Dictionary with extra attributes.
25 |
26 | If last_time is None, then the last frame of labels will be used.
27 | '''
28 | self.init_time = init_time
29 | self.init_rect = init_rect
30 | if labels:
31 | if init_time in labels:
32 | raise ValueError('labels should not contain init time')
33 | self.labels = labels
34 | if last_time is None and labels is not None:
35 | self.last_time = labels.sorted_keys()[-1]
36 | else:
37 | self.last_time = last_time
38 | self.attributes = attributes or {}
39 |
40 | def len(self):
41 | return self.last_time - self.init_time + 1
42 |
43 |
44 | def make_task_from_track(track):
45 | '''Creates a tracking task from a track annotation dict (oxuva.annot.make_track_label).
46 |
47 | The first frame is adopted as initialization.
48 | The remaining frames become the ground-truth rectangles.
49 | '''
50 | frames = list(track['frames'].sorted_items())
51 | init_time, init_annot = frames[0]
52 | labels = util.SparseTimeSeries(frames[1:])
53 | # TODO: Check that init_annot['exemplar'] is True.
54 | init_rect = {k: init_annot[k] for k in ['xmin', 'xmax', 'ymin', 'ymax']}
55 | attributes = {k: v for k, v in track.items() if k not in {'frames'}}
56 | return Task(init_time, init_rect, labels=labels, attributes=attributes)
57 |
--------------------------------------------------------------------------------
/python/oxuva/test_assess.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | import unittest
6 |
7 | import math
8 | import numpy as np
9 |
10 | from oxuva.assess import subset_using_previous_if_missing
11 | from oxuva.assess import iou
12 | from oxuva.assess import max_geometric_mean_line
13 | from oxuva.assess import posthoc_threshold
14 | from oxuva import util
15 |
16 |
17 | class TestSubsetUsingPrevious(unittest.TestCase):
18 |
19 | def test_missing(self):
20 | source = util.SparseTimeSeries({1: 'one', 3: 'three'})
21 | times = [1, 2, 3]
22 | got = subset_using_previous_if_missing(source, times)
23 | want = [(1, 'one'), (2, 'one'), (3, 'three')]
24 | self.assertEqual(list(got.sorted_items()), want)
25 |
26 | def test_beyond_end(self):
27 | source = util.SparseTimeSeries({1: 'one', 3: 'three'})
28 | times = [2, 4, 6]
29 | got = subset_using_previous_if_missing(source, times)
30 | want = [(2, 'one'), (4, 'three'), (6, 'three')]
31 | self.assertEqual(list(got.sorted_items()), want)
32 |
33 | def test_before_start(self):
34 | source = util.SparseTimeSeries({3: 'three'})
35 | times = [1, 2, 3]
36 | self.assertRaises(
37 | Exception, lambda: subset_using_previous_if_missing(source, times))
38 |
39 | def test_idempotent(self):
40 | source = util.SparseTimeSeries({1: 'one', 3: 'three'})
41 | times = [1, 2, 3, 4]
42 | once = subset_using_previous_if_missing(source, times)
43 | twice = subset_using_previous_if_missing(once, times)
44 | self.assertEqual(list(once.sorted_items()), list(twice.sorted_items()))
45 |
46 |
47 | class TestIOU(unittest.TestCase):
48 |
49 | def test_simple(self):
50 | p = {'xmin': 0.1, 'xmax': 0.4, 'ymin': 0.5, 'ymax': 0.9}
51 | q = {'xmin': 0.3, 'xmax': 0.5, 'ymin': 0.5, 'ymax': 1.0}
52 | # vol(p) is 0.3 * 0.4 = 0.12
53 | # vol(q) is 0.2 * 0.5 = 0.1
54 | # intersection is 0.1 * 0.4 = 0.04
55 | # union is (0.12 + 0.1) - 0.04 = 0.22 - 0.04 = 0.18
56 | want = 0.04 / 0.18
57 | np.testing.assert_almost_equal(iou(p, q), want)
58 | np.testing.assert_almost_equal(iou(q, p), want)
59 |
60 | def test_same(self):
61 | p = {'xmin': 0.1, 'xmax': 0.4, 'ymin': 0.6, 'ymax': 0.8}
62 | np.testing.assert_almost_equal(iou(p, p), 1.0)
63 |
64 | def test_empty_intersection(self):
65 | p = {'xmin': 0.1, 'xmax': 0.4, 'ymin': 0.6, 'ymax': 0.8}
66 | q = {'xmin': 0.4, 'xmax': 0.8, 'ymin': 0.6, 'ymax': 0.8}
67 | np.testing.assert_almost_equal(iou(p, q), 0.0)
68 |
69 |
70 | class TestMaxGeometricMeanLineSeg(unittest.TestCase):
71 |
72 | def test_simple(self):
73 | x1, y1 = 0, 1
74 | x2, y2 = 1, 0
75 | got = max_geometric_mean_line(x1, y1, x2, y2)
76 | want = 0.5
77 | np.testing.assert_almost_equal(got, want)
78 |
79 | def test_bound(self):
80 | x1, y1 = 0.3, 0.9
81 | x2, y2 = 0.7, 0.4
82 | max_val = max_geometric_mean_line(x1, y1, x2, y2)
83 | self.assertLessEqual(util.geometric_mean(x1, y1), max_val)
84 | self.assertLessEqual(util.geometric_mean(x2, y2), max_val)
85 | xm, ym = 0.5 * (x1 + x2), 0.5 * (y1 + y2)
86 | self.assertLessEqual(util.geometric_mean(xm, ym), max_val)
87 |
88 |
89 | class TestPosthocThreshold(unittest.TestCase):
90 |
91 | def test_main(self):
92 | assessments = [
93 | {'TP': 1, 'FP': 0, 'TN': 0, 'FN': 0, 'score': 40}, # true positive
94 | {'TP': 1, 'FP': 0, 'TN': 0, 'FN': 0, 'score': 30}, # false positive
95 | {'TP': 0, 'FP': 1, 'TN': 0, 'FN': 0, 'score': 20}, # true positive
96 | {'TP': 1, 'FP': 0, 'TN': 0, 'FN': 0, 'score': 10}, # true positive
97 | # Treat score zero as threshold for this imaginary classifier.
98 | {'TP': 0, 'FP': 0, 'TN': 1, 'FN': 0, 'score': -10}, # true negative
99 | {'TP': 0, 'FP': 0, 'TN': 0, 'FN': 1, 'score': -20}, # false negative
100 | {'TP': 0, 'FP': 0, 'TN': 1, 'FN': 0, 'score': -30}, # true negative
101 | ]
102 | got = posthoc_threshold(assessments)
103 | want = [
104 | {'TP': 0, 'FP': 0, 'TN': 3, 'FN': 4},
105 | {'TP': 1, 'FP': 0, 'TN': 3, 'FN': 3},
106 | {'TP': 2, 'FP': 0, 'TN': 3, 'FN': 2},
107 | {'TP': 2, 'FP': 1, 'TN': 2, 'FN': 2},
108 | {'TP': 3, 'FP': 1, 'TN': 2, 'FN': 1},
109 | ]
110 | self.assertEqual(len(got), len(want))
111 | for point_got, point_want in zip(got, want):
112 | for k in ['TP', 'FP', 'TN', 'FN']:
113 | np.testing.assert_almost_equal(point_got[k], point_want[k])
114 |
115 |
116 | if __name__ == '__main__':
117 | unittest.main()
118 |
--------------------------------------------------------------------------------
/python/oxuva/tools/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/oxuva/long-term-tracking-benchmark/fd49bed27af85bb78120598ce65397470a387520/python/oxuva/tools/__init__.py
--------------------------------------------------------------------------------
/python/oxuva/tools/analyze.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | import argparse
6 | import itertools
7 | import functools
8 | import json
9 | import numpy as np
10 | import math
11 | import os
12 | import subprocess
13 | import sys
14 |
15 | import logging
16 | logger = logging.getLogger(__name__)
17 |
18 | import matplotlib
19 | matplotlib.use('Agg')
20 | import matplotlib.pyplot as plt
21 |
22 | import oxuva
23 |
24 | #
/python/oxuva/tools/analyze.py
25 | TOOLS_DIR = os.path.dirname(os.path.realpath(__file__))
26 | REPO_DIR = os.path.realpath(os.path.join(TOOLS_DIR, '..', '..', '..'))
27 |
28 | FRAME_RATE = 30
29 | MARKERS = ['o', 'v', '^', '<', '>', 's', 'd'] # '*'
30 | CMAP_PREFERENCE = ['tab10', 'tab20', 'hsv']
31 | GRID_COLOR = '0.85' # plt.rcParams['grid.color']
32 | CLEARANCE = 1.1 # Axis range is CLEARANCE * max_value, rounded up.
33 | ARGS_FORMATTER = argparse.ArgumentDefaultsHelpFormatter # Show default values
34 | INTERVAL_TYPES = ['before', 'after', 'between']
35 | INTERVAL_AXIS_LABEL = {
36 | 'before': 'Frames before time (min)',
37 | 'after': 'Frames after time (min)',
38 | 'between': 'Frames in interval (min)',
39 | }
40 |
41 |
42 | def _add_arguments(parser):
43 | common = argparse.ArgumentParser(add_help=False)
44 | common.add_argument('--data', default='dev', help='{dev,test,devtest}')
45 | common.add_argument('--challenge', default='open',
46 | help='Assess trackers from which challenge?',
47 | choices=['constrained', 'open', 'open_minus_constrained'])
48 | # common.add_argument('--verbose', '-v', action='store_true')
49 | common.add_argument('--loglevel', default='info', choices=['info', 'debug', 'warning'])
50 | common.add_argument('--permissive', action='store_true',
51 | help='Silently exclude tracks which caused an error')
52 | # common.add_argument('--ignore_cache', action='store_true')
53 | common.add_argument('--no_use_summary', dest='use_summary', action='store_false',
54 | help='Do not load/dump assessment summaries')
55 | common.add_argument('--iou_thresholds', nargs='+', type=float, default=[0.5],
56 | help='List of IOU thresholds to use', metavar='IOU')
57 | common.add_argument('--top', type=int, default=10,
58 | help='Only show top n trackers (zero to show all)')
59 | common.add_argument('--no_bootstrap', dest='bootstrap', action='store_false',
60 | help='Disable results that require bootstrap sampling')
61 | common.add_argument('--bootstrap_trials', type=int, default=100,
62 | help='Number of trials for bootstrap sampling')
63 | common.add_argument('--errorbar_size', type=float,
64 | default=1.64485, # scipy.stats.norm.ppf(0.5 + 0.9 / 2)
65 | help='Number of standard deviations')
66 | common.add_argument('--convert_to_png', action='store_true',
67 | help='Convert PDF figures to PNG for web')
68 | common.add_argument('--png_resolution', type=int, default=150,
69 | help='Dots-per-inch for PNG conversion')
70 |
71 | plot_args = argparse.ArgumentParser(add_help=False)
72 | plot_args.add_argument('--width_inches', type=float, default=4.2)
73 | plot_args.add_argument('--height_inches', type=float, default=4.0)
74 | plot_args.add_argument('--legend_inches', type=float, default=1.3)
75 |
76 | tpr_tnr_args = argparse.ArgumentParser(add_help=False)
77 | tpr_tnr_args.add_argument('--no_level_sets', dest='level_sets', action='store_false')
78 | tpr_tnr_args.add_argument('--no_lower_bounds', dest='lower_bounds', action='store_false')
79 | tpr_tnr_args.add_argument('--asterisk', action='store_true')
80 |
81 | subparsers = parser.add_subparsers(dest='subcommand', help='Analysis mode')
82 | subparsers.required = True # https://bugs.python.org/issue9253#msg186387
83 | # table: Produce a table (one column per IOU threshold)
84 | subparser = subparsers.add_parser('table', formatter_class=ARGS_FORMATTER, parents=[common])
85 | # plot_tpr_tnr: Produce a figure (one figure per IOU threshold)
86 | subparser = subparsers.add_parser('plot_tpr_tnr', formatter_class=ARGS_FORMATTER,
87 | parents=[common, plot_args, tpr_tnr_args])
88 | # plot_tpr_tnr_intervals: Produce a figure (one figure per IOU threshold)
89 | subparser = subparsers.add_parser('plot_tpr_tnr_intervals', formatter_class=ARGS_FORMATTER,
90 | parents=[common, plot_args, tpr_tnr_args])
91 | subparser.add_argument('--times', type=int, default=[0, 60, 120, 240], help='seconds')
92 | # plot_tpr_time: Produce a figure for interval ranges (0, t) and (t, inf).
93 | subparser = subparsers.add_parser('plot_tpr_time', formatter_class=ARGS_FORMATTER,
94 | parents=[common, plot_args])
95 | subparser.add_argument('--max_time', type=int, default=600, help='seconds')
96 | subparser.add_argument('--time_step', type=int, default=60, help='seconds')
97 | subparser.add_argument('--no_same_axes', dest='same_axes', action='store_false')
98 | # plot_present_absent: Produce a figure (one figure per IOU threshold)
99 | subparser = subparsers.add_parser('plot_present_absent', formatter_class=ARGS_FORMATTER,
100 | parents=[common, plot_args])
101 |
102 |
103 | def main():
104 | parser = argparse.ArgumentParser(formatter_class=ARGS_FORMATTER)
105 | _add_arguments(parser)
106 | global args
107 | args = parser.parse_args()
108 | logging.basicConfig(level=getattr(logging, args.loglevel.upper()))
109 |
110 | dataset_names = _get_datasets(args.data)
111 | # Load tasks without annotations.
112 | dataset_tasks = {
113 | dataset: _load_tasks(os.path.join(REPO_DIR, 'dataset', 'tasks', dataset + '.csv'))
114 | for dataset in dataset_names}
115 | # Take union of all datasets.
116 | tasks = {key: task for dataset in dataset_names
117 | for key, task in dataset_tasks[dataset].items()}
118 |
119 | tracker_names = _load_tracker_names()
120 | trackers = set(tracker_names.keys())
121 | dataset_assessments = {}
122 | for dataset in dataset_names:
123 | dataset_assessments[dataset] = _get_assessments(dataset, trackers)
124 | # Take subset of trackers for which it was possible to load results.
125 | trackers = set(dataset_assessments[dataset].keys())
126 | if len(trackers) < 1:
127 | raise RuntimeError('could not obtain assessment of any trackers')
128 |
129 | # Assign colors and markers alphabetically to achieve invariance across plots.
130 | trackers = sorted(trackers, key=lambda s: s.lower())
131 | color_list = _generate_colors(len(trackers))
132 | tracker_colors = dict(zip(trackers, color_list))
133 | tracker_markers = dict(zip(trackers, itertools.cycle(MARKERS)))
134 |
135 | # Merge tracks from all datasets.
136 | # TODO: Ensure that none have same key?
137 | assessments = {}
138 | for tracker in trackers:
139 | assessments[tracker] = {}
140 | for iou in args.iou_thresholds:
141 | assessments[tracker][iou] = functools.reduce(
142 | oxuva.union_dataset_assessment,
143 | (dataset_assessments[dataset][tracker][iou] for dataset in dataset_names),
144 | None)
145 |
146 | # Use simple metrics to get ranking.
147 | rank_quality = {
148 | tracker: oxuva.dataset_quality(assessments[tracker][0.5]['totals'], enable_bootstrap=False)
149 | for tracker in trackers}
150 | trackers = _sort_quality(rank_quality)
151 | top_trackers = trackers[:args.top] if args.top else trackers
152 |
153 | if args.subcommand == 'table':
154 | _print_statistics(assessments, trackers, tracker_names)
155 | elif args.subcommand == 'plot_tpr_tnr':
156 | _plot_tpr_tnr_overall(assessments, top_trackers,
157 | tracker_names, tracker_colors, tracker_markers)
158 | elif args.subcommand == 'plot_tpr_tnr_intervals':
159 | _plot_tpr_tnr_intervals(assessments, top_trackers,
160 | tracker_names, tracker_colors, tracker_markers)
161 | elif args.subcommand == 'plot_tpr_time':
162 | for iou in args.iou_thresholds:
163 | for bootstrap in ([False, True] if args.bootstrap else [False]):
164 | _plot_intervals(assessments, top_trackers, iou, bootstrap,
165 | tracker_names, tracker_colors, tracker_markers)
166 | elif args.subcommand == 'plot_present_absent':
167 | for iou in args.iou_thresholds:
168 | for bootstrap in ([False, True] if args.bootstrap else [False]):
169 | _plot_present_absent(assessments, top_trackers, iou, bootstrap,
170 | tracker_names, tracker_colors, tracker_markers)
171 |
172 |
173 | def _get_assessments(dataset, trackers):
174 | '''
175 | Args:
176 | dataset: String that identifies dataset ("dev" or "test").
177 | trackers: List of tracker names.
178 |
179 | Returns:
180 | Dictionary that maps [tracker][iou] to dataset assessment.
181 | Only returns assessments for subset of trackers that were successful.
182 | '''
183 | # Create functions to load tasks with annotations on demand.
184 | # (Do not need annotations if using cached assessments.)
185 | # TODO: Code would be easier to read using a class with lazy-cached elements as members?
186 | get_annotations = oxuva.LazyCacheCaller(functools.partial(
187 | _load_tasks_with_annotations,
188 | os.path.join(REPO_DIR, 'dataset', 'annotations', dataset + '.csv')))
189 |
190 | assessments = {}
191 | for tracker_ind, tracker in enumerate(trackers):
192 | try:
193 | log_context = 'tracker {}/{} {}'.format(tracker_ind + 1, len(trackers), tracker)
194 | tracker_assessments = {}
195 | # Load predictions at most once for all IOU thresholds (can be slow).
196 | get_predictions = oxuva.LazyCacheCaller(
197 | lambda: oxuva.load_predictions_and_select_frames(
198 | get_annotations(),
199 | os.path.join('predictions', dataset, tracker),
200 | permissive=args.permissive,
201 | log_prefix=log_context + ': '))
202 | # Obtain results at all IOU thresholds in order to make axes equal in all graphs.
203 | # TODO: Is it unsafe to use float (iou) as dictionary key?
204 | for iou in args.iou_thresholds:
205 | logger.info('assess tracker "%s" with iou %g', tracker, iou)
206 | assess_func = lambda: oxuva.assess_dataset(get_annotations(), get_predictions(),
207 | iou, resolution_seconds=30)
208 | if args.use_summary:
209 | tracker_assessments[iou] = oxuva.cache(
210 | oxuva.Protocol(
211 | dump=oxuva.dump_dataset_assessment_json,
212 | load=oxuva.load_dataset_assessment_json, binary=False),
213 | os.path.join(
214 | 'assess', dataset, tracker, 'iou_{}.json'.format(oxuva.float2str(iou))),
215 | assess_func)
216 | else:
217 | # When it is not cached, it will include frame_assessments.
218 | # TODO: Could cache (selected frames of) predictions to file if this is slow.
219 | tracker_assessments[iou] = assess_func()
220 | except IOError as ex:
221 | logger.warning('could not obtain assessment of tracker "%s" on dataset "%s": %s',
222 | tracker, dataset, ex)
223 | else:
224 | assessments[tracker] = tracker_assessments
225 | return assessments
226 |
227 |
228 | def _load_tracker_names():
229 | with open('trackers.json', 'r') as f:
230 | trackers = json.load(f)
231 | trackers = {key: tracker for key, tracker in trackers.items()
232 | if ((args.challenge == 'open') or
233 | (args.challenge == 'constrained' and tracker['constrained']) or
234 | (args.challenge == 'open_minus_constrained' and not tracker['constrained']))}
235 | tracker_names = {key: tracker['name'] for key, tracker in trackers.items()}
236 | return tracker_names
237 |
238 |
239 | def _get_datasets(name):
240 | if name == 'devtest':
241 | return ['dev', 'test']
242 | else:
243 | return [name]
244 |
245 |
246 | def _load_tasks(fname):
247 | logger.debug('load tasks without annotations from "%s"', fname)
248 | with open(fname, 'r') as fp:
249 | return oxuva.load_dataset_tasks_csv(fp)
250 |
251 |
252 | def _load_tasks_with_annotations(fname):
253 | logger.debug('load tasks with annotations from "%s"', fname)
254 | with open(fname, 'r') as fp:
255 | # if fname.endswith('.json'):
256 | # tracks = json.load(fp)
257 | if fname.endswith('.csv'):
258 | tracks = oxuva.load_dataset_annotations_csv(fp)
259 | else:
260 | raise ValueError('unknown extension: {}'.format(fname))
261 | return oxuva.map_dict(oxuva.make_task_from_track, tracks)
262 |
263 |
264 | def _print_statistics(assessments, trackers, names=None):
265 | fields = ['TPR', 'TNR', 'GM', 'MaxGM']
266 | if args.bootstrap:
267 | # Include xxx_var keys too.
268 | fields = list(itertools.chain.from_iterable([key, key + '_var'] for key in fields))
269 | names = names or {}
270 | stats = {tracker: {iou: (
271 | oxuva.dataset_quality(assessments[tracker][iou]['totals'],
272 | enable_bootstrap=args.bootstrap, num_trials=args.bootstrap_trials))
273 | for iou in args.iou_thresholds} for tracker in trackers}
274 | table_dir = os.path.join('analysis', args.data, args.challenge)
275 | _ensure_dir_exists(table_dir)
276 | table_file = os.path.join(table_dir, 'table.csv')
277 | logger.info('write table to %s', table_file)
278 | with open(table_file, 'w') as f:
279 | fieldnames = ['tracker'] + [
280 | metric + '_' + str(iou) for iou in args.iou_thresholds for metric in fields]
281 | print(','.join(fieldnames), file=f)
282 | for tracker in trackers:
283 | row = [names.get(tracker, tracker)] + [
284 | '{:.6g}'.format(stats[tracker][iou][metric])
285 | for iou in args.iou_thresholds for metric in fields]
286 | print(','.join(row), file=f)
287 |
288 |
289 | def _plot_tpr_tnr_overall(assessments, trackers,
290 | names=None, colors=None, markers=None):
291 | bootstrap_modes = [False, True] if args.bootstrap else [False]
292 |
293 | for iou in args.iou_thresholds:
294 | for bootstrap in bootstrap_modes:
295 | for posthoc in ([False] if bootstrap else [False, True]):
296 | _plot_tpr_tnr(('tpr_tnr_iou_' + oxuva.float2str(iou) +
297 | ('_posthoc' if posthoc else '') +
298 | ('_bootstrap' if bootstrap else '')),
299 | assessments, trackers, iou, bootstrap, posthoc,
300 | names=names, colors=colors, markers=markers,
301 | min_time_sec=None, max_time_sec=None, include_score=True)
302 |
303 |
304 | def _plot_tpr_tnr_intervals(assessments, trackers,
305 | names=None, colors=None, markers=None):
306 | modes = ['before', 'after']
307 | intervals_sec = {}
308 | for mode in modes:
309 | intervals_sec[mode], _ = _make_intervals(args.times, mode)
310 |
311 | bootstrap_modes = [False, True] if args.bootstrap else [False]
312 | for bootstrap in bootstrap_modes:
313 | for iou in args.iou_thresholds:
314 | # Order by performance on all frames.
315 | stats = {tracker: oxuva.dataset_quality(assessments[tracker][iou]['totals'],
316 | enable_bootstrap=bootstrap,
317 | num_trials=args.bootstrap_trials)
318 | for tracker in trackers}
319 | order = _sort_quality(stats, use_bootstrap_mean=False)
320 |
321 | tpr_key = 'TPR_mean' if bootstrap else 'TPR'
322 | # Get stats for all plots to establish axis range.
323 | # Note: This means that dataset_quality_interval() is called twice.
324 | max_tpr = max([max([max([
325 | oxuva.dataset_quality_interval(
326 | assessments[tracker][iou]['quantized_totals'],
327 | min_time=None if min_time is None else FRAME_RATE * min_time,
328 | max_time=None if max_time is None else FRAME_RATE * max_time,
329 | enable_bootstrap=bootstrap, num_trials=args.bootstrap_trials)[tpr_key]
330 | for tracker in trackers])
331 | for min_time, max_time in intervals_sec[mode]])
332 | for mode in modes])
333 |
334 | for mode in modes:
335 | for min_time_sec, max_time_sec in intervals_sec[mode]:
336 | base_name = '_'.join(
337 | ['tpr_tnr', 'iou_' + oxuva.float2str(iou),
338 | 'interval_{}_{}'.format(oxuva.float2str(min_time_sec),
339 | oxuva.float2str(max_time_sec))] +
340 | (['bootstrap'] if bootstrap else []))
341 | _plot_tpr_tnr(base_name, assessments, trackers, iou, bootstrap, posthoc=False,
342 | min_time_sec=min_time_sec, max_time_sec=max_time_sec,
343 | max_tpr=max_tpr, order=order,
344 | names=names, colors=colors, markers=markers)
345 |
346 |
347 | def _plot_tpr_tnr(base_name, assessments, trackers, iou_threshold, bootstrap, posthoc,
348 | min_time_sec=None, max_time_sec=None, include_score=False,
349 | max_tpr=None, order=None,
350 | names=None, colors=None, markers=None, legend_kwargs=None):
351 | names = names or {}
352 | colors = colors or {}
353 | markers = markers or {}
354 | legend_kwargs = legend_kwargs or {}
355 |
356 | tpr_key = 'TPR_mean' if bootstrap else 'TPR'
357 | tnr_key = 'TNR_mean' if bootstrap else 'TNR'
358 |
359 | for iou in args.iou_thresholds:
360 | stats = {
361 | tracker: oxuva.dataset_quality_interval(
362 | assessments[tracker][iou_threshold]['quantized_totals'],
363 | min_time=None if min_time_sec is None else FRAME_RATE * min_time_sec,
364 | max_time=None if max_time_sec is None else FRAME_RATE * max_time_sec,
365 | enable_bootstrap=bootstrap, num_trials=args.bootstrap_trials)
366 | for tracker in trackers}
367 | if order is None:
368 | order = _sort_quality(stats, use_bootstrap_mean=False)
369 |
370 | plt.figure(figsize=(args.width_inches, args.height_inches))
371 | plt.xlabel('True Negative Rate (Absent)')
372 | plt.ylabel('True Positive Rate (Present)')
373 | if args.level_sets:
374 | _plot_level_sets()
375 |
376 | for tracker in order:
377 | if bootstrap:
378 | plot_func = functools.partial(
379 | _errorbar,
380 | xerr=args.errorbar_size * np.sqrt([stats[tracker]['TNR_var']]),
381 | yerr=args.errorbar_size * np.sqrt([stats[tracker]['TPR_var']]),
382 | capsize=3)
383 | else:
384 | plot_func = plt.plot
385 | plot_func([stats[tracker][tnr_key]], [stats[tracker][tpr_key]],
386 | label=_tracker_label(names.get(tracker, tracker), include_score,
387 | stats[tracker], use_bootstrap_mean=False),
388 | color=colors.get(tracker, None),
389 | marker=markers.get(tracker, None),
390 | markerfacecolor='none', markeredgewidth=2, clip_on=False)
391 | if args.lower_bounds:
392 | plt.plot(
393 | [stats[tracker][tnr_key], 1], [stats[tracker][tpr_key], 0],
394 | color=colors.get(tracker, None),
395 | linestyle='dashed', marker='')
396 |
397 | if posthoc:
398 | num_posthoc = 0
399 | for tracker in order:
400 | # Add posthoc-threshold curves to figure.
401 | # TODO: Plot distribution of post-hoc curves when using bootstrap sampling?
402 | if assessments[tracker][iou_threshold].get('frame_assessments', None) is None:
403 | logger.warning('cannot do posthoc curve for tracker "%s" at iou %g',
404 | tracker, iou_threshold)
405 | else:
406 | _plot_posthoc_curve(assessments[tracker][iou_threshold]['frame_assessments'],
407 | marker='', color=colors.get(tracker, None))
408 | num_posthoc += 1
409 | if num_posthoc == 0:
410 | logger.warning('skip posthoc plot: zero trackers')
411 | return
412 |
413 | if max_tpr is None:
414 | max_tpr = max([stats[tracker][tpr_key] for tracker in trackers])
415 | plt.xlim(xmin=0, xmax=1)
416 | plt.ylim(ymin=0, ymax=_ceil_nearest(CLEARANCE * max_tpr, 0.1))
417 | plt.grid(color=GRID_COLOR, clip_on=False)
418 | _hide_spines()
419 | plot_dir = os.path.join('analysis', args.data, args.challenge)
420 | _ensure_dir_exists(plot_dir)
421 | _save_fig(os.path.join(plot_dir, base_name + '_no_legend.pdf'))
422 | _legend_outside(**legend_kwargs)
423 | _save_fig(os.path.join(plot_dir, base_name + '.pdf'))
424 |
425 |
426 | def _plot_posthoc_curve(assessments, **kwargs):
427 | frames = list(itertools.chain.from_iterable(
428 | series.values() for series in assessments.values()))
429 | operating_points = oxuva.posthoc_threshold(frames)
430 | metrics = list(map(oxuva.quality_metrics, operating_points))
431 | plt.plot([point['TNR'] for point in metrics],
432 | [point['TPR'] for point in metrics], **kwargs)
433 |
434 |
435 | def _plot_intervals(assessments, trackers, iou_threshold, bootstrap,
436 | names=None, colors=None, markers=None):
437 | # TODO: Add errorbars using bootstrap sampling?
438 | names = names or {}
439 | colors = colors or {}
440 | markers = markers or {}
441 | times_sec = range(0, args.max_time + 1, args.time_step)
442 |
443 | # Get overall stats for order in legend.
444 | overall_stats = {tracker: oxuva.dataset_quality(assessments[tracker][iou_threshold]['totals'],
445 | enable_bootstrap=bootstrap,
446 | num_trials=args.bootstrap_trials)
447 | for tracker in trackers}
448 | order = _sort_quality(overall_stats, use_bootstrap_mean=False)
449 |
450 | intervals_sec = {}
451 | points = {}
452 | for mode in INTERVAL_TYPES:
453 | intervals_sec[mode], points[mode] = _make_intervals(times_sec, mode)
454 |
455 | stats = {mode: {tracker: [
456 | oxuva.dataset_quality_interval(
457 | assessments[tracker][iou_threshold]['quantized_totals'],
458 | min_time=None if min_time is None else FRAME_RATE * min_time,
459 | max_time=None if max_time is None else FRAME_RATE * max_time,
460 | enable_bootstrap=bootstrap, num_trials=args.bootstrap_trials)
461 | for min_time, max_time in intervals_sec[mode]]
462 | for tracker in trackers} for mode in INTERVAL_TYPES}
463 | tpr_key = 'TPR_mean' if bootstrap else 'TPR'
464 | # Find maximum TPR value over all plots (to have same axes).
465 | max_tpr = {mode: max(s[tpr_key] for tracker in trackers for s in stats[mode][tracker])
466 | for mode in INTERVAL_TYPES}
467 |
468 | for mode in INTERVAL_TYPES:
469 | plt.figure(figsize=(args.width_inches, args.height_inches))
470 | plt.xlabel(INTERVAL_AXIS_LABEL[mode])
471 | plt.ylabel('True Positive Rate')
472 | for tracker in order:
473 | tpr = [s.get(tpr_key, None) for s in stats[mode][tracker]]
474 | if bootstrap:
475 | tpr_var = [s.get('TPR_var', None) for s in stats[mode][tracker]]
476 | plot_func = functools.partial(
477 | _errorbar,
478 | yerr=args.errorbar_size * np.sqrt(tpr_var),
479 | capsize=3)
480 | else:
481 | plot_func = plt.plot
482 | plot_func(1 / 60.0 * np.asarray(points[mode]), tpr,
483 | label=names.get(tracker, tracker),
484 | marker=markers.get(tracker, None),
485 | color=colors.get(tracker, None),
486 | markerfacecolor='none', markeredgewidth=2, clip_on=False)
487 | plt.xlim(xmin=0, xmax=args.max_time / 60.0)
488 | ymax = max(max_tpr.values()) if args.same_axes else max_tpr[mode]
489 | plt.ylim(ymin=0, ymax=_ceil_nearest(CLEARANCE * ymax, 0.1))
490 | plt.grid(color=GRID_COLOR, clip_on=False)
491 | _hide_spines()
492 | plot_dir = os.path.join('analysis', args.data, args.challenge)
493 | _ensure_dir_exists(plot_dir)
494 | base_name = ('tpr_time_iou_{}_interval_{}'.format(oxuva.float2str(iou_threshold), mode) +
495 | ('_bootstrap' if bootstrap else ''))
496 | _save_fig(os.path.join(plot_dir, base_name + '_no_legend.pdf'))
497 | _legend_outside()
498 | _save_fig(os.path.join(plot_dir, base_name + '.pdf'))
499 |
500 |
501 | def _plot_present_absent(
502 | assessments, trackers, iou_threshold, bootstrap,
503 | names=None, colors=None, markers=None):
504 | names = names or {}
505 | colors = colors or {}
506 | markers = markers or {}
507 |
508 | stats_whole = {
509 | tracker: oxuva.dataset_quality(
510 | assessments[tracker][iou_threshold]['totals'],
511 | enable_bootstrap=bootstrap, num_trials=args.bootstrap_trials)
512 | for tracker in trackers}
513 | stats_all_present = {
514 | tracker: oxuva.dataset_quality_filter(
515 | assessments[tracker][iou_threshold]['totals'], require_none_absent=True,
516 | enable_bootstrap=bootstrap, num_trials=args.bootstrap_trials)
517 | for tracker in trackers}
518 | stats_any_absent = {
519 | tracker: oxuva.dataset_quality_filter(
520 | assessments[tracker][iou_threshold]['totals'], require_some_absent=True,
521 | enable_bootstrap=bootstrap, num_trials=args.bootstrap_trials)
522 | for tracker in trackers}
523 |
524 | order = _sort_quality(stats_whole)
525 | tpr_key = 'TPR_mean' if bootstrap else 'TPR'
526 | max_tpr = max(max([stats_all_present[tracker][tpr_key] for tracker in trackers]),
527 | max([stats_any_absent[tracker][tpr_key] for tracker in trackers]))
528 |
529 | plt.figure(figsize=(args.width_inches, args.height_inches))
530 | plt.xlabel('TPR (tracks without absent labels)')
531 | plt.ylabel('TPR (tracks with some absent labels)')
532 | for tracker in order:
533 | if bootstrap:
534 | plot_func = functools.partial(
535 | _errorbar,
536 | xerr=args.errorbar_size * np.sqrt([stats_all_present[tracker]['TPR_var']]),
537 | yerr=args.errorbar_size * np.sqrt([stats_any_absent[tracker]['TPR_var']]),
538 | capsize=3)
539 | else:
540 | plot_func = plt.plot
541 | plot_func(
542 | [stats_all_present[tracker][tpr_key]], [stats_any_absent[tracker][tpr_key]],
543 | label=names.get(tracker, tracker),
544 | color=colors.get(tracker, None),
545 | marker=markers.get(tracker, None),
546 | markerfacecolor='none', markeredgewidth=2, clip_on=False)
547 | plt.xlim(xmin=0, xmax=_ceil_nearest(CLEARANCE * max_tpr, 0.1))
548 | plt.ylim(ymin=0, ymax=_ceil_nearest(CLEARANCE * max_tpr, 0.1))
549 | plt.grid(color=GRID_COLOR, clip_on=False)
550 | _hide_spines()
551 | # Draw a diagonal line.
552 | plt.plot([0, 1], [0, 1], color=GRID_COLOR, linewidth=1, linestyle='dotted')
553 | plot_dir = os.path.join('analysis', args.data, args.challenge)
554 | _ensure_dir_exists(plot_dir)
555 | base_name = ('present_absent_iou_{}'.format(oxuva.float2str(iou_threshold)) +
556 | ('_bootstrap' if bootstrap else ''))
557 | _save_fig(os.path.join(plot_dir, base_name + '_no_legend.pdf'))
558 | _legend_outside()
559 | _save_fig(os.path.join(plot_dir, base_name + '.pdf'))
560 |
561 |
562 | def _make_intervals(values, interval_type):
563 | '''Produces intervals and points at which to plot them.
564 |
565 | Returns:
566 | intervals, points
567 |
568 | Example:
569 | >> _make_intervals([0, 1, 2, 3], 'before')
570 | [(0, 1), (0, 2), (0, 3)], [1, 2, 3]
571 |
572 | >> _make_intervals([0, 1, 2, 3], 'after')
573 | [(0, inf), (1, inf), (2, inf), (3, inf)], [1, 2, 3]
574 |
575 | >> _make_intervals([0, 1, 2, 3], 'between')
576 | [(0, 1), (1, 2), (2, 3)], [0.5, 1.5, 2.5]
577 | '''
578 | if interval_type == 'before':
579 | intervals = [(0, x) for x in values if x > 0]
580 | points = [x for x in values if x > 0]
581 | elif interval_type == 'after':
582 | intervals = [(x, float('inf')) for x in values]
583 | points = list(values)
584 | elif interval_type == 'between':
585 | intervals = list(zip(values, values[1:]))
586 | points = [0.5 * (a + b) for a, b in intervals]
587 | return intervals, points
588 |
589 |
590 | def _ensure_dir_exists(dir):
591 | if not os.path.exists(dir):
592 | os.makedirs(dir)
593 |
594 |
595 | def _generate_colors(n):
596 | # return [colorsys.hsv_to_rgb(i / n, s, v) for i in range(n)]
597 | for cmap_name in CMAP_PREFERENCE:
598 | cmap = matplotlib.cm.get_cmap(cmap_name)
599 | if n <= cmap.N:
600 | break
601 | return [cmap(float(i) / n) for i in range(n)]
602 |
603 |
604 | def _save_fig(plot_file):
605 | logger.info('write plot to %s', plot_file)
606 | plt.savefig(plot_file)
607 |
608 | if args.convert_to_png:
609 | name, ext = os.path.splitext(plot_file)
610 | if not ext.lower() == '.pdf':
611 | raise ValueError('plot file does not have pdf extension: {:s}'.format(plot_file))
612 | logger.debug('convert to png: %s', plot_file)
613 | png_file = name + '.png'
614 | try:
615 | subprocess.check_call(['convert', '-density', str(args.png_resolution), plot_file,
616 | '-quality', '90', png_file])
617 | except subprocess.CalledProcessError as ex:
618 | logger.warning('could not convert to png: %s', ex)
619 |
620 |
621 | def _plot_level_sets(n=10, num_points=100):
622 | x = np.linspace(0, 1, num_points + 1)[1:]
623 | for gm in np.asfarray(range(1, n)) / n:
624 | # gm = sqrt(x*y); y = gm^2 / x
625 | y = gm**2 / x
626 | plt.plot(x, y, color=GRID_COLOR, linewidth=1, linestyle='dotted')
627 |
628 |
629 | def _ceil_nearest(x, step):
630 | '''Rounds up to nearest multiple of step.'''
631 | return math.ceil(x / step) * step
632 |
633 |
634 | def _sort_quality(quality, use_bootstrap_mean=False):
635 | '''
636 | Args:
637 | quality: Dict that maps tracker name to quality dict.
638 | '''
639 | def sort_key(tracker):
640 | return _quality_sort_key(quality[tracker],
641 | use_bootstrap_mean=use_bootstrap_mean)
642 |
643 | return sorted(quality.keys(), key=sort_key, reverse=True)
644 |
645 |
646 | def _quality_sort_key(stats, use_bootstrap_mean=False):
647 | if use_bootstrap_mean:
648 | return (stats['MaxGM_mean'], stats['TPR_mean'], stats['TNR_mean'])
649 | else:
650 | return (stats['MaxGM'], stats['TPR'], stats['TNR'])
651 |
652 |
653 | def _tracker_label(name, include_score, stats, use_bootstrap_mean):
654 | if not include_score:
655 | return name
656 | gm_key = 'GM_mean' if use_bootstrap_mean else 'GM'
657 | max_gm_key = 'MaxGM_mean' if use_bootstrap_mean else 'MaxGM'
658 | max_at_point = abs(stats[gm_key] - stats[max_gm_key]) <= 1e-3
659 | asterisk = '*' if args.asterisk and max_at_point else ''
660 | return '{} ({:.2f}{})'.format(name, stats[max_gm_key], asterisk)
661 |
662 |
663 | def _errorbar(*args, **kwargs):
664 | container = plt.errorbar(*args, **kwargs)
665 | # Disable clipping for caps of errorbars.
666 | _, caplines, barlinecols = container
667 | for capline in caplines:
668 | capline.set_clip_on(False)
669 | for barlinecol in barlinecols:
670 | barlinecol.set_clip_on(False)
671 | # return container
672 |
673 |
674 | def _hide_spines():
675 | ax = plt.gca()
676 | ax.spines['left'].set_visible(False)
677 | ax.spines['right'].set_visible(False)
678 | ax.spines['top'].set_visible(False)
679 | ax.spines['bottom'].set_visible(False)
680 | plt.tick_params(axis='both', which='both', top=False, bottom=False, left=False, right=False)
681 |
682 |
683 | def _legend_no_errorbars(**kwargs):
684 | '''Replaces plt.legend(). Excludes errorbars from legend.'''
685 | # https://swdg.io/2015/errorbar-legends/
686 | ax = plt.gca()
687 | handles, labels = ax.get_legend_handles_labels()
688 | handles = [h[0] if isinstance(h, matplotlib.container.ErrorbarContainer) else h
689 | for h in handles]
690 | ax.legend(handles, labels, **kwargs)
691 |
692 |
693 | def _legend_outside(**kwargs):
694 | fig = plt.gcf()
695 | fig.set_size_inches(args.width_inches + args.legend_inches, args.height_inches)
696 | frac = float(args.width_inches) / (args.width_inches + args.legend_inches)
697 | ax = plt.gca()
698 | box = ax.get_position()
699 | ax.set_position([box.x0, box.y0, box.width * frac, box.height])
700 | _legend_no_errorbars(loc='center left', bbox_to_anchor=(1.02, 0.5), **kwargs)
701 |
702 |
703 | if __name__ == '__main__':
704 | main()
705 |
--------------------------------------------------------------------------------
/python/oxuva/tools/visualize.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | import argparse
6 | import contextlib
7 | import os
8 | import shutil
9 | import subprocess
10 | import tempfile
11 | from PIL import Image, ImageDraw, ImageColor, ImageFont
12 |
13 | import logging
14 | logger = logging.getLogger(__name__)
15 |
16 | import oxuva
17 |
18 |
19 | # /python/oxuva/tools/analyze.py
20 | TOOLS_DIR = os.path.dirname(os.path.realpath(__file__))
21 | REPO_DIR = os.path.realpath(os.path.join(TOOLS_DIR, '..', '..', '..'))
22 | DATA_DIR = os.path.join(REPO_DIR, 'dataset')
23 |
24 |
25 | def _add_arguments(parser):
26 | parser.add_argument('tracker', help='Name of tracker to visualize')
27 | parser.add_argument('--loglevel', default='info', choices=['info', 'debug', 'warning'])
28 | parser.add_argument('--data', default='dev', choices=['dev', 'test'], help='{dev,test}')
29 | # TODO: Allow user to specify single video?
30 | # TODO: Plot multiple trackers together?
31 |
32 |
33 | def main():
34 | parser = argparse.ArgumentParser()
35 | _add_arguments(parser)
36 | args = parser.parse_args()
37 | logging.basicConfig(level=getattr(logging, args.loglevel.upper()))
38 |
39 | tasks_file = os.path.join(DATA_DIR, 'tasks', args.data + '.csv')
40 | images_dir = os.path.join(DATA_DIR, 'images', args.data)
41 | predictions_dir = os.path.join('predictions', args.data, args.tracker)
42 | output_dir = os.path.join('visualize', args.data, args.tracker)
43 | if not os.path.exists(output_dir):
44 | os.makedirs(output_dir, 0o755)
45 |
46 | with open(tasks_file, 'r') as fp:
47 | tasks = oxuva.load_dataset_tasks_csv(fp)
48 |
49 | for i, (key, task) in enumerate(sorted(tasks.items())):
50 | vid, obj = key
51 | logger.info('task %d/%d: (%s, %s)', i + 1, len(tasks), vid, obj)
52 | video_file = os.path.join(output_dir, '{}_{}.mp4'.format(vid, obj))
53 | if os.path.exists(video_file):
54 | logger.debug('skip %s %s: already exists', vid, obj)
55 | continue
56 | try:
57 | predictions_file = os.path.join(predictions_dir, '{}_{}.csv'.format(vid, obj))
58 | with open(predictions_file, 'r') as fp:
59 | predictions = oxuva.load_predictions_csv(fp)
60 | image_file_func = lambda t: os.path.join(images_dir, vid, '{:06d}.jpeg'.format(t))
61 | output_file = os.path.join(output_dir, '{}_{}.mp4'.format(vid, obj))
62 | _visualize_video(task, predictions, image_file_func, output_file)
63 | except (IOError, subprocess.CalledProcessError) as ex:
64 | logger.warning('could not visualize %s %s: %s', vid, obj, ex)
65 |
66 |
67 | def _visualize_video(task, predictions, image_file_func, output_file):
68 | times = list(range(task.init_time, task.last_time + 1))
69 | predictions = oxuva.subset_using_previous_if_missing(predictions, times[1:])
70 |
71 | with _make_temp_dir(prefix='tmp-visualize-') as tmp_dir:
72 | logger.debug('write image files to %s', tmp_dir)
73 | # TODO: Delete temporary directory?
74 | pattern = os.path.join(tmp_dir, '%06d.jpeg') # Messy but used by ffmpeg.
75 |
76 | for i, t in enumerate(times):
77 | im = Image.open(image_file_func(t))
78 | draw = ImageDraw.Draw(im)
79 | if t == task.init_time:
80 | # Draw initial rectangle.
81 | draw.rectangle(_pil_rect(task.init_rect, im.size), outline=_get_color('green'))
82 | else:
83 | # Draw predicted rectangle.
84 | draw.rectangle(_pil_rect(predictions[t], im.size), outline=_get_color('yellow'))
85 | del draw
86 | im.save(pattern % i)
87 |
88 | # TODO: Make video with ffmpeg.
89 | tmp_output_file = _tmp_name(output_file)
90 | command = _ffmpeg_command(['-i', pattern,
91 | '-r', '30', # fps
92 | '-vf', 'scale=trunc(iw/2)*2:trunc(ih/2)*2',
93 | tmp_output_file])
94 | subprocess.check_call(command)
95 | os.rename(tmp_output_file, output_file)
96 |
97 |
98 | def _pil_rect(rect, size_xy):
99 | width, height = size_xy
100 | xmin = int(round(rect['xmin'] * width))
101 | xmax = int(round(rect['xmax'] * width))
102 | ymin = int(round(rect['ymin'] * height))
103 | ymax = int(round(rect['ymax'] * height))
104 | return [(xmin, ymin), (xmax, ymax)]
105 |
106 |
107 | # https://github.com/mrmrs/colors/blob/master/js/colors.js
108 | NICE_COLORS = {
109 | 'aqua': '#7fdbff',
110 | 'blue': '#0074d9',
111 | 'lime': '#01ff70',
112 | 'navy': '#001f3f',
113 | 'teal': '#39cccc',
114 | 'olive': '#3d9970',
115 | 'green': '#2ecc40',
116 | 'red': '#ff4136',
117 | 'maroon': '#85144b',
118 | 'orange': '#ff851b',
119 | 'purple': '#b10dc9',
120 | 'yellow': '#ffdc00',
121 | 'fuchsia': '#f012be',
122 | 'gray': '#aaaaaa',
123 | 'white': '#ffffff',
124 | 'black': '#111111',
125 | 'silver': '#dddddd',
126 | }
127 |
128 |
129 | def _get_color(name):
130 | return ImageColor.getrgb(NICE_COLORS[name])
131 |
132 |
133 | def _ffmpeg_command(args):
134 | return (
135 | ['ffmpeg',
136 | '-loglevel', 'error', # Quiet.
137 | '-y', # Overwrite output without asking.
138 | '-nostdin', # No interaction with user (q to quit).
139 | ] + args)
140 |
141 |
142 | def _tmp_name(fname):
143 | head, tail = os.path.split(fname)
144 | return os.path.join(head, 'tmp_' + tail)
145 |
146 |
147 | @contextlib.contextmanager
148 | def _make_temp_dir(*args, **kwargs):
149 | temp_dir = tempfile.mkdtemp(*args, **kwargs)
150 | try:
151 | yield temp_dir
152 | finally:
153 | shutil.rmtree(temp_dir)
154 |
155 |
156 | if __name__ == '__main__':
157 | main()
158 |
--------------------------------------------------------------------------------
/python/oxuva/util.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from __future__ import division
3 | from __future__ import print_function
4 |
5 | import collections
6 | import functools
7 | import json
8 | import numpy as np
9 | import os
10 | import pickle
11 | import sys
12 |
13 | import logging
14 | logger = logging.getLogger(__name__)
15 |
16 |
17 | def str2bool(x):
18 | x = x.strip().lower()
19 | if x in ['t', 'true', 'y', 'yes', '1']:
20 | return True
21 | if x in ['f', 'false', 'n', 'no', '0']:
22 | return False
23 | raise ValueError('warning: unclear value: {}'.format(x))
24 |
25 |
26 | def str2bool_or_none(x):
27 | try:
28 | return str2bool(x)
29 | except ValueError:
30 | return None
31 |
32 |
33 | def bool2str(x):
34 | return str(x).lower()
35 |
36 |
37 | def default_if_none(x, value):
38 | return value if x is None else x
39 |
40 |
41 | def harmonic_mean(*args):
42 | assert all([x >= 0 for x in args])
43 | if any([x == 0 for x in args]):
44 | return 0.
45 | return np.asscalar(1. / np.mean(1. / np.asfarray(args)))
46 |
47 |
48 | def geometric_mean(*args):
49 | with np.errstate(divide='ignore'):
50 | # log(zero) leads to -inf
51 | # log(negative) leads to nan
52 | # log(nan) leads to nan
53 | # nan + anything is nan
54 | # -inf + (any finite value) is -inf
55 | # exp(-inf) is 0
56 | return np.exp(np.mean(np.log(args))).tolist()
57 |
58 |
59 | def cache(protocol, filename, func, makedir=True, ignore_existing=False):
60 | '''Caches the result of a function in a file.
61 |
62 | Args:
63 | func -- Function with no arguments.
64 | makedir -- Create parent directory if it does not exist.
65 | ignore_existing -- Ignore existing cache file and call function.
66 | If it existed, the old cache file will be over-written.
67 | '''
68 | if (not ignore_existing) and os.path.exists(filename):
69 | logger.info('load from cache: %s', filename)
70 | with open(filename, 'rb' if protocol.binary else 'r') as r:
71 | result = protocol.load(r)
72 | else:
73 | logger.debug('cache file not found: %s', filename)
74 | dir = os.path.dirname(filename)
75 | if makedir and (not os.path.exists(dir)):
76 | os.makedirs(dir)
77 | result = func()
78 | # Write to a temporary file and then perform atomic rename.
79 | # This guards against partial cache files.
80 | tmp = filename + '.tmp'
81 | with open(tmp, 'wb' if protocol.binary else 'w') as w:
82 | protocol.dump(result, w)
83 | os.rename(tmp, filename)
84 | return result
85 |
86 |
87 | Protocol = collections.namedtuple('Protocol', ['dump', 'load', 'binary'])
88 | protocol_json = Protocol(dump=functools.partial(json.dump, sort_keys=True),
89 | load=json.load, binary=False)
90 | protocol_pickle = Protocol(dump=pickle.dump, load=pickle.load, binary=True)
91 |
92 | cache_json = functools.partial(cache, protocol_json)
93 | cache_pickle = functools.partial(cache, protocol_pickle)
94 |
95 |
96 | def dict_sum(xs, initializer=None):
97 | if initializer is None:
98 | total = {}
99 | else:
100 | total = dict(initializer)
101 | for x in xs:
102 | for k, v in x.items():
103 | if k in total:
104 | total[k] += v
105 | else:
106 | total[k] = v
107 | return total
108 |
109 |
110 | def dict_sum_strict(xs, initializer):
111 | total = dict(initializer)
112 | for x in xs:
113 | for k in initializer.keys():
114 | total[k] += x[k]
115 | return total
116 |
117 |
118 | def map_dict(f, x):
119 | return {k: f(v) for k, v in x.items()}
120 |
121 |
122 | class SparseTimeSeries(object):
123 | '''Dictionary with integer keys in sorted order.'''
124 |
125 | def __init__(self, frames=None):
126 | self._frames = {} if frames is None else dict(frames)
127 |
128 | def __len__(self):
129 | return len(self._frames)
130 |
131 | def __getitem__(self, t):
132 | return self._frames[t]
133 |
134 | def __setitem__(self, t, value):
135 | self._frames[t] = value
136 |
137 | def __delitem__(self, t):
138 | del self._frames[t]
139 |
140 | def get(self, t, default):
141 | return self._frames.get(t, default)
142 |
143 | def setdefault(self, t, default):
144 | return self._frames.setdefault(t, default)
145 |
146 | def keys(self):
147 | return self._frames.keys()
148 |
149 | def sorted_keys(self):
150 | '''Returns times in sequential order.'''
151 | return sorted(self._frames.keys())
152 |
153 | def values(self):
154 | return self._frames.values()
155 |
156 | def sorted_items(self):
157 | '''Returns (time, value) pairs in sequential order.'''
158 | times = sorted(self._frames.keys())
159 | return zip(times, [self._frames[t] for t in times])
160 |
161 | def items(self):
162 | return self._frames.items()
163 |
164 | def __iter__(self):
165 | for t in sorted(self._frames.keys()):
166 | yield t
167 |
168 | def __contains__(self, t):
169 | return t in self._frames
170 |
171 |
172 | def select_interval(series, min_time=None, max_time=None, init_time=0):
173 | return SparseTimeSeries({
174 | t: x for t, x in series.items()
175 | if ((min_time is None or min_time <= t - init_time) and
176 | (max_time is None or t - init_time <= max_time))})
177 |
178 |
179 | class LazyCacheCaller(object):
180 |
181 | def __init__(self, func):
182 | self.func = func
183 | self.evaluated = False
184 | self.result = None
185 |
186 | def __call__(self):
187 | if not self.evaluated:
188 | self.result = self.func()
189 | self.evaluated = True
190 | return self.result
191 |
192 |
193 | def float2str(x):
194 | return str(x).replace('.', 'd')
195 |
--------------------------------------------------------------------------------
/pythonpath.sh:
--------------------------------------------------------------------------------
1 | dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
2 | export PYTHONPATH="$dir/python:$PYTHONPATH"
3 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy
2 | matplotlib
3 |
--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
1 | [pycodestyle]
2 | ignore = E501,E402,E731
3 |
--------------------------------------------------------------------------------