├── waymo2argo ├── __init__.py ├── launch_all_trackers.py ├── transform_utils.py ├── dump_waymo_persweep_detections.py ├── create_submission_bin_file.py ├── waymo_dets_to_argoverse.py ├── waymo_raw_data_to_argoverse.py └── waymo_data_splits.py ├── .gitignore ├── run_tracker.sh ├── requirements.txt ├── setup.py ├── submission.txtpb ├── .github └── workflows │ └── test-python.yml ├── LICENSE ├── tests ├── test_transform_utils.py └── test_waymo_raw_data_to_argoverse.py └── README.md /waymo2argo/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | */.vscode/* 3 | */build/* 4 | */dist/* 5 | *.egg-info 6 | *.ply -------------------------------------------------------------------------------- /run_tracker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SPLIT=$1 4 | DETS_DATAROOT=$2 5 | POSE_DIR=$3 6 | TRACKS_DUMP_DIR=$4 7 | MIN_CONF=$5 8 | MIN_HITS=$6 9 | 10 | python -u run_ab3dmot.py --split $SPLIT \ 11 | --dets_dataroot $DETS_DATAROOT \ 12 | --pose_dir $POSE_DIR \ 13 | --tracks_dump_dir $TRACKS_DUMP_DIR \ 14 | --min_conf $MIN_CONF \ 15 | --min_hits $MIN_HITS -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Note: all version numbers may be removed, except for `waymo_open_dataset_tf_2_1_0` 2 | argoverse @ git+https://github.com/argoai/argoverse-api.git@master 3 | imageio # ==2.9.0 4 | waymo_open_dataset_tf_2_1_0==1.2.0 5 | pandas # ==1.1.1 6 | opencv_python # ==4.4.0.42 7 | # tensorflow_gpu==2.5.1 8 | numpy # ==1.19.1 9 | scipy # ==1.4.1 10 | pyntcloud==0.1.2 11 | # tensorflow==2.5.1 12 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | with open('requirements.txt') as f: 7 | requirements = f.read().splitlines() 8 | 9 | # dependency_links not needed, install_requires sufficient 10 | # per PEP 508 https://www.python.org/dev/peps/pep-0508/ 11 | # and https://stackoverflow.com/a/54216163 12 | 13 | setup( 14 | name="waymo2argo", 15 | version="0.0.1", 16 | long_description=long_description, 17 | classifiers=[ 18 | "Programming Language :: Python :: 3", 19 | "Operating System :: POSIX", 20 | ], 21 | packages=find_packages(exclude=["tests"]), 22 | install_requires=requirements 23 | ) 24 | -------------------------------------------------------------------------------- /submission.txtpb: -------------------------------------------------------------------------------- 1 | # Modify this file to fill in your information and then 2 | # run create_submission to generate a submission file. 3 | 4 | task: TRACKING_3D 5 | account_name: "johnwlambert@gmail.com" 6 | # Change this to your unique method name. Max 25 chars. 7 | unique_method_name: "PPBA AB3DMOT" 8 | 9 | authors: "John Lambert" 10 | authors: "" 11 | 12 | affiliation: "Georgia Tech" 13 | description: "AB3DMOT-style KF on provided PointPillars (PPBA) detections" 14 | 15 | method_link: "https://github.com/johnwlambert/waymo_to_argoverse" 16 | 17 | # See submission.proto for allowed types. 18 | sensor_type: LIDAR_ALL 19 | 20 | number_past_frames_exclude_current: 0 21 | number_future_frames_exclude_current: 0 22 | 23 | object_types: TYPE_VEHICLE 24 | object_types: TYPE_PEDESTRIAN 25 | object_types: TYPE_CYCLIST 26 | 27 | # Inference latency in seconds. 28 | latency_second: -1 -------------------------------------------------------------------------------- /.github/workflows/test-python.yml: -------------------------------------------------------------------------------- 1 | name: Python CI 2 | 3 | # Run this workflow every time a new commit is pushed to repository 4 | on: [pull_request] 5 | 6 | jobs: 7 | run-unit-tests: 8 | 9 | name: Run all unit tests in codebase 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Set up Python 3.7 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: 3.7 18 | 19 | - name: Install dependencies 20 | run: | 21 | pip install -r requirements.txt 22 | 23 | - name: Test with pytest 24 | run: | 25 | pip install -e . 26 | pip install pytest 27 | pytest tests/ 28 | 29 | - name: Flake check 30 | run: | 31 | pip install flake8 32 | flake8 --max-line-length 120 --ignore E201,E202,E203,E231,W291,W293,E303,W391,E402,W503,E731 waymo2argo 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 John Lambert (johnlambert@gatech.edu) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to use it 5 | for purely noncommercial, research purposes, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all 8 | copies or substantial portions of the Software. 9 | 10 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 11 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 12 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 13 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 14 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 15 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 16 | SOFTWARE. 17 | 18 | Anyone using Waymo Open Dataset data or their devkit code must abide by their terms and conditions: https://waymo.com/open/terms/ 19 | -------------------------------------------------------------------------------- /waymo2argo/launch_all_trackers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from waymo_data_splits import get_val_log_ids, get_test_log_ids 4 | 5 | import mseg_semantic.utils.subprocess_utils as subprocess_utils 6 | 7 | 8 | def launch_all_trackers() -> None: 9 | """Run a hyperparameter sweep over different 3d object tracking settings.""" 10 | split = "test" # 'val' 11 | dets_dataroot = "/w_o_d/detections" 12 | 13 | min_conf = 0.475 14 | min_hits = 6 15 | 16 | tracks_dump_dir = f"/w_o_d/ab3dmot_tracks_conf{min_conf}_complete_sharded_{split}_minhits{min_hits}" 17 | 18 | if split == "val": 19 | log_ids = get_val_log_ids() 20 | elif split == "test": 21 | log_ids = get_test_log_ids() 22 | 23 | for log_id in log_ids: 24 | pose_dir = f"/w_o_d/{split.upper()}_RAW_DATA_SHARDED/sharded_pose_logs/{split}_{log_id}" 25 | 26 | cmd = "sbatch -p cpu -c 5" 27 | cmd += f" run_tracker.sh {split} {dets_dataroot} {pose_dir} {tracks_dump_dir} {min_conf} {min_hits}" 28 | print(cmd) 29 | subprocess_utils.run_command(cmd) 30 | 31 | 32 | if __name__ == "__main__": 33 | launch_all_trackers() 34 | -------------------------------------------------------------------------------- /tests/test_transform_utils.py: -------------------------------------------------------------------------------- 1 | """Unit tests on coordinate transform utilities.""" 2 | 3 | import numpy as np 4 | 5 | import waymo2argo.transform_utils as transform_utils 6 | 7 | 8 | def test_transform() -> None: 9 | """ """ 10 | yaw = 90 11 | for yaw in np.random.randn(10) * 360: 12 | R = transform_utils.rotMatZ_3D(yaw) 13 | w, x, y, z = transform_utils.rotmat2quat(R) 14 | qx, qy, qz, qw = transform_utils.yaw_to_quaternion3d(yaw) 15 | 16 | print(w, qw) 17 | print(x, qx) 18 | print(y, qy) 19 | print(z, qz) 20 | # assert np.allclose(w, qw) 21 | # assert np.allclose(x, qx) 22 | # assert np.allclose(y, qy) 23 | # assert np.allclose(z, qz) 24 | 25 | 26 | def test_cycle() -> None: 27 | """ """ 28 | R = np.eye(3) 29 | q = transform_utils.rotmat2quat(R) 30 | R_cycle = transform_utils.quat2rotmat(q) 31 | assert np.allclose(R, R_cycle) 32 | 33 | 34 | def test_quaternion3d_to_yaw() -> None: 35 | """ """ 36 | num_trials = 100000 37 | for yaw in np.linspace(-np.pi, np.pi, num_trials): 38 | qx, qy, qz, qw = transform_utils.yaw_to_quaternion3d(yaw) 39 | q_argo = np.array([qw, qx, qy, qz]) 40 | new_yaw = transform_utils.quaternion3d_to_yaw(q_argo) 41 | assert np.isclose(yaw, new_yaw) 42 | if not np.allclose(yaw, new_yaw): 43 | print(yaw, new_yaw) 44 | -------------------------------------------------------------------------------- /tests/test_waymo_raw_data_to_argoverse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import numpy as np 3 | from unittest.mock import patch, Mock 4 | 5 | from argoverse.utils.ply_loader import load_ply 6 | 7 | from waymo2argo.waymo_raw_data_to_argoverse import ( 8 | build_argo_label, 9 | dump_point_cloud, 10 | get_log_ids_from_files, 11 | round_to_micros 12 | ) 13 | 14 | def test_round_to_micros(): 15 | """ 16 | test_round_to_micros() 17 | """ 18 | t_nanos = 1508103378165379072 19 | t_micros = 1508103378165379000 20 | assert t_micros == round_to_micros(t_nanos, base=1000) 21 | 22 | 23 | @patch("waymo2argo.waymo_raw_data_to_argoverse.glob") 24 | def test_get_log_ids_from_files(mock_glob): 25 | mock_glob.glob.return_value = [ 26 | "data/segment-123.tfrecord", 27 | "data/segment-456_with_camera_labels.tfrecord", 28 | "data/segment-789.tfrecord", 29 | ] 30 | actual_log_ids = get_log_ids_from_files("data") 31 | expected_log_ids = { 32 | "123": "data/segment-123.tfrecord", 33 | "456": "data/segment-456_with_camera_labels.tfrecord", 34 | "789": "data/segment-789.tfrecord", 35 | } 36 | assert actual_log_ids == expected_log_ids 37 | 38 | 39 | def test_dump_point_cloud(): 40 | points = np.array([[3, 4, 5], [2, 4, 1], [1, 5, 2], [5, 2, 1]]) 41 | test_dir = "test_dir" 42 | timestamp = 0 43 | log_id = 123 44 | dump_point_cloud(points, timestamp, log_id, test_dir) 45 | file_name = "test_dir/123/lidar/PC_0.ply" 46 | ret_pts = load_ply(file_name) 47 | assert np.array_equal(points, ret_pts) 48 | 49 | 50 | def test_build_argo_label(): 51 | mock_label = Mock() 52 | mock_label.box.center_x = 5 53 | mock_label.box.center_y = 5 54 | mock_label.box.center_z = 5 55 | mock_label.box.length = 10 56 | mock_label.box.width = 10 57 | mock_label.box.height = 10 58 | mock_label.box.heading = 0 59 | mock_label.id = "100" 60 | mock_label.type = 1 61 | track_ids = {"100": "123"} 62 | timestamp = "55" 63 | expected_label = { 64 | "center": {"x": 5, "y": 5, "z": 5}, 65 | "length": 10, 66 | "width": 10, 67 | "height": 10, 68 | "rotation": {"x": 0, "y": 0, "z": 0, "w": 1}, 69 | "label_class": "VEHICLE", 70 | "timestamp": timestamp, 71 | "track_label_uuid": "123", 72 | } 73 | actual_label = build_argo_label( 74 | mock_label, timestamp, track_ids 75 | ) 76 | assert actual_label == expected_label 77 | -------------------------------------------------------------------------------- /waymo2argo/transform_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit tests on rigid body transformation utilities. 3 | 4 | Authors: John Lambert 5 | """ 6 | 7 | from typing import Tuple 8 | 9 | import numpy as np 10 | from scipy.spatial.transform import Rotation 11 | 12 | 13 | def se2_to_yaw(B_SE2_A) -> float: 14 | """Computes the pose vector v from a homogeneous transform A. 15 | 16 | Args: 17 | B_SE2_A 18 | 19 | Returns: 20 | theta: 21 | """ 22 | R = B_SE2_A.rotation 23 | theta = np.arctan2(R[1, 0], R[0, 0]) 24 | return theta 25 | 26 | 27 | def quaternion3d_to_yaw(q: np.ndarray) -> float: 28 | """ 29 | Args: 30 | q: qx,qy,qz,qw: quaternion coefficients 31 | 32 | Returns: 33 | yaw: float, rotation about the z-axis 34 | """ 35 | w, x, y, z = q # in argo format 36 | q_scipy = x, y, z, w 37 | R = Rotation.from_quat(q_scipy).as_matrix() 38 | # tan (yaw) = s / c 39 | yaw = np.arctan2(R[1, 0], R[0, 0]) 40 | return yaw 41 | 42 | 43 | def yaw_to_quaternion3d(yaw: float) -> Tuple[float, float, float, float]: 44 | """ 45 | Args: 46 | yaw: rotation about the z-axis 47 | 48 | Returns: 49 | qx,qy,qz,qw: quaternion coefficients 50 | """ 51 | qx, qy, qz, qw = Rotation.from_euler("z", yaw).as_quat() 52 | return qx, qy, qz, qw 53 | 54 | 55 | def rotmat2quat(R: np.ndarray) -> np.ndarray: 56 | """ """ 57 | q_scipy = Rotation.from_matrix(R).as_quat() 58 | x, y, z, w = q_scipy 59 | q_argo = w, x, y, z 60 | return q_argo 61 | 62 | 63 | def quat2rotmat(q: np.ndarray) -> np.ndarray: 64 | """Convert a unit-length quaternion into a rotation matrix. 65 | Note that libraries such as Scipy expect a quaternion in scalar-last [x, y, z, w] format, 66 | whereas at Argo we work with scalar-first [w, x, y, z] format, so we convert between the 67 | two formats here. We use the [w, x, y, z] order because this corresponds to the 68 | multidimensional complex number `w + ix + jy + kz`. 69 | 70 | Args: 71 | q: Array of shape (4,) representing (w, x, y, z) coordinates 72 | 73 | Returns: 74 | R: Array of shape (3, 3) representing a rotation matrix. 75 | """ 76 | assert np.isclose(np.linalg.norm(q), 1.0, atol=1e-12) 77 | w, x, y, z = q 78 | q_scipy = np.array([x, y, z, w]) 79 | return Rotation.from_quat(q_scipy).as_matrix() 80 | 81 | 82 | def rotMatZ_3D(yaw: float) -> np.ndarray: 83 | """ 84 | Args: 85 | yaw: angle in radians 86 | 87 | Returns: 88 | rot_z 89 | """ 90 | c = np.cos(yaw) 91 | s = np.sin(yaw) 92 | rot_z = np.array([[c, -s, 0], [s, c, 0], [0, 0, 1]]) 93 | return rot_z 94 | 95 | 96 | def rotX(deg: float) -> np.ndarray: 97 | """Compute 3x3 rotation matrix about the X-axis. 98 | 99 | Args: 100 | deg: Euler angle in degrees 101 | """ 102 | t = np.deg2rad(deg) 103 | return Rotation.from_euler("x", t).as_matrix() 104 | 105 | 106 | def rotZ(deg: float) -> np.ndarray: 107 | """Compute 3x3 rotation matrix about the Z-axis. 108 | 109 | Args: 110 | deg: Euler angle in degrees 111 | """ 112 | t = np.deg2rad(deg) 113 | return Rotation.from_euler("z", t).as_matrix() 114 | 115 | 116 | def rotY(deg: float) -> np.ndarray: 117 | """Compute 3x3 rotation matrix about the Y-axis. 118 | 119 | Args: 120 | deg: Euler angle in degrees 121 | """ 122 | t = np.deg2rad(deg) 123 | return Rotation.from_euler("y", t).as_matrix() 124 | -------------------------------------------------------------------------------- /waymo2argo/dump_waymo_persweep_detections.py: -------------------------------------------------------------------------------- 1 | """ 2 | Given sharded JSON files containing labeled objects or detections in random order, 3 | accumulate objects according to frame, at each nanosecond timestamp, and 4 | write them to disk in JSON again. 5 | 6 | Also, writes corresponding dummy PLY files for each frame. 7 | """ 8 | 9 | import glob 10 | import json 11 | import os 12 | from collections import defaultdict 13 | from pathlib import Path 14 | from typing import Any, Dict, Union 15 | 16 | 17 | def round_to_micros(t_nanos, base=1000): 18 | """Round nanosecond timestamp to nearest microsecond timestamp.""" 19 | return base * round(t_nanos / base) 20 | 21 | 22 | def test_round_to_micros(): 23 | """ 24 | test_round_to_micros() 25 | """ 26 | t_nanos = 1508103378165379072 27 | t_micros = 1508103378165379000 28 | 29 | assert t_micros == round_to_micros(t_nanos, base=1000) 30 | 31 | 32 | def check_mkdir(dirpath): 33 | """ """ 34 | if not Path(dirpath).exists(): 35 | os.makedirs(dirpath, exist_ok=True) 36 | 37 | 38 | def read_json_file(fpath: Union[str, "os.PathLike[str]"]) -> Any: 39 | """Load dictionary from JSON file. 40 | Args: 41 | fpath: Path to JSON file. 42 | Returns: 43 | Deserialized Python dictionary. 44 | """ 45 | with open(fpath, "rb") as f: 46 | return json.load(f) 47 | 48 | 49 | def save_json_dict(json_fpath: Union[str, "os.PathLike[str]"], dictionary: Dict[Any, Any]) -> None: 50 | """Save a Python dictionary to a JSON file. 51 | Args: 52 | json_fpath: Path to file to create. 53 | dictionary: Python dictionary to be serialized. 54 | """ 55 | with open(json_fpath, "w") as f: 56 | json.dump(dictionary, f) 57 | 58 | 59 | def main(verbose=False): 60 | """ """ 61 | DETS_DATAROOT = "/Users/johnlamb/Downloads/waymo_logs_dets" 62 | RAW_DATAROOT = "/Users/johnlamb/Downloads/waymo_logs_raw_data" 63 | SHARD_DIR = "/Users/johnlamb/Downloads/waymo_pointpillars_detections" 64 | for split in ["validation", "test"]: 65 | print(split) 66 | for classname in ["cyclist", "pedestrian", "vehicle"]: 67 | print(f"\t{classname}") 68 | 69 | shard_fpaths = glob.glob(f"{SHARD_DIR}/{split}/detection_3d_{classname}*{split}_shard*.json") 70 | shard_fpaths.sort() 71 | for shard_fpath in shard_fpaths: 72 | 73 | log_to_timestamp_to_dets_dict = defaultdict(dict) 74 | print(f"\t\t{Path(shard_fpath).stem}") 75 | shard_data = read_json_file(shard_fpath) 76 | for i, det in enumerate(shard_data): 77 | 78 | log_id = det["context_name"] 79 | if i % 100000 == 0: 80 | print(f"On {i}/{len(shard_data)}") 81 | timestamp_ms = det["timestamp"] 82 | timestamp_ns = int(1000 * timestamp_ms) 83 | 84 | if log_id not in log_to_timestamp_to_dets_dict: 85 | log_to_timestamp_to_dets_dict[log_id] = defaultdict(list) 86 | 87 | log_to_timestamp_to_dets_dict[log_id][timestamp_ns].append(det) 88 | 89 | for log_id, timestamp_to_dets_dict in log_to_timestamp_to_dets_dict.items(): 90 | print(log_id) 91 | for timestamp_ns, dets in timestamp_to_dets_dict.items(): 92 | sweep_json_fpath = ( 93 | f"{DETS_DATAROOT}/{log_id}/per_sweep_annotations/tracked_object_labels_{timestamp_ns}.json" 94 | ) 95 | dummy_lidar_fpath = f"{RAW_DATAROOT}/{log_id}/lidar/PC_{timestamp_ns}.ply" 96 | 97 | if Path(sweep_json_fpath).exists(): 98 | # accumulate tracks of another class together 99 | prev_dets = read_json_file(sweep_json_fpath) 100 | dets.extend(prev_dets) 101 | 102 | check_mkdir(str(Path(sweep_json_fpath).parent)) 103 | save_json_dict(sweep_json_fpath, dets) 104 | 105 | check_mkdir(str(Path(dummy_lidar_fpath).parent)) 106 | save_json_dict(dummy_lidar_fpath, {}) 107 | 108 | if verbose: 109 | print("Shared timestamps:") 110 | #print(timestamps_counts) 111 | 112 | 113 | if __name__ == "__main__": 114 | 115 | # test_transform() 116 | main() 117 | -------------------------------------------------------------------------------- /waymo2argo/create_submission_bin_file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import glob 4 | import json 5 | import numpy as np 6 | import os 7 | import time 8 | from pathlib import Path 9 | from typing import Any, Union 10 | 11 | from waymo_open_dataset import label_pb2 12 | from waymo_open_dataset.protos import metrics_pb2 13 | 14 | import waymo2argo.transform_utils as transform_utils 15 | from waymo2argo.waymo_data_splits import get_val_log_ids, get_test_log_ids 16 | 17 | """ 18 | Given tracks in Argoverse format, convert them to Waymo submission format. 19 | """ 20 | 21 | OBJECT_TYPES = [ 22 | "UNKNOWN", # 0 23 | "VEHICLE", # 1 24 | "PEDESTRIAN", # 2 25 | "SIGN", # 3 26 | "CYCLIST", # 4 27 | ] 28 | 29 | 30 | def read_json_file(fpath: Union[str, "os.PathLike[str]"]) -> Any: 31 | """Load dictionary from JSON file. 32 | Args: 33 | fpath: Path to JSON file. 34 | Returns: 35 | Deserialized Python dictionary. 36 | """ 37 | with open(fpath, "rb") as f: 38 | return json.load(f) 39 | 40 | 41 | def create_submission(min_conf: float, min_hits: int) -> None: 42 | """Creates a prediction objects file.""" 43 | objects = metrics_pb2.Objects() 44 | 45 | split = "test" 46 | exp_name = f"ab3dmot_tracks_conf{min_conf}_complete_sharded_{split}_minhits{min_hits}" 47 | TRACKER_OUTPUT_DATAROOT = f"{exp_name}/{split}-split-track-preds-maxage15-minhits{min_hits}-conf{min_conf}" 48 | if split == "val": 49 | log_ids = get_val_log_ids() 50 | elif split == "test": 51 | log_ids = get_test_log_ids() 52 | 53 | # loop over the logs in the split 54 | for i, log_id in enumerate(log_ids): 55 | print(f"On {i}th log {log_id}") 56 | start = time.time() 57 | # get all the per_sweep_annotations_amodal files 58 | json_fpaths = glob.glob(f"{TRACKER_OUTPUT_DATAROOT}/{log_id}/per_sweep_annotations_amodal/*.json") 59 | # for each per_sweep_annotation file 60 | for json_fpath in json_fpaths: 61 | timestamp_ns = int(Path(json_fpath).stem.split("_")[-1]) 62 | timestamp_objs = read_json_file(json_fpath) 63 | # loop over all objects 64 | for obj_json in timestamp_objs: 65 | o = create_object_description(log_id, timestamp_ns, obj_json) 66 | objects.objects.append(o) 67 | end = time.time() 68 | duration = end - start 69 | print(f"\tTook {duration} sec") 70 | 71 | # Add more objects. Note that a reasonable detector should limit its maximum 72 | # number of boxes predicted per frame. A reasonable value is around 400. A 73 | # huge number of boxes can slow down metrics computation. 74 | 75 | # Write objects to a file. 76 | f = open(f"/w_o_d/{exp_name}.bin", "wb") 77 | f.write(objects.SerializeToString()) 78 | f.close() 79 | 80 | 81 | def create_object_description(log_id: str, timestamp_ns: int, obj_json) -> metrics_pb2.Object: 82 | """ """ 83 | o = metrics_pb2.Object() 84 | # The following 3 fields are used to uniquely identify a frame a prediction 85 | # is predicted at. Make sure you set them to values exactly the same as what 86 | # we provided in the raw data. Otherwise your prediction is considered as a 87 | # false negative. 88 | o.context_name = log_id 89 | # The frame timestamp for the prediction. See Frame::timestamp_micros in 90 | # dataset.proto. 91 | invalid_ts = -1 92 | o.frame_timestamp_micros = int(timestamp_ns / 1000) # save as microseconds 93 | 94 | tx, ty, tz = obj_json["center"]["x"], obj_json["center"]["y"], obj_json["center"]["z"] 95 | qx, qy, qz, qw = ( 96 | obj_json["rotation"]["x"], 97 | obj_json["rotation"]["y"], 98 | obj_json["rotation"]["z"], 99 | obj_json["rotation"]["w"], 100 | ) 101 | q_argo = np.array([qw, qx, qy, qz]) 102 | yaw = transform_utils.quaternion3d_to_yaw(q_argo) 103 | 104 | # Populating box and score. 105 | box = label_pb2.Label.Box() 106 | box.center_x = tx 107 | box.center_y = ty 108 | box.center_z = tz 109 | box.length = obj_json["length"] 110 | box.width = obj_json["width"] 111 | box.height = obj_json["height"] 112 | box.heading = yaw 113 | o.object.box.CopyFrom(box) 114 | # This must be within [0.0, 1.0]. It is better to filter those boxes with 115 | # small scores to speed up metrics computation. 116 | o.score = 0.5 117 | # For tracking, this must be set and it must be unique for each tracked 118 | # sequence. 119 | o.object.id = obj_json["track_label_uuid"] 120 | 121 | if obj_json["label_class"] == "PEDESTRIAN": 122 | obj_type = label_pb2.Label.TYPE_PEDESTRIAN 123 | elif obj_json["label_class"] == "CYCLIST": 124 | obj_type = label_pb2.Label.TYPE_CYCLIST 125 | elif obj_json["label_class"] == "VEHICLE": 126 | obj_type = label_pb2.Label.TYPE_VEHICLE 127 | else: 128 | print("Unknown obj. type...") 129 | quit() 130 | 131 | # Use correct type. 132 | o.object.type = obj_type 133 | return o 134 | 135 | 136 | if __name__ == "__main__": 137 | """ """ 138 | create_submission(min_conf=0.475, min_hits=6) 139 | -------------------------------------------------------------------------------- /waymo2argo/waymo_dets_to_argoverse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Convert provided Waymo detections to Argoverse format. 5 | 6 | Within Colab, download the detection files, and dump them to disk in Argoverse form. 7 | To avoid exceeding Colab RAM, we shard the data. 8 | 9 | If run in Colab, add the following: 10 | !rm -rf waymo-od > /dev/null 11 | !git clone https://github.com/waymo-research/waymo-open-dataset.git waymo-od 12 | !cd waymo-od && git branch -a 13 | !cd waymo-od && git checkout remotes/origin/r2.0 14 | !pip3 install --upgrade pip 15 | 16 | pip install waymo-open-dataset-tf-2-1-0==1.2.0 17 | !pip3 install waymo-open-dataset-tf-2-1-0==1.2.0 18 | 19 | # from google.colab import files 20 | # uploaded = files.upload() 21 | 22 | or 23 | from google.colab import drive 24 | drive.mount('/content/gdrive') 25 | """ 26 | 27 | import json 28 | import os 29 | from pathlib import Path 30 | from typing import Any, Dict, Union 31 | 32 | import tensorflow as tf 33 | 34 | tf.compat.v1.enable_eager_execution() 35 | from waymo_open_dataset.protos import metrics_pb2 36 | 37 | import waymo2argo.transform_utils as transform_utils 38 | 39 | 40 | def round_to_micros(t_nanos: int, base=1000) -> int: 41 | """Round nanosecond timestamp to nearest microsecond timestamp.""" 42 | return base * round(t_nanos / base) 43 | 44 | 45 | def test_round_to_micros() -> None: 46 | """ 47 | test_round_to_micros() 48 | """ 49 | t_nanos = 1508103378165379072 50 | t_micros = 1508103378165379000 51 | 52 | assert t_micros == round_to_micros(t_nanos, base=1000) 53 | 54 | 55 | def save_json_dict(json_fpath: Union[str, "os.PathLike[str]"], dictionary: Dict[Any, Any]) -> None: 56 | """Save a Python dictionary to a JSON file. 57 | Args: 58 | json_fpath: Path to file to create. 59 | dictionary: Python dictionary to be serialized. 60 | """ 61 | with open(json_fpath, "w") as f: 62 | json.dump(dictionary, f) 63 | 64 | 65 | # DRIVE_DIR = '/content/gdrive/My Drive/WaymoOpenDatasetTracking' 66 | DRIVE_DIR = "/srv/datasets/waymo_opendataset/waymo_open_dataset_v_1_0_0/training" 67 | 68 | bin_fnames = [ 69 | "detection_3d_cyclist_detection_test.bin", 70 | "detection_3d_cyclist_detection_validation.bin", 71 | "detection_3d_pedestrian_detection_test.bin", 72 | "detection_3d_pedestrian_detection_validation.bin", 73 | "detection_3d_vehicle_detection_test.bin", 74 | "detection_3d_vehicle_detection_validation.bin", 75 | ] 76 | 77 | 78 | def main() -> None: 79 | """ """ 80 | SHARD_SZ = 500000 81 | for bin_fname in bin_fnames: 82 | print(bin_fname) 83 | bin_fpath = f"{DRIVE_DIR}/{bin_fname}" 84 | shard_counter = 0 85 | json_fpath = f"{DRIVE_DIR}/{Path(bin_fname).stem}_shard_{shard_counter}.json" 86 | 87 | objects = metrics_pb2.Objects() 88 | 89 | f = open(bin_fpath, "rb") 90 | objects.ParseFromString(f.read()) 91 | f.close() 92 | 93 | OBJECT_TYPES = [ 94 | "UNKNOWN", # 0 95 | "VEHICLE", # 1 96 | "PEDESTRIAN", # 2 97 | "SIGN", # 3 98 | "CYCLIST", # 4 99 | ] 100 | 101 | gt_num_objs = len(objects.objects) 102 | print(f"num_objs={gt_num_objs}") 103 | tracked_labels = [] 104 | for i, object in enumerate(objects.objects): 105 | if i % 50000 == 0: 106 | print(f"On {i}/{len(objects.objects)}") 107 | height = object.object.box.height 108 | width = object.object.box.width 109 | length = object.object.box.length 110 | score = object.score 111 | x = object.object.box.center_x 112 | y = object.object.box.center_y 113 | z = object.object.box.center_z 114 | 115 | # Waymo provides SE(3) transformation from 116 | # labeled_object->egovehicle like Argoverse 117 | obj_yaw_ego = object.object.box.heading 118 | 119 | qx, qy, qz, qw = transform_utils.yaw_to_quaternion3d(obj_yaw_ego) 120 | label_class = OBJECT_TYPES[object.object.type] 121 | 122 | tracked_labels.append( 123 | { 124 | "center": {"x": x, "y": y, "z": z}, 125 | "rotation": {"x": qx, "y": qy, "z": qz, "w": qw}, 126 | "length": length, 127 | "width": width, 128 | "height": height, 129 | "track_label_uuid": None, 130 | # TODO: write as int(nanoseconds) instead. 131 | "timestamp": object.frame_timestamp_micros, # 1522688014970187 132 | "label_class": label_class, 133 | "score": object.score, # float in [0,1] 134 | "context_name": object.context_name, 135 | } 136 | ) 137 | if len(tracked_labels) >= SHARD_SZ: 138 | save_json_dict(json_fpath, tracked_labels) 139 | tracked_labels = [] 140 | shard_counter += 1 141 | json_fpath = f"{DRIVE_DIR}/{Path(bin_fname).stem}_shard_{shard_counter}.json" 142 | 143 | # label_dir = os.path.join(tracks_dump_dir, log_id, "per_sweep_annotations_amodal") 144 | # check_mkdir(label_dir) 145 | # json_fname = f"tracked_object_labels_{current_lidar_timestamp}.json" 146 | # json_fpath = os.path.join(label_dir, json_fname) 147 | 148 | # if Path(json_fpath).exists(): 149 | # # accumulate tracks of another class together 150 | # prev_tracked_labels = read_json_file(json_fpath) 151 | # tracked_labels.extend(prev_tracked_labels) 152 | 153 | # ensure sharding correct 154 | print(f"Shard sz, {SHARD_SZ}, num_objs={gt_num_objs}") 155 | print(f"shard_counter={shard_counter}, len_tracked_labels{len(tracked_labels)}") 156 | assert gt_num_objs // SHARD_SZ == shard_counter 157 | assert gt_num_objs % SHARD_SZ == len(tracked_labels) 158 | 159 | save_json_dict(json_fpath, tracked_labels) 160 | 161 | 162 | if __name__ == "__main__": 163 | main() 164 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | | Platform | Build Status | 3 | |:------------:| :-------------:| 4 | | Ubuntu 20.04.3 | ![Linux CI](https://github.com/johnwlambert/waymo_to_argoverse/workflows/Python%20CI/badge.svg) | 5 | 6 | ## Waymo Open Dataset -> Argoverse Converter 7 | 8 |

9 | 10 | 11 |

12 |

13 | 14 | 15 |

16 | 17 | ### Repo Overview 18 | 19 | Simple utility to convert Waymo Open Dataset raw data, ground truth, and detections to the Argoverse format [ [paper](https://arxiv.org/abs/1911.02620), [repo](https://github.com/argoai/argoverse-api) ], run a tracker that accepts Argoverse-format data, and then submit to Waymo Open Dataset leaderboard. 20 | 21 | Achieves the following on the Waymo 3d Tracking Leaderboard, using `run_ab3dmot.py` from my [argoverse_cbgs_kf_tracker](https://github.com/johnwlambert/argoverse_cbgs_kf_tracker) repo. 22 | 23 | | Model | MOTA/L2 | MOTP/L2 | FP/L2 | Mismatch/L2 | Miss/L2 | 24 | | :-------------------------: | :-------: | :--------: | :--------:| :--------: | :--------: | 25 | | HorizonMOT3D | 0.6345 | 0.2396 | 0.0728 | 0.0029 | 0.2899 | 26 | | PV-RCNN-KF | 0.5553 | 0.2497 | 0.0866 | 0.0063 | 0.3518 | 27 | | Probabilistic 3DMOT | 0.4765 | 0.2482 | 0.0899 | 0.0101 | 0.4235 | 28 | | ... | ... | ... | ... | ... | ... | 29 | | **PPBA AB3DMOT (this repo)**| **0.2914** | 0.2696 | 0.1714 | 0.0025 | 0.5347 | 30 | | Waymo Baseline | 0.2592 | 0.1753 | 0.0932 | 0.0020 | 0.3122 | 31 | 32 | 33 | ## Data Format Overview 34 | 35 | Waymo raw data follows a rough class structure, as defined in [Frame protobuffer](https://github.com/waymo-research/waymo-open-dataset/blob/master/waymo_open_dataset/dataset.proto). 36 | Waymo labels and the detections they provide also follow a rough class structure, defined in [Label protobuffer](https://github.com/waymo-research/waymo-open-dataset/blob/master/waymo_open_dataset/label.proto). 37 | 38 | Argoverse also uses a notion of Frame at 10 Hz, but only for LiDAR and annotated cuboids in LiDAR. This is because Argoverse imagery is at 30 Hz (ring camera) and 5 Hz (stereo). Argoverse data is provided at integer nanosecond frequency throughout, whereas Waymo mixes seconds and microseconds in different places. **Argoverse LiDAR points are provided directly in the egovehicle frame, not in the LiDAR sensor frame, as [.PLY](http://paulbourke.net/dataformats/ply/) files.** 39 | 40 | A Waymo object defines a coordinate transformation from the labeled object coordinate frame, to the egovehicle coordinate frame, as an SE(3) comprised of rotation (derived from heading) and a translation: 41 | ```python 42 | object { 43 | box { 44 | center_x: 67.52523040771484 45 | center_y: -1.3868849277496338 46 | center_z: 0.8951533436775208 47 | width: 0.8146794438362122 48 | length: 1.8189797401428223 49 | height: 1.790642261505127 50 | heading: -0.11388802528381348 51 | } 52 | type: TYPE_CYCLIST 53 | } 54 | score: 0.19764792919158936 55 | context_name: "10203656353524179475_7625_000_7645_000" 56 | frame_timestamp_micros: 1522688014970187 57 | ``` 58 | 59 | Argoverse data is provided similarly, but in JSON with full 6 dof instead of 4 dof transformation from labeled object coordinate frame to egovehicle frame. A quaternion is used for the SO(3) parameterization: 60 | ```python 61 | { 62 | "center": {"x": -25.627050258944625, "y": -3.6203567237860375, "z": 0.4981851744013227}, 63 | "rotation": 64 | {"x": -0.000662416717311173, 65 | "y": -0.000193607239199898, 66 | "z": 0.000307246307353097, "w": 0.999999714659978}, 67 | "length": 4.784992980957031, 68 | "width": 2.107541785708549, 69 | "height": 1.8, 70 | "track_label_uuid": "215056a9-9325-4a25-bbbd-92d445d60168", 71 | "timestamp": 315969629119937000, 72 | "label_class": "VEHICLE" 73 | }, 74 | ``` 75 | Whereas Waymo uses "context.name" as a unique log identifier, Argoverse uses "log_id". 76 | 77 | ### Installation 78 | Before you use this code, you will need to download a few packages. To install the `waymo_open_dataset` library, use the commands [here](https://github.com/waymo-research/waymo-open-dataset/blob/master/docs/quick_start.md#use-pre-compiled-pippip3-packages) to install the pre-compiled pip packages. 79 | You will also need to download `argoverse`. You can use [this command](https://github.com/argoai/argoverse-api#4-install-argoverse-module) to download the package. 80 | 81 | ### Guide to Repo Code Structure 82 | - `waymo2argo/waymo_dets_to_argoverse.py`: Convert provided Waymo detections to Argoverse format. Use shards to not exceed Colab RAM. 83 | - `waymo2argo/dump_waymo_persweep_detections.py`: Given sharded JSON files containing labeled objects or detections in random order, accumulate objects according to frame, at each nanosecond timestamp. Write to disk. 84 | - `waymo_data_splits.py`: functions to provide list of log_ids's in Waymo val and test splits, respectively. 85 | - `waymo2argo/waymo_raw_data_to_argoverse.py`: Extract poses, LiDAR, images, and camera calibration from raw Waymo Open Dataset TFRecords. 86 | - `run_tracker.sh`: script to run [AB3DMOT-style tracker](https://github.com/johnwlambert/argoverse_cbgs_kf_tracker) on Argoverse-format detections, and write tracks to disk. 87 | - `create_submission_bin_file.py`: Given tracks in Argoverse format, convert them to Waymo submission format. 88 | 89 | ### Converting Waymo Raw Data to Argoverse Format 90 | To convert the Waymo dataset to Argoverse format, you will need to run 91 | ```bash 92 | python waymo_raw_data_to_argoverse.py --waymo-dir /path-to-waymo-data/ --argo-dir /path-to-write-argo-data/ 93 | ``` 94 | e.g. 95 | ```bash 96 | python waymo2argo/waymo_raw_data_to_argoverse.py --save-labels true --argo-dir waymo_data_in_argoverse_form_2021_10_06 --waymo-dir /srv/datasets/waymo_opendataset/waymo_open_dataset_v_1_0_0/training 97 | ``` 98 | After running this script, you will also need to run 99 | ```bash 100 | python argoverse/utils/make_track_label_folders.py /path-to-write-argo-data/ 101 | ``` 102 | to create the `track_labels_amodal` folder. There is more information about that [here](https://github.com/argoai/argoverse-api#optional-remake-the-object-oriented-label-folders). 103 | 104 | ### Usage Instructions for Waymo Leaderboard 105 | 106 | 1. Download test split files (~150 logs) from [Waymo Open Dataset website](https://waymo.com/open/download/) which include TFRecords. 107 | 2. Download provided detections from PointPillars Progressive Population-Based Augmentation detector, as .bin files. 108 | 3. Convert to Argoverse format using scripts provided here in this repo. 109 | 4. Run tracker 110 | 5. Convert track results to .bin file 111 | 6. Populate a `submission.txtpb` file with metadata describing your submission ([example here](https://raw.githubusercontent.com/waymo-research/waymo-open-dataset/master/waymo_open_dataset/metrics/tools/submission.txtpb)). 112 | 7. Run `create_submission` binary to get tar.gz file. Binary is only compiled using Bazel. I used Google Colab. 113 | 8. Submit to [Waymo eval server](https://waymo.com/open/challenges/3d-tracking/). 114 | 115 | 116 | Submission process overview is [here](https://github.com/waymo-research/waymo-open-dataset/blob/master/docs/quick_start.md#use-pre-compiled-pippip3-packages). 117 | 118 | 119 | ## References 120 | ``` 121 | @InProceedings{Chang_2019_CVPR, 122 | author = {Chang, Ming-Fang and Lambert, John and Sangkloy, Patsorn and Singh, Jagjeet and Bak, Slawomir and Hartnett, Andrew and Wang, De and Carr, Peter and Lucey, Simon and Ramanan, Deva and Hays, James}, 123 | title = {Argoverse: 3D Tracking and Forecasting With Rich Maps}, 124 | booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 125 | month = {June}, 126 | year = {2019} 127 | } 128 | ``` 129 | -------------------------------------------------------------------------------- /waymo2argo/waymo_raw_data_to_argoverse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Extract poses, images, and camera calibration from raw Waymo Open Dataset TFRecords. 5 | 6 | See the Frame structure here: 7 | https://github.com/waymo-research/waymo-open-dataset/blob/master/waymo_open_dataset/dataset.proto 8 | 9 | See paper: 10 | https://arxiv.org/pdf/1912.04838.pdf 11 | """ 12 | 13 | import argparse 14 | import glob 15 | import imageio 16 | import numpy as np 17 | import os 18 | import pandas as pd 19 | import uuid 20 | from pathlib import Path 21 | from typing import Any, Dict, List 22 | 23 | import argoverse.utils.json_utils as json_utils 24 | import cv2 25 | import google 26 | import tensorflow.compat.v1 as tf 27 | import waymo_open_dataset 28 | from argoverse.utils.se3 import SE3 29 | from pyntcloud import PyntCloud 30 | from waymo_open_dataset.utils import frame_utils 31 | from waymo_open_dataset import dataset_pb2 as open_dataset 32 | 33 | import waymo2argo.transform_utils as transform_utils 34 | 35 | tf.enable_eager_execution() 36 | 37 | 38 | # Mapping from Argo Camera names to Waymo Camera names 39 | # The indices correspond to Waymo's cameras 40 | CAMERA_NAMES = [ 41 | "unknown", # 0, 'UNKNOWN', 42 | "ring_front_center", # 1, 'FRONT' 43 | "ring_front_left", # 2, 'FRONT_LEFT', 44 | "ring_front_right", # 3, 'FRONT_RIGHT', 45 | "ring_side_left", # 4, 'SIDE_LEFT', 46 | "ring_side_right", # 5, 'SIDE_RIGHT' 47 | ] 48 | 49 | # Mapping from Argo Label types to Waymo Label types 50 | # Argo label types are on the left, Waymo's are on the right 51 | # Argoverse labels: https://github.com/argoai/argoverse-api/blob/master/argoverse/data_loading/object_classes.py#L6 52 | # The indices correspond to Waymo's label types 53 | LABEL_TYPES = [ 54 | "UNKNOWN", # 0, TYPE_UNKNOWN 55 | "VEHICLE", # 1, TYPE_VEHICLE 56 | "PEDESTRIAN", # 2, TYPE_PEDESTRIAN 57 | "SIGN", # 3, TYPE_SIGN 58 | "BICYCLIST", # 4, TYPE_CYCLIST 59 | ] 60 | 61 | RING_IMAGE_SIZES = { 62 | # width x height 63 | "ring_front_center": (1920, 1280), 64 | "ring_front_left": (1920, 1280), 65 | "ring_side_left": (1920, 886), 66 | "ring_side_right": (1920, 886), 67 | } 68 | 69 | 70 | def round_to_micros(t_nanos: int, base: int = 1000) -> int: 71 | """Round nanosecond timestamp to nearest microsecond timestamp.""" 72 | return base * round(t_nanos / base) 73 | 74 | 75 | def check_mkdir(dirpath: str) -> None: 76 | """ """ 77 | if not Path(dirpath).exists(): 78 | os.makedirs(dirpath, exist_ok=True) 79 | 80 | 81 | def get_log_ids_from_files(record_dir: str) -> Dict[str, str]: 82 | """Get the log IDs of the Waymo records from the directory 83 | where they are stored 84 | 85 | Args: 86 | record_dir: The path to the directory where the Waymo data 87 | is stored 88 | Example: "/path-to-waymo-data" 89 | The args.waymo_dir is used here by default 90 | Returns: 91 | log_ids: A map of log IDs to tf records from the Waymo dataset 92 | """ 93 | files = glob.glob(f"{record_dir}/*.tfrecord") 94 | log_ids = {} 95 | for i, file in enumerate(files): 96 | file = file.replace(record_dir, "") 97 | file = file.replace("/segment-", "") 98 | file = file.replace(".tfrecord", "") 99 | file = file.replace("_with_camera_labels", "") 100 | log_ids[file] = files[i] 101 | return log_ids 102 | 103 | 104 | def main(args: argparse.Namespace) -> None: 105 | """Main script to convert Waymo object labels, LiDAR, images, pose, and calibration to 106 | the Argoverse data format on disk. 107 | """ 108 | TFRECORD_DIR = args.waymo_dir 109 | ARGO_WRITE_DIR = args.argo_dir 110 | track_id_dict = {} 111 | img_count = 0 112 | log_ids = get_log_ids_from_files(TFRECORD_DIR) 113 | for log_id, tf_fpath in log_ids.items(): 114 | dataset = tf.data.TFRecordDataset(tf_fpath, compression_type="") 115 | log_calib_json = None 116 | for data in dataset: 117 | frame = open_dataset.Frame() 118 | frame.ParseFromString(bytearray(data.numpy())) 119 | # Checking if we extracted the correct log ID 120 | assert log_id == frame.context.name 121 | # Frame start time, which is the timestamp 122 | # of the first top lidar spin within this frame, in microseconds 123 | timestamp_ms = frame.timestamp_micros 124 | timestamp_ns = int(timestamp_ms * 1000) # to nanoseconds 125 | SE3_flattened = np.array(frame.pose.transform) 126 | city_SE3_egovehicle = SE3_flattened.reshape(4, 4) 127 | if args.save_poses: 128 | dump_pose(city_SE3_egovehicle, timestamp_ns, log_id, ARGO_WRITE_DIR) 129 | # Reading lidar data and saving it in point cloud format 130 | # We are only using the first range image (Waymo provides two range images) 131 | # If you want to use the second one, you can change it in the arguments 132 | ( 133 | range_images, 134 | camera_projections, 135 | range_image_top_pose, 136 | ) = frame_utils.parse_range_image_and_camera_projection(frame) 137 | if args.range_image == 1: 138 | points_ri, cp_points_ri = frame_utils.convert_range_image_to_point_cloud( 139 | frame, range_images, camera_projections, range_image_top_pose 140 | ) 141 | elif args.range_image == 2: 142 | points_ri, cp_points_ri = frame_utils.convert_range_image_to_point_cloud( 143 | frame, 144 | range_images, 145 | camera_projections, 146 | range_image_top_pose, 147 | ri_index=1, 148 | ) 149 | points_all_ri = np.concatenate(points_ri, axis=0) 150 | if args.save_cloud: 151 | dump_point_cloud(points_all_ri, timestamp_ns, log_id, ARGO_WRITE_DIR) 152 | # Saving labels 153 | if args.save_labels: 154 | dump_object_labels( 155 | frame.laser_labels, 156 | timestamp_ns, 157 | log_id, 158 | ARGO_WRITE_DIR, 159 | track_id_dict, 160 | ) 161 | if args.save_calibration: 162 | calib_json = form_calibration_json(frame.context.camera_calibrations) 163 | if log_calib_json is None: 164 | log_calib_json = calib_json 165 | calib_json_fpath = f"{ARGO_WRITE_DIR}/{log_id}/vehicle_calibration_info.json" 166 | check_mkdir(str(Path(calib_json_fpath).parent)) 167 | json_utils.save_json_dict(calib_json_fpath, calib_json) 168 | else: 169 | assert calib_json == log_calib_json 170 | 171 | # 5 images per frame 172 | for index, tf_cam_image in enumerate(frame.images): 173 | # 4x4 row major transform matrix that transforms 174 | # 3d points from one frame to another. 175 | SE3_flattened = np.array(tf_cam_image.pose.transform) 176 | city_SE3_egovehicle = SE3_flattened.reshape(4, 4) 177 | # in seconds 178 | timestamp_s = tf_cam_image.pose_timestamp 179 | # TODO: this looks buggy, need to confirm 180 | timestamp_ns = int(round_to_micros(int(timestamp_s * 1e9)) * 1000) # to nanoseconds 181 | if args.save_poses: 182 | dump_pose(city_SE3_egovehicle, timestamp_ns, log_id, ARGO_WRITE_DIR) 183 | 184 | if args.save_images: 185 | camera_name = CAMERA_NAMES[tf_cam_image.name] 186 | img = tf.image.decode_jpeg(tf_cam_image.image) 187 | new_img = undistort_image( 188 | np.asarray(img), 189 | frame.context.camera_calibrations, 190 | tf_cam_image.name, 191 | ) 192 | img_save_fpath = f"{ARGO_WRITE_DIR}/{log_id}/{camera_name}/" 193 | img_save_fpath += f"{camera_name}_{timestamp_ns}.jpg" 194 | check_mkdir(str(Path(img_save_fpath).parent)) 195 | imageio.imwrite(img_save_fpath, new_img) 196 | img_count += 1 197 | if img_count % 100 == 0: 198 | print(f"\tSaved {img_count}'th image for log = {log_id}") 199 | 200 | 201 | def undistort_image( 202 | img: np.ndarray, 203 | calib_data: google.protobuf.pyext._message.RepeatedCompositeContainer, 204 | camera_name: int, 205 | ) -> np.ndarray: 206 | """Undistort the image from the Waymo dataset given camera calibration data.""" 207 | for camera_calib in calib_data: 208 | if camera_calib.name == camera_name: 209 | f_u, f_v, c_u, c_v, k1, k2, p1, p2, k3 = camera_calib.intrinsic 210 | # k1, k2 and k3 are the tangential distortion coefficients 211 | # p1, p2 are the radial distortion coefficients 212 | camera_matrix = np.array([[f_u, 0, c_u], [0, f_v, c_v], [0, 0, 1]]) 213 | dist_coeffs = np.array([k1, k2, p1, p2, k3]) 214 | return cv2.undistort(img, camera_matrix, dist_coeffs) 215 | 216 | 217 | def form_calibration_json( 218 | calib_data: google.protobuf.pyext._message.RepeatedCompositeContainer, 219 | ) -> Dict[str, Any]: 220 | """Create a JSON file per log containing calibration information, in the Argoverse format. 221 | 222 | Argoverse expects to receive "egovehicle_T_camera", i.e. from camera -> egovehicle, with 223 | rotation parameterized as a quaternion. 224 | Waymo provides the same SE(3) transformation, but with rotation parameterized as a 3x3 matrix. 225 | """ 226 | calib_dict = {"camera_data_": []} 227 | for camera_calib in calib_data: 228 | cam_name = CAMERA_NAMES[camera_calib.name] 229 | # They provide "Camera frame to vehicle frame." 230 | # https://github.com/waymo-research/waymo-open-dataset/blob/master/waymo_open_dataset/dataset.proto 231 | egovehicle_SE3_waymocam = np.array(camera_calib.extrinsic.transform).reshape(4, 4) 232 | standardcam_R_waymocam = transform_utils.rotY(-90) @ transform_utils.rotX(90) 233 | standardcam_SE3_waymocam = SE3(rotation=standardcam_R_waymocam, translation=np.zeros(3)) 234 | egovehicle_SE3_waymocam = SE3( 235 | rotation=egovehicle_SE3_waymocam[:3, :3], 236 | translation=egovehicle_SE3_waymocam[:3, 3], 237 | ) 238 | standardcam_SE3_egovehicle = standardcam_SE3_waymocam.compose(egovehicle_SE3_waymocam.inverse()) 239 | egovehicle_SE3_standardcam = standardcam_SE3_egovehicle.inverse() 240 | egovehicle_q_camera = transform_utils.rotmat2quat(egovehicle_SE3_standardcam.rotation) 241 | x, y, z = egovehicle_SE3_standardcam.translation 242 | qw, qx, qy, qz = egovehicle_q_camera 243 | f_u, f_v, c_u, c_v, k1, k2, p1, p2, k3 = camera_calib.intrinsic 244 | cam_dict = { 245 | "key": "image_raw_" + cam_name, 246 | "value": { 247 | "focal_length_x_px_": f_u, 248 | "focal_length_y_px_": f_v, 249 | "focal_center_x_px_": c_u, 250 | "focal_center_y_px_": c_v, 251 | "skew_": 0, 252 | "distortion_coefficients_": [0, 0, 0], 253 | "vehicle_SE3_camera_": { 254 | "rotation": {"coefficients": [qw, qx, qy, qz]}, 255 | "translation": [x, y, z], 256 | }, 257 | }, 258 | } 259 | calib_dict["camera_data_"] += [cam_dict] 260 | return calib_dict 261 | 262 | 263 | def dump_pose(city_SE3_egovehicle: np.ndarray, timestamp: int, log_id: str, parent_path: str) -> None: 264 | """Saves the pose of the egovehicle in the city coordinate frame at a particular timestamp. 265 | 266 | The SE(3) transformation is stored as a quaternion and length-3 translation vector. 267 | 268 | Args: 269 | city_SE3_egovehicle: A (4,4) numpy array representing the 270 | SE3 transformation from city to egovehicle frame 271 | timestamp: Timestamp in nanoseconds when the lidar reading occurred 272 | log_id: Log ID that the reading belongs to 273 | parent_path: The directory that the converted data is written to 274 | """ 275 | x, y, z = city_SE3_egovehicle[:3, 3] 276 | R = city_SE3_egovehicle[:3, :3] 277 | assert np.allclose(city_SE3_egovehicle[3], np.array([0, 0, 0, 1])) 278 | q = transform_utils.rotmat2quat(R) 279 | qw, qx, qy, qz = q 280 | pose_dict = {"rotation": [qw, qx, qy, qz], "translation": [x, y, z]} 281 | json_fpath = f"{parent_path}/{log_id}/poses/city_SE3_egovehicle_{timestamp}.json" 282 | check_mkdir(str(Path(json_fpath).parent)) 283 | json_utils.save_json_dict(json_fpath, pose_dict) 284 | 285 | 286 | def dump_point_cloud(points: np.ndarray, timestamp: int, log_id: str, parent_path: str) -> None: 287 | """Saves point cloud as .ply file extracted from Waymo's range images 288 | 289 | Args: 290 | points: A (N,3) numpy array representing the point cloud created from lidar readings 291 | timestamp: Timestamp in nanoseconds when the lidar reading occurred 292 | log_id: Log ID that the reading belongs to 293 | parent_path: The directory that the converted data is written to 294 | """ 295 | # Point cloud needs to be of type float 296 | points = points.astype(float) 297 | data = {"x": points[:, 0], "y": points[:, 1], "z": points[:, 2]} 298 | cloud = PyntCloud(pd.DataFrame(data)) 299 | cloud_fpath = f"{parent_path}/{log_id}/lidar/PC_{timestamp}.ply" 300 | check_mkdir(str(Path(cloud_fpath).parent)) 301 | cloud.to_file(cloud_fpath) 302 | 303 | 304 | def dump_object_labels( 305 | labels: List[waymo_open_dataset.label_pb2.Label], 306 | timestamp: int, 307 | log_id: str, 308 | parent_path: str, 309 | track_id_dict: Dict[str, str], 310 | ) -> None: 311 | """Saves object labels from Waymo dataset as json files. 312 | 313 | Args: 314 | labels: A list of Waymo labels 315 | timestamp: Timestamp in nanoseconds when the lidar reading occurred 316 | log_id: Log ID that the reading belongs to 317 | parent_path: The directory that the converted data is written to 318 | track_id_dict: Dictionary to store object ID to track ID mappings 319 | """ 320 | argoverse_labels = [] 321 | for label in labels: 322 | # We don't want signs, as that is not a category in Argoverse 323 | if label.type != LABEL_TYPES.index("SIGN") and label.type != LABEL_TYPES.index("UNKNOWN"): 324 | argoverse_labels.append(build_argo_label(label, timestamp, track_id_dict)) 325 | json_fpath = f"{parent_path}/{log_id}/per_sweep_annotations_amodal/" 326 | json_fpath += f"tracked_object_labels_{timestamp}.json" 327 | check_mkdir(str(Path(json_fpath).parent)) 328 | json_utils.save_json_dict(json_fpath, argoverse_labels) 329 | 330 | 331 | def build_argo_label( 332 | label: waymo_open_dataset.label_pb2.Label, timestamp: int, track_id_dict: Dict[str, str] 333 | ) -> Dict[str, Any]: 334 | """Builds a dictionary that represents an object detection in Argoverse format from a Waymo label 335 | 336 | Args: 337 | labels: A Waymo label 338 | timestamp: Timestamp in nanoseconds when the lidar reading occurred 339 | track_id_dict: Dictionary to store object ID to track ID mappings 340 | 341 | Returns: 342 | label_dict: A dictionary representing the object label in Argoverse format 343 | """ 344 | label_dict = {} 345 | label_dict["center"] = {} 346 | label_dict["center"]["x"] = label.box.center_x 347 | label_dict["center"]["y"] = label.box.center_y 348 | label_dict["center"]["z"] = label.box.center_z 349 | label_dict["length"] = label.box.length 350 | label_dict["width"] = label.box.width 351 | label_dict["height"] = label.box.height 352 | label_dict["rotation"] = {} 353 | qx, qy, qz, qw = transform_utils.yaw_to_quaternion3d(label.box.heading) 354 | label_dict["rotation"]["x"] = qx 355 | label_dict["rotation"]["y"] = qy 356 | label_dict["rotation"]["z"] = qz 357 | label_dict["rotation"]["w"] = qw 358 | label_dict["label_class"] = LABEL_TYPES[label.type] 359 | label_dict["timestamp"] = timestamp 360 | if label.id not in track_id_dict.keys(): 361 | track_id = uuid.uuid4().hex 362 | track_id_dict[label.id] = track_id 363 | else: 364 | track_id = track_id_dict[label.id] 365 | label_dict["track_label_uuid"] = track_id 366 | return label_dict 367 | 368 | 369 | def str2bool(v): 370 | if isinstance(v, bool): 371 | return v 372 | elif v.lower() in ("yes", "true", "t", "y", 1): 373 | return True 374 | elif v.lower() in ("no", "false", "f", "n", 0): 375 | return False 376 | else: 377 | raise argparse.ArgumentTypeError("Boolean value expected.") 378 | 379 | 380 | if __name__ == "__main__": 381 | parser = argparse.ArgumentParser() 382 | parser.add_argument( 383 | "--save-images", 384 | default=True, 385 | type=str2bool, 386 | help="whether to save images or not", 387 | ) 388 | parser.add_argument("--save-poses", default=True, type=str2bool, help="whether to save poses or not") 389 | parser.add_argument( 390 | "--save-calibration", 391 | default=True, 392 | type=str2bool, 393 | help="whether to save camera calibration information or not", 394 | ) 395 | parser.add_argument( 396 | "--save-cloud", 397 | default=True, 398 | type=str2bool, 399 | help="whether to save point clouds or not", 400 | ) 401 | parser.add_argument( 402 | "--save-labels", 403 | default=True, 404 | type=str2bool, 405 | help="whether to save object labels or not", 406 | ) 407 | parser.add_argument( 408 | "--range-image", 409 | default=1, 410 | type=int, 411 | choices=[1, 2], 412 | help="which range image to use from Waymo", 413 | ) 414 | parser.add_argument( 415 | "--waymo-dir", 416 | type=str, 417 | required=True, 418 | help="the path to the directory where the Waymo data is stored", 419 | ) 420 | parser.add_argument( 421 | "--argo-dir", 422 | type=str, 423 | required=True, 424 | help="the path to the directory where the converted data should be written", 425 | ) 426 | args = parser.parse_args() 427 | main(args) 428 | -------------------------------------------------------------------------------- /waymo2argo/waymo_data_splits.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from typing import List 4 | 5 | 6 | def get_val_log_ids() -> List[str]: 7 | """ """ 8 | val_log_ids = [ 9 | "11450298750351730790_1431_750_1451_750", 10 | "11406166561185637285_1753_750_1773_750", 11 | "16979882728032305374_2719_000_2739_000", 12 | "12831741023324393102_2673_230_2693_230", 13 | "14486517341017504003_3406_349_3426_349", 14 | "12358364923781697038_2232_990_2252_990", 15 | "6183008573786657189_5414_000_5434_000", 16 | "3126522626440597519_806_440_826_440", 17 | "17152649515605309595_3440_000_3460_000", 18 | "10359308928573410754_720_000_740_000", 19 | "16751706457322889693_4475_240_4495_240", 20 | "14165166478774180053_1786_000_1806_000", 21 | "14081240615915270380_4399_000_4419_000", 22 | "1071392229495085036_1844_790_1864_790", 23 | "18305329035161925340_4466_730_4486_730", 24 | "13336883034283882790_7100_000_7120_000", 25 | "11356601648124485814_409_000_429_000", 26 | "1943605865180232897_680_000_700_000", 27 | "13178092897340078601_5118_604_5138_604", 28 | "15488266120477489949_3162_920_3182_920", 29 | "12306251798468767010_560_000_580_000", 30 | "4612525129938501780_340_000_360_000", 31 | "260994483494315994_2797_545_2817_545", 32 | "2105808889850693535_2295_720_2315_720", 33 | "18446264979321894359_3700_000_3720_000", 34 | "8331804655557290264_4351_740_4371_740", 35 | "4013125682946523088_3540_000_3560_000", 36 | "14931160836268555821_5778_870_5798_870", 37 | "14300007604205869133_1160_000_1180_000", 38 | "13573359675885893802_1985_970_2005_970", 39 | "8079607115087394458_1240_000_1260_000", 40 | "18331704533904883545_1560_000_1580_000", 41 | "12866817684252793621_480_000_500_000", 42 | "12374656037744638388_1412_711_1432_711", 43 | "7493781117404461396_2140_000_2160_000", 44 | "15224741240438106736_960_000_980_000", 45 | "11616035176233595745_3548_820_3568_820", 46 | "10837554759555844344_6525_000_6545_000", 47 | "10689101165701914459_2072_300_2092_300", 48 | "8845277173853189216_3828_530_3848_530", 49 | "6680764940003341232_2260_000_2280_000", 50 | "30779396576054160_1880_000_1900_000", 51 | "17791493328130181905_1480_000_1500_000", 52 | "9024872035982010942_2578_810_2598_810", 53 | "8302000153252334863_6020_000_6040_000", 54 | "4246537812751004276_1560_000_1580_000", 55 | "346889320598157350_798_187_818_187", 56 | "16229547658178627464_380_000_400_000", 57 | "13356997604177841771_3360_000_3380_000", 58 | "1105338229944737854_1280_000_1300_000", 59 | "4816728784073043251_5273_410_5293_410", 60 | "18024188333634186656_1566_600_1586_600", 61 | "14956919859981065721_1759_980_1779_980", 62 | "14244512075981557183_1226_840_1246_840", 63 | "13184115878756336167_1354_000_1374_000", 64 | "12102100359426069856_3931_470_3951_470", 65 | "2506799708748258165_6455_000_6475_000", 66 | "14739149465358076158_4740_000_4760_000", 67 | "2308204418431899833_3575_000_3595_000", 68 | "18333922070582247333_320_280_340_280", 69 | "17763730878219536361_3144_635_3164_635", 70 | "12657584952502228282_3940_000_3960_000", 71 | "11901761444769610243_556_000_576_000", 72 | "17135518413411879545_1480_000_1500_000", 73 | "16767575238225610271_5185_000_5205_000", 74 | "14127943473592757944_2068_000_2088_000", 75 | "6324079979569135086_2372_300_2392_300", 76 | "5847910688643719375_180_000_200_000", 77 | "447576862407975570_4360_000_4380_000", 78 | "3015436519694987712_1300_000_1320_000", 79 | "271338158136329280_2541_070_2561_070", 80 | "9164052963393400298_4692_970_4712_970", 81 | "7932945205197754811_780_000_800_000", 82 | "5183174891274719570_3464_030_3484_030", 83 | "2736377008667623133_2676_410_2696_410", 84 | "17626999143001784258_2760_000_2780_000", 85 | "15724298772299989727_5386_410_5406_410", 86 | "12134738431513647889_3118_000_3138_000", 87 | "5990032395956045002_6600_000_6620_000", 88 | "5832416115092350434_60_000_80_000", 89 | "4195774665746097799_7300_960_7320_960", 90 | "3915587593663172342_10_000_30_000", 91 | "17136314889476348164_979_560_999_560", 92 | "14687328292438466674_892_000_912_000", 93 | "4575389405178805994_4900_000_4920_000", 94 | "14663356589561275673_935_195_955_195", 95 | "13469905891836363794_4429_660_4449_660", 96 | "12820461091157089924_5202_916_5222_916", 97 | "11660186733224028707_420_000_440_000", 98 | "9243656068381062947_1297_428_1317_428", 99 | "9041488218266405018_6454_030_6474_030", 100 | "8679184381783013073_7740_000_7760_000", 101 | "6637600600814023975_2235_000_2255_000", 102 | "2367305900055174138_1881_827_1901_827", 103 | "14811410906788672189_373_113_393_113", 104 | "11048712972908676520_545_000_565_000", 105 | "7253952751374634065_1100_000_1120_000", 106 | "4423389401016162461_4235_900_4255_900", 107 | "4409585400955983988_3500_470_3520_470", 108 | "2834723872140855871_1615_000_1635_000", 109 | "17244566492658384963_2540_000_2560_000", 110 | "15096340672898807711_3765_000_3785_000", 111 | "15021599536622641101_556_150_576_150", 112 | "1331771191699435763_440_000_460_000", 113 | "8133434654699693993_1162_020_1182_020", 114 | "5574146396199253121_6759_360_6779_360", 115 | "366934253670232570_2229_530_2249_530", 116 | "3077229433993844199_1080_000_1100_000", 117 | "3039251927598134881_1240_610_1260_610", 118 | "15611747084548773814_3740_000_3760_000", 119 | "1405149198253600237_160_000_180_000", 120 | "1024360143612057520_3580_000_3600_000", 121 | "4759225533437988401_800_000_820_000", 122 | "2335854536382166371_2709_426_2729_426", 123 | "1505698981571943321_1186_773_1206_773", 124 | "12496433400137459534_120_000_140_000", 125 | "8398516118967750070_3958_000_3978_000", 126 | "8137195482049459160_3100_000_3120_000", 127 | "17860546506509760757_6040_000_6060_000", 128 | "16204463896543764114_5340_000_5360_000", 129 | "15948509588157321530_7187_290_7207_290", 130 | "10868756386479184868_3000_000_3020_000", 131 | "7988627150403732100_1487_540_1507_540", 132 | "5772016415301528777_1400_000_1420_000", 133 | "3577352947946244999_3980_000_4000_000", 134 | "17612470202990834368_2800_000_2820_000", 135 | "10335539493577748957_1372_870_1392_870", 136 | "933621182106051783_4160_000_4180_000", 137 | "89454214745557131_3160_000_3180_000", 138 | "5289247502039512990_2640_000_2660_000", 139 | "3651243243762122041_3920_000_3940_000", 140 | "16213317953898915772_1597_170_1617_170", 141 | "14383152291533557785_240_000_260_000", 142 | "14333744981238305769_5658_260_5678_260", 143 | "15959580576639476066_5087_580_5107_580", 144 | "14262448332225315249_1280_000_1300_000", 145 | "13415985003725220451_6163_000_6183_000", 146 | "12940710315541930162_2660_000_2680_000", 147 | "11387395026864348975_3820_000_3840_000", 148 | "5302885587058866068_320_000_340_000", 149 | "4690718861228194910_1980_000_2000_000", 150 | "1906113358876584689_1359_560_1379_560", 151 | "9114112687541091312_1100_000_1120_000", 152 | "2624187140172428292_73_000_93_000", 153 | "13694146168933185611_800_000_820_000", 154 | "9472420603764812147_850_000_870_000", 155 | "9443948810903981522_6538_870_6558_870", 156 | "902001779062034993_2880_000_2900_000", 157 | "7799643635310185714_680_000_700_000", 158 | "7650923902987369309_2380_000_2400_000", 159 | "7119831293178745002_1094_720_1114_720", 160 | "1464917900451858484_1960_000_1980_000", 161 | "967082162553397800_5102_900_5122_900", 162 | "8888517708810165484_1549_770_1569_770", 163 | "4764167778917495793_860_000_880_000", 164 | "10247954040621004675_2180_000_2200_000", 165 | "6074871217133456543_1000_000_1020_000", 166 | "11434627589960744626_4829_660_4849_660", 167 | "9231652062943496183_1740_000_1760_000", 168 | "4490196167747784364_616_569_636_569", 169 | "10448102132863604198_472_000_492_000", 170 | "6491418762940479413_6520_000_6540_000", 171 | "13299463771883949918_4240_000_4260_000", 172 | "10203656353524179475_7625_000_7645_000", 173 | "662188686397364823_3248_800_3268_800", 174 | "17962792089966876718_2210_933_2230_933", 175 | "8956556778987472864_3404_790_3424_790", 176 | "8907419590259234067_1960_000_1980_000", 177 | "8506432817378693815_4860_000_4880_000", 178 | "2551868399007287341_3100_000_3120_000", 179 | "7163140554846378423_2717_820_2737_820", 180 | "6707256092020422936_2352_392_2372_392", 181 | "6001094526418694294_4609_470_4629_470", 182 | "17344036177686610008_7852_160_7872_160", 183 | "15396462829361334065_4265_000_4285_000", 184 | "13941626351027979229_3363_930_3383_930", 185 | "15496233046893489569_4551_550_4571_550", 186 | "18252111882875503115_378_471_398_471", 187 | "17539775446039009812_440_000_460_000", 188 | "4426410228514970291_1620_000_1640_000", 189 | "7732779227944176527_2120_000_2140_000", 190 | "5373876050695013404_3817_170_3837_170", 191 | "14107757919671295130_3546_370_3566_370", 192 | "10289507859301986274_4200_000_4220_000", 193 | "17703234244970638241_220_000_240_000", 194 | "18045724074935084846_6615_900_6635_900", 195 | "9579041874842301407_1300_000_1320_000", 196 | "6161542573106757148_585_030_605_030", 197 | "2094681306939952000_2972_300_2992_300", 198 | "17694030326265859208_2340_000_2360_000", 199 | "15028688279822984888_1560_000_1580_000", 200 | "9265793588137545201_2981_960_3001_960", 201 | "17065833287841703_2980_000_3000_000", 202 | "5372281728627437618_2005_000_2025_000", 203 | "3731719923709458059_1540_000_1560_000", 204 | "14624061243736004421_1840_000_1860_000", 205 | "13982731384839979987_1680_000_1700_000", 206 | "191862526745161106_1400_000_1420_000", 207 | "1457696187335927618_595_027_615_027", 208 | "11037651371539287009_77_670_97_670", 209 | "4854173791890687260_2880_000_2900_000", 210 | "272435602399417322_2884_130_2904_130", 211 | ] 212 | return val_log_ids 213 | 214 | 215 | def get_test_log_ids() -> List[str]: 216 | """ """ 217 | test_log_ids = [ 218 | "10084636266401282188_1120_000_1140_000", 219 | "4054036670499089296_2300_000_2320_000", 220 | "16367045247642649300_3060_000_3080_000", 221 | "13732041959462600641_720_000_740_000", 222 | "10149575340910243572_2720_000_2740_000", 223 | "4593468568253300598_1620_000_1640_000", 224 | "16418654553014119039_4340_000_4360_000", 225 | "13748565785898537200_680_000_700_000", 226 | "10161761842905385678_760_000_780_000", 227 | "4632556232973423919_2940_000_2960_000", 228 | "1664548685643064400_2240_000_2260_000", 229 | "1376304843325714018_3420_000_3440_000", 230 | "6862795755554967162_2280_000_2300_000", 231 | "5810494922060252082_3720_000_3740_000", 232 | "11672844176539348333_4440_000_4460_000", 233 | "2383902674438058857_4420_000_4440_000", 234 | "10504764403039842352_460_000_480_000", 235 | "10410418118434245359_5140_000_5160_000", 236 | "15272375112495403395_620_000_640_000", 237 | "4140965781175793864_460_000_480_000", 238 | "4916600861562283346_3880_000_3900_000", 239 | "16721473705085324478_2580_000_2600_000", 240 | "7247823803417339098_2320_000_2340_000", 241 | "13781857304705519152_2740_000_2760_000", 242 | "1735154401471216485_440_000_460_000", 243 | "14586026017427828517_700_000_720_000", 244 | "5585555620508986875_720_000_740_000", 245 | "11867874114645674271_600_000_620_000", 246 | "10802932587105534078_1280_000_1300_000", 247 | "8229317157758012712_3860_000_3880_000", 248 | "6922883602463663456_2220_000_2240_000", 249 | "5927928428387529213_1640_000_1660_000", 250 | "2709541197299883157_1140_000_1160_000", 251 | "6228701001600487900_720_000_740_000", 252 | "5154724129640787887_4840_000_4860_000", 253 | "15410814825574326536_2620_000_2640_000", 254 | "16942495693882305487_4340_000_4360_000", 255 | "12056192874455954437_140_000_160_000", 256 | "1703056599550681101_4380_000_4400_000", 257 | "13790309965076620852_6520_000_6540_000", 258 | "10488772413132920574_680_000_700_000", 259 | "9145030426583202228_1060_000_1080_000", 260 | "2363225200168330815_760_000_780_000", 261 | "1936395688683397781_2580_000_2600_000", 262 | "16743182245734335352_1260_000_1280_000", 263 | "3510690431623954420_7700_000_7720_000", 264 | "13887882285811432765_740_000_760_000", 265 | "11096867396355523348_1460_000_1480_000", 266 | "6503078254504013503_3440_000_3460_000", 267 | "10998289306141768318_1280_000_1300_000", 268 | "10980133015080705026_780_000_800_000", 269 | "8085856200343017603_4120_000_4140_000", 270 | "7855150647548977812_3900_000_3920_000", 271 | "5046614299208670619_1760_000_1780_000", 272 | "2795127582672852315_4140_000_4160_000", 273 | "17595457728136868510_860_000_880_000", 274 | "7435516779413778621_4440_000_4460_000", 275 | "14386836877680112549_4460_000_4480_000", 276 | "5993415832220804439_1020_000_1040_000", 277 | "3328513486129168664_2080_000_2100_000", 278 | "2942662230423855469_880_000_900_000", 279 | "13347759874869607317_1540_000_1560_000", 280 | "6079272500228273268_2480_000_2500_000", 281 | "5026942594071056992_3120_000_3140_000", 282 | "17212025549630306883_2500_000_2520_000", 283 | "12555145882162126399_1180_000_1200_000", 284 | "10534368980139017457_4480_000_4500_000", 285 | "2257381802419655779_820_000_840_000", 286 | "6278307160249415497_1700_000_1720_000", 287 | "12892154548237137398_2820_000_2840_000", 288 | "8623236016759087157_3500_000_3520_000", 289 | "792520390268391604_780_000_800_000", 290 | "17136775999940024630_4860_000_4880_000", 291 | "16050146835908439029_4500_000_4520_000", 292 | "14918167237855418464_1420_000_1440_000", 293 | "13034900465317073842_1700_000_1720_000", 294 | "6174376739759381004_3240_000_3260_000", 295 | "5683383258122801095_1040_000_1060_000", 296 | "16951245307634830999_1400_000_1420_000", 297 | "10940141908690367388_4420_000_4440_000", 298 | "2218963221891181906_4360_000_4380_000", 299 | "17387485694427326992_760_000_780_000", 300 | "7844300897851889216_500_000_520_000", 301 | "7240042450405902042_580_000_600_000", 302 | "4037952268810331899_2420_000_2440_000", 303 | "14631629219048194483_2720_000_2740_000", 304 | "13787943721654585343_1220_000_1240_000", 305 | "9584760613582366524_1620_000_1640_000", 306 | "5444585006397501511_160_000_180_000", 307 | "4008112367880337022_3680_000_3700_000", 308 | "15865907199900332614_760_000_780_000", 309 | "8249122135171526629_520_000_540_000", 310 | "3275806206237593341_1260_000_1280_000", 311 | "2714318267497393311_480_000_500_000", 312 | "12537711031998520792_3080_000_3100_000", 313 | "8566480970798227989_500_000_520_000", 314 | "15739335479094705947_1420_000_1440_000", 315 | "13944616099709049906_1020_000_1040_000", 316 | "13585389505831587326_2560_000_2580_000", 317 | "10649066155322078676_1660_000_1680_000", 318 | "365416647046203224_1080_000_1100_000", 319 | "1417898473608326362_2560_000_2580_000", 320 | "2830680430134047327_1720_000_1740_000", 321 | "9350911198443552989_680_000_700_000", 322 | "614453665074997770_1060_000_1080_000", 323 | "14643284977980826278_520_000_540_000", 324 | "12153647356523920032_2560_000_2580_000", 325 | "11933765568165455008_2940_000_2960_000", 326 | "8688567562597583972_940_000_960_000", 327 | "7886090431228432618_1060_000_1080_000", 328 | "7511993111693456743_3880_000_3900_000", 329 | "39847154216997509_6440_000_6460_000", 330 | "17174012103392027911_3500_000_3520_000", 331 | "8920841445900141920_1700_000_1720_000", 332 | "17052666463197337241_4560_000_4580_000", 333 | "3459095437766396887_1600_000_1620_000", 334 | "8684065200957554260_2700_000_2720_000", 335 | "6259508587655502768_780_000_800_000", 336 | "2906594041697319079_3040_000_3060_000", 337 | "14470988792985854683_760_000_780_000", 338 | "5648007586817904385_3220_000_3240_000", 339 | "17756183617755834457_1940_000_1960_000", 340 | "1765211916310163252_4400_000_4420_000", 341 | "13849332693800388551_960_000_980_000", 342 | "5764319372514665214_2480_000_2500_000", 343 | "8197312656120253218_3120_000_3140_000", 344 | "3645211352574995740_3540_000_3560_000", 345 | "5638240639308158118_4220_000_4240_000", 346 | "3522804493060229409_3400_000_3420_000", 347 | "3341890853207909601_1020_000_1040_000", 348 | "18149616047892103767_2460_000_2480_000", 349 | "17835886859721116155_1860_000_1880_000", 350 | "16062780403777359835_2580_000_2600_000", 351 | "15370024704033662533_1240_000_1260_000", 352 | "11436803605426256250_1720_000_1740_000", 353 | "684234579698396203_2540_000_2560_000", 354 | "17792628511034220885_2360_000_2380_000", 355 | "14188689528137485670_2660_000_2680_000", 356 | "17262030607996041518_540_000_560_000", 357 | "2374138435300423201_2600_000_2620_000", 358 | "9806821842001738961_4460_000_4480_000", 359 | "8993680275027614595_2520_000_2540_000", 360 | "11987368976578218644_1340_000_1360_000", 361 | "3122599254941105215_2980_000_3000_000", 362 | "2601205676330128831_4880_000_4900_000", 363 | "4045613324047897473_940_000_960_000", 364 | "3485136235103477552_600_000_620_000", 365 | "9355489589631690177_4800_000_4820_000", 366 | "14737335824319407706_1980_000_2000_000", 367 | "3400465735719851775_1400_000_1420_000", 368 | ] 369 | return test_log_ids 370 | --------------------------------------------------------------------------------