├── .gitmodules ├── CMakeLists.txt ├── README.md └── demo.cpp /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "external/wiiuse"] 2 | path = external/wiiuse 3 | url = https://github.com/fta2012/wiiuse.git 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.6) 2 | 3 | project(WiimotePositionTrackingDemo) 4 | 5 | add_subdirectory(external/wiiuse) 6 | 7 | find_package(SDL REQUIRED) 8 | find_package(OpenGL REQUIRED) 9 | find_package(OpenCV REQUIRED) 10 | 11 | ADD_DEFINITIONS( 12 | -std=c++11 13 | ) 14 | set(CMAKE_CXX_FLAGS "-g -Wall") 15 | 16 | include_directories(${WIIUSE_INCLUDE_DIR} ${SDL_INCLUDE_DIR} ${OPENGL_INCLUDE_DIR} ${OpenCV_INCLUDE_DIRS}) 17 | add_executable(demo demo.cpp) 18 | target_link_libraries(demo ${WIIUSE_LIBRARY} ${SDL_LIBRARY} ${OPENGL_LIBRARIES} ${OpenCV_LIBS}) 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | A demo of how to do simple 6DOF position tracking using a wiimote. Requires 4 IR LEDs as tracking markers. [Blog](http://franklinta.com/2014/09/30/6dof-positional-tracking-with-the-wiimote/) has details on how to make one. 2 | 3 | Uses [wiiuse](https://github.com/rpavlik/wiiuse) for connecting to the wiimote and getting raw IR values. Uses OpenCV's [solvepnp](http://docs.opencv.org/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html#solvepnp) for solving for camera pose. Drawing was based off of wiiuse's example which requires SDL 1.2 and OpenGL. 4 | 5 | ### Build and run: 6 | git submodule init 7 | git submodule update 8 | mkdir build 9 | cd build 10 | cmake .. 11 | make 12 | ./demo 13 | 14 | Press the sync button on the back of a wiimote to connect while it is searching for devices. 15 | 16 | Controls: 17 | 18 | A - switch between world frame and camera frame 19 | B - draw the current path 20 | Home - clear the drawn path 21 | UP/DOWN - change the rendered camera size 22 | LEFT/RIGHT - rotate the mapping of the leds (press this if your world y-axis is not pointing up or your camera y-axis is not pointing down) 23 | PLUS/MINUS - change IR sensitivity 24 | ONE - toggle whether to draw the wiimote 25 | TWO - use the last few frames to print out a calibrated camera matrix 26 | -------------------------------------------------------------------------------- /demo.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | #include 14 | 15 | #include 16 | 17 | using namespace std; 18 | 19 | // Dimensions of the image taken by the IR camera 20 | int image_width = 1024; 21 | int image_height = 768; 22 | 23 | // Location of the infrared LEDs in world frame. 24 | // This tracker is in the shape of a square. 25 | vector object_points = { 26 | {1, 1, 0}, 27 | {1, 3, 0}, 28 | {3, 3, 0}, 29 | {3, 1, 0}, 30 | }; 31 | 32 | // The camera's position in world frame while 'B' was held 33 | vector camera_path; 34 | 35 | // Queue recording the last few images where all 4 points were visible. 36 | deque> image_points_queue; 37 | 38 | // Whether we should be drawing using the camera or world coordinate frame 39 | enum render_mode_t { 40 | CAMERA_FRAME = 1, 41 | WORLD_FRAME 42 | }; 43 | enum render_mode_t render_mode = CAMERA_FRAME; 44 | 45 | // How large to draw the camera 46 | float camera_scale = 3; 47 | 48 | // Whether to draw the wiimote 49 | bool draw_wiimote = true; 50 | 51 | 52 | void handle_event(struct wiimote_t* wiimote) { 53 | if (IS_JUST_PRESSED(wiimote, WIIMOTE_BUTTON_LEFT)) { 54 | rotate(object_points.begin(), object_points.end() - 1, object_points.end()); 55 | } 56 | if (IS_JUST_PRESSED(wiimote, WIIMOTE_BUTTON_RIGHT)) { 57 | rotate(object_points.begin(), object_points.begin() + 1, object_points.end()); 58 | } 59 | 60 | if (IS_JUST_PRESSED(wiimote, WIIMOTE_BUTTON_UP)) { 61 | camera_scale += 1; 62 | } 63 | if (IS_JUST_PRESSED(wiimote, WIIMOTE_BUTTON_DOWN)) { 64 | camera_scale -= 1; 65 | } 66 | 67 | if (IS_JUST_PRESSED(wiimote, WIIMOTE_BUTTON_A)) { 68 | render_mode = (render_mode == CAMERA_FRAME) ? WORLD_FRAME : CAMERA_FRAME; 69 | } 70 | 71 | if (IS_JUST_PRESSED(wiimote, WIIMOTE_BUTTON_HOME)) { 72 | camera_path.clear(); 73 | } 74 | 75 | if (IS_JUST_PRESSED(wiimote, WIIMOTE_BUTTON_PLUS)) { 76 | int level; 77 | WIIUSE_GET_IR_SENSITIVITY(wiimote, &level); 78 | wiiuse_set_ir_sensitivity(wiimote, level + 1); 79 | } 80 | if (IS_JUST_PRESSED(wiimote, WIIMOTE_BUTTON_MINUS)) { 81 | int level; 82 | WIIUSE_GET_IR_SENSITIVITY(wiimote, &level); 83 | wiiuse_set_ir_sensitivity(wiimote, level - 1); 84 | } 85 | 86 | if (IS_JUST_PRESSED(wiimote, WIIMOTE_BUTTON_ONE)) { 87 | draw_wiimote = !draw_wiimote; 88 | } 89 | 90 | if (IS_JUST_PRESSED(wiimote, WIIMOTE_BUTTON_TWO)) { 91 | if (!image_points_queue.empty()) { 92 | cout << "Calibrating... " << endl; 93 | vector> objectPointsVector(image_points_queue.size(), object_points); 94 | cv::Mat camera_matrix, dist_coeffs; 95 | vector rvecs, tvecs; 96 | calibrateCamera(objectPointsVector, 97 | vector>(image_points_queue.begin(), image_points_queue.end()), 98 | cvSize(image_width, image_height), 99 | camera_matrix, 100 | dist_coeffs, 101 | rvecs, 102 | tvecs); 103 | cout << "cameraMatrix " << camera_matrix << endl; 104 | cout << "distCoeffs " << dist_coeffs << endl; 105 | } 106 | } 107 | } 108 | 109 | vector get_image_points(wiimote* wiimote) { 110 | vector image_points; 111 | 112 | for (const auto & dot : wiimote->ir.dot) { 113 | // Only keep visible points 114 | if (dot.visible) { 115 | // Flip so that (0, 0) corresponds with the top left corner of the image 116 | image_points.emplace_back(image_width - 1 - dot.rx, image_height - 1 - dot.ry); 117 | } 118 | } 119 | 120 | // If all 4 points are visible, canonicalize the ordering 121 | if (image_points.size() == object_points.size()) { 122 | // Make sure it is in counterclockwise order so it can match up with object points 123 | convexHull(image_points, image_points); 124 | if (image_points.size() == object_points.size()) { 125 | // If there is a previous image, try to rotate the points until they match up 126 | if (!image_points_queue.empty()) { 127 | auto & previous_image = image_points_queue.back(); 128 | float min_dist = FLT_MAX; 129 | int min_offset = 0; 130 | for (int offset = 0; offset < image_points.size(); offset++) { 131 | float dist = 0; 132 | for (int i = 0; i < image_points.size(); i++) { 133 | cv::Point2f diff = previous_image[i] - image_points[(i + offset) % image_points.size()]; 134 | dist += norm(diff); 135 | } 136 | if (dist < min_dist) { 137 | min_dist = dist; 138 | min_offset = offset; 139 | } 140 | } 141 | rotate(image_points.begin(), image_points.begin() + min_offset, image_points.end()); 142 | } 143 | 144 | // Save the canonicalized image for this frame 145 | image_points_queue.push_back(image_points); 146 | // Limit the number of frames to keep 147 | while (image_points_queue.size() > 15) { 148 | image_points_queue.pop_front(); 149 | } 150 | } 151 | } 152 | 153 | return image_points; 154 | } 155 | 156 | void display(wiimote* wiimote) { 157 | vector image_points = get_image_points(wiimote); 158 | glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); 159 | 160 | if (render_mode == CAMERA_FRAME) { 161 | // Draw the LEDs in 2D 162 | glMatrixMode(GL_PROJECTION); 163 | glLoadIdentity(); 164 | gluOrtho2D(0, image_width, image_height, 0); 165 | 166 | glMatrixMode(GL_MODELVIEW); 167 | glLoadIdentity(); 168 | 169 | glPointSize(5.0); 170 | glBegin(GL_POINTS); 171 | for (int i = 0; i < image_points.size(); i++) { 172 | glColor3f(i == 0 || i == 3, i == 1 || i == 3, i == 2 || i == 3); 173 | glVertex2f(image_points[i].x, image_points[i].y); 174 | } 175 | glEnd(); 176 | } 177 | 178 | if (image_points.size() < object_points.size()) { 179 | // Not all 4 points are visible so we can't solve, just return 180 | if (render_mode == CAMERA_FRAME) { 181 | SDL_GL_SwapBuffers(); 182 | } 183 | return; 184 | } 185 | 186 | // Camera intrinsic parameters. These weren't obtained from calibration but works well enough. 187 | double fx = 1700; 188 | double fy = 1700; 189 | double cx = image_width / 2; 190 | double cy = image_height / 2; 191 | cv::Mat intrinsic = (cv::Mat_(3, 3) << 192 | fx, 0, cx, 193 | 0, fy, cy, 194 | 0, 0, 1 195 | ); 196 | 197 | // Solve for camera extrinsic parameters. 198 | // This gives us the rotation and translation of the world frame from the camera frame. 199 | cv::Mat rvec, tvec; 200 | solvePnP(object_points, image_points, intrinsic, cv::noArray(), rvec, tvec); 201 | cv::Mat R; 202 | Rodrigues(rvec, R); 203 | cv::Mat extrinsic = cv::Mat::eye(4, 4, CV_64F); 204 | R.copyTo(extrinsic.rowRange(0, 3).colRange(0, 3)); 205 | tvec.copyTo(extrinsic.rowRange(0, 3).col(3)); 206 | 207 | // Find the inverse of the extrinsic matrix (should be the same as just calling extrinsic.inv()) 208 | cv::Mat extrinsic_inv_R = R.t(); // inverse of a rotational matrix is its transpose 209 | cv::Mat extrinsic_inv_tvec = -extrinsic_inv_R * tvec; 210 | cv::Mat extrinsic_inv = cv::Mat::eye(4, 4, CV_64F); 211 | extrinsic_inv_R.copyTo(extrinsic_inv.rowRange(0, 3).colRange(0, 3)); 212 | extrinsic_inv_tvec.copyTo(extrinsic_inv.rowRange(0, 3).col(3)); 213 | 214 | // Find the inverse of the intrinsic matrix 215 | cv::Mat intrinsic_inv = (cv::Mat_(4, 4) << 216 | 1 / fx, 0, -cx / fx, 0, 217 | 0, 1 / fy, -cy / fy, 0, 218 | 0, 0, 1, 0, 219 | 0, 0, 0, 1 220 | ); 221 | 222 | // Record the position of the camera, which is (extrinsic_inv * [0, 0, 0, 1]) 223 | if (IS_PRESSED(wiimote, WIIMOTE_BUTTON_B)) { 224 | camera_path.push_back(extrinsic_inv_tvec); 225 | } 226 | 227 | if (render_mode == CAMERA_FRAME) { 228 | glMatrixMode(GL_PROJECTION); 229 | glLoadIdentity(); 230 | // Since our intrinsic matrix has center in the middle of image we can use gluPerspective with the correct fov and aspect. 231 | gluPerspective(2 * atan(cy / fy) * 180 / M_PI, fy * image_width / (fx * image_height), 0.1, 100.0); 232 | 233 | // The front of the camera in computer vision is the positive z-axis but is the negative z-axis in opengl 234 | // Rotate the z axis around 235 | GLfloat cv_to_gl[16] = { 236 | 1, 0, 0, 0, 237 | 0, -1, 0, 0, 238 | 0, 0, -1, 0, 239 | 0, 0, 0, 1, 240 | }; 241 | glMultMatrixf(cv_to_gl); 242 | 243 | // Apply the extrinsic matrix in column major order. 244 | // We can now draw stuff in world coordinates and to show what the IR camera would see. 245 | glMultMatrixd(cv::Mat(extrinsic.t()).ptr(0)); 246 | } else { 247 | // Some arbitrary fixed viewing direction of the world frame 248 | glMatrixMode(GL_PROJECTION); 249 | glLoadIdentity(); 250 | gluPerspective(60.0f, (float)image_width / image_height, 0.1f, 100.0f); 251 | 252 | glMatrixMode(GL_MODELVIEW); 253 | glLoadIdentity(); 254 | gluLookAt(10, 10, 30, 1.5, 1.5, 0, 0, 1, 0); 255 | } 256 | 257 | /* Draw in world coordinates */ 258 | 259 | // Draw world frame axes 260 | glBegin(GL_LINES); 261 | glColor3f(1.0, 0.0, 0.0); glVertex3f(0, 0, 0); glVertex3f(1000, 0, 0); 262 | glColor3f(0.0, 1.0, 0.0); glVertex3f(0, 0, 0); glVertex3f(0, 1000, 0); 263 | glColor3f(0.0, 0.0, 1.0); glVertex3f(0, 0, 0); glVertex3f(0, 0, 1000); 264 | glEnd(); 265 | 266 | // Draw the square representing the LEDs 267 | glBegin(GL_LINE_LOOP); 268 | glColor3f(1.0, 1.0, 1.0); 269 | for (const auto & object_point : object_points) { 270 | glVertex3f(object_point.x, object_point.y, 0); 271 | } 272 | glEnd(); 273 | 274 | // Draw lines from the leds to where they are projected on the camera 275 | vector projected_points; 276 | projectPoints(object_points, rvec, tvec, intrinsic, cv::noArray(), projected_points); 277 | glColor3f(0.0, 0.0, 1.0); 278 | glBegin(GL_LINES); 279 | for (int i = 0; i < object_points.size(); i++) { 280 | glVertex3f(object_points[i].x, object_points[i].y, 0); 281 | cv::Mat p = (cv::Mat_(4, 1) << projected_points[i].x * camera_scale, projected_points[i].y * camera_scale, camera_scale, 1); 282 | p = extrinsic_inv * intrinsic_inv * p; 283 | assert(p.at(3) == 1); 284 | glVertex3f(p.at(0), p.at(1), p.at(2)); 285 | } 286 | glEnd(); 287 | 288 | // Draw the camera path 289 | glBegin(GL_LINES); 290 | glColor3f(1.0, 1.0, 1.0); 291 | for (int i = 1; i < camera_path.size(); i++) { 292 | if (cv::norm(camera_path[i], camera_path[i - 1]) > .5) // Skip consecutive points that are too far apart 293 | continue; 294 | glVertex3f(camera_path[i - 1].at(0), camera_path[i - 1].at(1), camera_path[i - 1].at(2)); 295 | glVertex3f(camera_path[i].at(0), camera_path[i].at(1), camera_path[i].at(2)); 296 | } 297 | glEnd(); 298 | 299 | /* Draw in camera frame */ 300 | glMultMatrixd(cv::Mat(extrinsic_inv.t()).ptr(0)); 301 | 302 | // Draw camera frame axes 303 | glBegin(GL_LINES); 304 | glColor3f(1.0, 0.0, 0.0); glVertex3f(0.0, 0.0, 0.0); glVertex3f(3.0, 0.0, 0.0); 305 | glColor3f(0.0, 1.0, 0.0); glVertex3f(0.0, 0.0, 0.0); glVertex3f(0.0, 3.0, 0.0); 306 | glEnd(); 307 | 308 | // Draw the wiimote 309 | if (draw_wiimote) { 310 | float wiimote_width = 1.43; 311 | float wiimote_height = 1.21; 312 | float wiimote_length = 5.8; 313 | glColor3f(1, 1, 1); 314 | glBegin(GL_LINE_LOOP); 315 | glVertex3f( wiimote_width / 2, wiimote_height / 2, 0); 316 | glVertex3f( wiimote_width / 2, -wiimote_height / 2, 0); 317 | glVertex3f(-wiimote_width / 2, -wiimote_height / 2, 0); 318 | glVertex3f(-wiimote_width / 2, wiimote_height / 2, 0); 319 | glEnd(); 320 | glBegin(GL_LINES); 321 | glVertex3f( wiimote_width / 2, wiimote_height / 2, 0); glVertex3f( wiimote_width / 2, wiimote_height / 2, -wiimote_length); 322 | glVertex3f( wiimote_width / 2, -wiimote_height / 2, 0); glVertex3f( wiimote_width / 2, -wiimote_height / 2, -wiimote_length); 323 | glVertex3f(-wiimote_width / 2, -wiimote_height / 2, 0); glVertex3f(-wiimote_width / 2, -wiimote_height / 2, -wiimote_length); 324 | glVertex3f(-wiimote_width / 2, wiimote_height / 2, 0); glVertex3f(-wiimote_width / 2, wiimote_height / 2, -wiimote_length); 325 | glEnd(); 326 | glBegin(GL_LINE_LOOP); 327 | glVertex3f( wiimote_width / 2, wiimote_height / 2, -wiimote_length); 328 | glVertex3f( wiimote_width / 2, -wiimote_height / 2, -wiimote_length); 329 | glVertex3f(-wiimote_width / 2, -wiimote_height / 2, -wiimote_length); 330 | glVertex3f(-wiimote_width / 2, wiimote_height / 2, -wiimote_length); 331 | glEnd(); 332 | } 333 | 334 | /* Draw on the image plane of the camera */ 335 | glMultMatrixd(cv::Mat(intrinsic_inv.t()).ptr(0)); 336 | 337 | vector image_plane = { 338 | {0, 0, camera_scale}, 339 | {image_width * camera_scale, 0, camera_scale}, 340 | {image_width * camera_scale, image_height * camera_scale, camera_scale}, 341 | {0, image_height * camera_scale, camera_scale}, 342 | }; 343 | 344 | // Draw the boundaries for the image plane 345 | glBegin(GL_LINE_LOOP); 346 | glColor3f(0.5, 0.5, 0.5); 347 | for (const auto & corner : image_plane) { 348 | glVertex3f(corner.x, corner.y, corner.z); 349 | } 350 | glEnd(); 351 | 352 | // Draw the lines connecting the plane to the pinhole of the camera 353 | glBegin(GL_LINES); 354 | glColor3f(0.5, 0.5, 0.5); 355 | for (const auto & corner : image_plane) { 356 | glVertex3f(0.0, 0.0, 0.0); 357 | glVertex3f(corner.x, corner.y, corner.z); 358 | } 359 | glEnd(); 360 | 361 | // Draw the image points on the image plane 362 | glColor3f(1.0, 0.0, 0.0); 363 | glBegin(GL_LINE_LOOP); 364 | for (const auto & p : image_points) { 365 | glVertex3f(p.x * camera_scale, p.y * camera_scale, camera_scale); 366 | } 367 | glEnd(); 368 | 369 | // Draw the projected object points on the image plane 370 | // Whenever an exact solution is found the actual image points (drawn in red) should be covered by this 371 | glColor3f(1.0, 1.0, 1.0); 372 | glBegin(GL_LINE_LOOP); 373 | for (const auto & p : projected_points) { 374 | glVertex3f(p.x * camera_scale, p.y * camera_scale, camera_scale); 375 | } 376 | glEnd(); 377 | 378 | SDL_GL_SwapBuffers(); 379 | } 380 | 381 | int main(int argc, char** argv) { 382 | wiimote** wiimotes = wiiuse_init(1); 383 | int found = wiiuse_find(wiimotes, 1, 5); 384 | if (!found) { 385 | printf("Failed to find any wiimote.\n"); 386 | return 0; 387 | } 388 | int connected = wiiuse_connect(wiimotes, 1); 389 | if (connected) { 390 | printf("Connected to %i wiimotes (of %i found).\n", connected, found); 391 | } else { 392 | printf("Failed to connect to any wiimote.\n"); 393 | return 0; 394 | } 395 | wiiuse_rumble(wiimotes[0], 1); 396 | this_thread::sleep_for(chrono::milliseconds(200)); 397 | wiiuse_rumble(wiimotes[0], 0); 398 | wiiuse_set_leds(wiimotes[0], WIIMOTE_LED_1); 399 | wiiuse_set_ir(wiimotes[0], 1); 400 | 401 | if (SDL_Init(SDL_INIT_VIDEO) < 0) { 402 | printf("Failed to initialize SDL: %s\n", SDL_GetError()); 403 | return 0; 404 | } 405 | SDL_WM_SetCaption("Wiimote camera pose estimation", "Wiimote camera pose estimation"); 406 | SDL_GL_SetAttribute(SDL_GL_DOUBLEBUFFER, 1); 407 | SDL_GL_SetAttribute(SDL_GL_DEPTH_SIZE, 16); 408 | SDL_SetVideoMode(image_width, image_height, 16, SDL_OPENGL); 409 | 410 | glEnable(GL_DEPTH_TEST); 411 | glDepthFunc(GL_LEQUAL); 412 | glClearColor(0, 0, 0, 0); 413 | glViewport(0, 0, image_width, image_height); 414 | 415 | chrono::high_resolution_clock::time_point last_render; 416 | chrono::high_resolution_clock::time_point last_report; 417 | int fps = 0; 418 | 419 | display(wiimotes[0]); 420 | while (1) { 421 | SDL_Event event; 422 | if (SDL_PollEvent(&event)) { 423 | switch (event.type) { 424 | case SDL_QUIT: 425 | SDL_Quit(); 426 | wiiuse_cleanup(wiimotes, 1); 427 | return 0; 428 | default: 429 | break; 430 | } 431 | } 432 | 433 | if (wiiuse_poll(wiimotes, 1)) { 434 | switch (wiimotes[0]->event) { 435 | case WIIUSE_EVENT: 436 | handle_event(wiimotes[0]); 437 | break; 438 | default: 439 | break; 440 | } 441 | } 442 | 443 | auto now = chrono::high_resolution_clock::now(); 444 | if (now - last_report >= std::chrono::seconds(1)) { 445 | printf("fps: %d\n", fps); 446 | fps = 0; 447 | last_report = now; 448 | } 449 | if (now - last_render >= std::chrono::milliseconds(1000) / 60) { 450 | display(wiimotes[0]); 451 | fps++; 452 | last_render = now; 453 | } 454 | } 455 | } 456 | --------------------------------------------------------------------------------