├── doc ├── requirements.txt ├── index.rst └── conf.py ├── examples ├── .gitignore ├── dump_undistorted.py ├── ir_camera.py ├── dump_big_depth.py └── dump_pcd.py ├── MANIFEST.in ├── requirements.txt ├── tox.ini ├── .gitignore ├── setup.py ├── .travis.yml ├── LICENCE.txt ├── README.rst ├── binding ├── freenect2_build.py └── freenect2-c.cpp └── freenect2 └── __init__.py /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | sphinxcontrib-napoleon 3 | -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints/ 2 | example*.pcd 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | recursive-include binding *.py *.cpp 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cffi 2 | enum34;python_version<"3.4" 3 | Pillow 4 | numpy 5 | future 6 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist=doc 3 | 4 | [testenv:doc] 5 | deps= 6 | -rrequirements.txt 7 | -rdoc/requirements.txt 8 | commands=sphinx-build -b html doc {envdir}/build/doc 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Various build artefacts 2 | build 3 | *.so 4 | *.dylib 5 | *.egg-info 6 | .eggs 7 | 8 | # Python bytecode cache 9 | __pycache__ 10 | *.pyc 11 | output.pcd 12 | dist 13 | .tox 14 | freenect2/_freenect2.* 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup, find_packages 3 | 4 | this_dir = os.path.abspath(os.path.dirname(__file__)) 5 | with open(os.path.join(this_dir, 'requirements.txt')) as fobj: 6 | install_requires = fobj.readlines() 7 | 8 | setup( 9 | name='freenect2', 10 | description='Python bindings for the libfreenect2 Kinect for Windows driver', 11 | version='0.2.3', 12 | author='Rich Wareham', 13 | author_email='rich.freenect2@richwareham.com', 14 | url='https://github.com/rjw57/freenect2-python', 15 | packages=find_packages(), 16 | install_requires=install_requires, 17 | setup_requires=['cffi>=1.0.0'], 18 | cffi_modules=['binding/freenect2_build.py:ffibuilder'], 19 | ) 20 | -------------------------------------------------------------------------------- /examples/dump_undistorted.py: -------------------------------------------------------------------------------- 1 | from freenect2 import Device, FrameType 2 | import numpy as np 3 | 4 | def main(): 5 | device = Device() 6 | frames = {} 7 | with device.running(): 8 | for type_, frame in device: 9 | frames[type_] = frame 10 | if FrameType.Color in frames and FrameType.Depth in frames: 11 | break 12 | 13 | rgb, depth = frames[FrameType.Color], frames[FrameType.Depth] 14 | undistorted, registered = device.registration.apply(rgb, depth) 15 | points_array = device.registration.get_points_xyz_array(undistorted) 16 | 17 | undistorted.to_image().save('undistorted_depth.tiff') 18 | registered.to_image().save('registered_rgb.png') 19 | np.savez('points.npz', points=points_array) 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | language: python 3 | python: 4 | - "2.7" 5 | - "3.4" 6 | - "3.5" 7 | addons: 8 | apt: 9 | sources: 10 | - sourceline: "ppa:floe/libusb" 11 | packages: 12 | - git 13 | - cmake 14 | - libusb-1.0-0-dev 15 | - libjpeg-turbo8-dev 16 | - libturbojpeg 17 | before_install: 18 | - git clone --depth=1 https://github.com/OpenKinect/libfreenect2 19 | - cd libfreenect2 20 | - mkdir build 21 | - cd build 22 | - cmake -DCMAKE_INSTALL_PREFIX=$HOME/.local .. 23 | - make all install 24 | - cd ../.. 25 | - pip install --upgrade pip setuptools 26 | - pip install tox 27 | script: 28 | - tox 29 | env: 30 | global: 31 | - PKG_CONFIG_PATH=$HOME/.local/lib/pkgconfig 32 | - LD_LIBRARY_PATH=$HOME/.local/lib 33 | deploy: 34 | provider: pages 35 | skip_cleanup: true 36 | github_token: $GITHUB_TOKEN 37 | local_dir: .tox/doc/build/doc 38 | on: 39 | branch: master 40 | python: "3.5" 41 | -------------------------------------------------------------------------------- /examples/ir_camera.py: -------------------------------------------------------------------------------- 1 | """ 2 | Simple IR camera using freenect2. Saves captured IR image 3 | to output.jpg. 4 | 5 | """ 6 | # Import parts of freenect2 we're going to use 7 | from freenect2 import Device, FrameType 8 | 9 | # We use numpy to process the raw IR frame 10 | import numpy as np 11 | 12 | # We use the Pillow library for saving the captured image 13 | from PIL import Image 14 | 15 | # Open default device 16 | device = Device() 17 | 18 | # Start the device 19 | with device.running(): 20 | # For each received frame... 21 | for type_, frame in device: 22 | # ...stop only when we get an IR frame 23 | if type_ is FrameType.Ir: 24 | break 25 | 26 | # Outside of the 'with' block, the device has been stopped again 27 | 28 | # The received IR frame is in the range 0 -> 65535. Normalise the 29 | # range to 0 -> 1 and take square root as a simple form of gamma 30 | # correction. 31 | ir_image = frame.to_array() 32 | ir_image /= ir_image.max() 33 | ir_image = np.sqrt(ir_image) 34 | 35 | # Use Pillow to save the IR image. 36 | Image.fromarray(256 * ir_image).convert('L').save('output.jpg') 37 | -------------------------------------------------------------------------------- /examples/dump_big_depth.py: -------------------------------------------------------------------------------- 1 | import json 2 | from freenect2 import Device, FrameType 3 | import numpy as np 4 | 5 | def main(): 6 | device = Device() 7 | frames = {} 8 | with device.running(): 9 | for type_, frame in device: 10 | frames[type_] = frame 11 | if FrameType.Color in frames and FrameType.Depth in frames: 12 | break 13 | 14 | rgb, depth = frames[FrameType.Color], frames[FrameType.Depth] 15 | undistorted, registered, big_depth = device.registration.apply( 16 | rgb, depth, with_big_depth=True) 17 | 18 | rgb.to_image().save('output_rgb.png') 19 | big_depth.to_image().save('output_depth.tiff') 20 | 21 | with open('output_calib.json', 'w') as fobj: 22 | json.dump({ 23 | 'color': dict( 24 | (k, getattr(device.color_camera_params, k)) 25 | for k in 'fx fy cx cy'.split()), 26 | 'ir': dict( 27 | (k, getattr(device.ir_camera_params, k)) 28 | for k in 'fx fy cx cy k1 k2 k3 p1 p2'.split()), 29 | }, fobj) 30 | 31 | if __name__ == '__main__': 32 | main() 33 | -------------------------------------------------------------------------------- /LICENCE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 Rich Wareham 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /examples/dump_pcd.py: -------------------------------------------------------------------------------- 1 | """ 2 | Capture a single RGB and depth frame and save them to output.pcd in 3 | the libpcl PCD format. View the resulting cloud with: 4 | 5 | pcl_viewer output.pcd 6 | 7 | """ 8 | from freenect2 import Device, FrameType 9 | import numpy as np 10 | 11 | # Open the default device and capture a color and depth frame. 12 | device = Device() 13 | frames = {} 14 | with device.running(): 15 | for type_, frame in device: 16 | frames[type_] = frame 17 | if FrameType.Color in frames and FrameType.Depth in frames: 18 | break 19 | 20 | # Use the factory calibration to undistort the depth frame and register the RGB 21 | # frame onto it. 22 | rgb, depth = frames[FrameType.Color], frames[FrameType.Depth] 23 | undistorted, registered, big_depth = device.registration.apply( 24 | rgb, depth, with_big_depth=True) 25 | 26 | # Combine the depth and RGB data together into a single point cloud. 27 | with open('output.pcd', 'wb') as fobj: 28 | device.registration.write_pcd(fobj, undistorted, registered) 29 | 30 | with open('output_big.pcd', 'wb') as fobj: 31 | device.registration.write_big_pcd(fobj, big_depth, rgb) 32 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | freenect2: Python bindings to libfreenect2 2 | ========================================== 3 | 4 | .. image:: https://travis-ci.org/rjw57/freenect2-python.svg?branch=master 5 | :target: https://travis-ci.org/rjw57/freenect2-python 6 | .. image:: https://zenodo.org/badge/85711795.svg 7 | :target: https://zenodo.org/badge/latestdoi/85711795 8 | 9 | The freenect2 module provides a Python interface to the `libfreenect2 10 | `_ library. The libfreenect2 11 | library provides a library allowing depth and RGB data to be extracted from a 12 | Kinect for Windows v2 (K4W2) device. 13 | 14 | If using this library in an academic context, please use the DOI linked above. 15 | 16 | Although a lot of libfreenect2 functionality is exposed, simple "single grab" 17 | usage of freenect2 should be simple. For example, here is how to grab a single 18 | depth frame and save it to a grayscale JPEG: 19 | 20 | .. code:: python 21 | 22 | from PIL.ImageMath import eval as im_eval 23 | from freenect2 import Device, FrameType 24 | 25 | device = Device() 26 | with device.running(): 27 | for frame_type, frame in device: 28 | if frame_type is FrameType.Depth: 29 | # Convert range of depth image to 0->255. 30 | norm_im = im_eval('convert(I / 16, "L")', I=frame.to_image()) 31 | norm_im.save('depth.jpg') 32 | break 33 | 34 | Getting started 35 | --------------- 36 | 37 | Installation instructions and reference documentation are available on the 38 | associated `documentation `_ pages. 39 | 40 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | freenect2 2 | ========= 3 | 4 | The :py:mod:`freenect2` module provides a Python interface to the `libfreenect2 5 | `_ library. The libfreenect2 library 6 | provides a library allowing depth and RGB data to be extracted from a Kinect for 7 | Windows v2 (K4W2) device. 8 | 9 | Installation 10 | ------------ 11 | 12 | libfreenect2 must be installed prior to building this module. See the 13 | `installation instructions 14 | `_ 15 | provided by libfreenect2. The following is a brief summary of the required 16 | actions: 17 | 18 | .. code:: console 19 | 20 | $ git clone https://github.com/OpenKinect/libfreenect2 21 | ... 22 | $ cd libfreenect2; mkdir build; cd build 23 | $ cmake -DCMAKE_INSTALL_PREFIX=$HOME/.local .. && make all install 24 | ... 25 | $ export PKG_CONFIG_PATH=$HOME/.local/lib/pkgconfig 26 | $ pip install --user freenect2 27 | 28 | pkg-config errors 29 | ''''''''''''''''' 30 | 31 | The libfreenect2 library uses pkg-config to record where it is installed on the 32 | system. If you install libfreenect2 manually as outlined above you will need to 33 | set the `PKG_CONFIG_PATH` environment variable. 34 | 35 | Simple usage 36 | ------------ 37 | 38 | The library is intended to provide an easy to use Pythonic interface to 39 | libfreenect2. As an example of library usage, here is a simple infrared (IR) 40 | camera which can be used to capture an image from the Kinect's build in IR 41 | camera: 42 | 43 | .. literalinclude:: ../examples/ir_camera.py 44 | :language: python 45 | 46 | The library also supports extracting real-world 3D co-ordinates from the depth 47 | maps and saving `libpcl `_-compatible PCD files: 48 | 49 | .. literalinclude:: ../examples/dump_pcd.py 50 | :language: python 51 | 52 | Reference 53 | --------- 54 | 55 | .. automodule:: freenect2 56 | :members: 57 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # freenect2 documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Mar 21 11:49:44 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | # import os 21 | # import sys 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ['sphinx.ext.autodoc', 35 | 'sphinx.ext.mathjax', 36 | 'sphinx.ext.githubpages', 37 | 'sphinxcontrib.napoleon'] 38 | 39 | # Add any paths that contain templates here, relative to this directory. 40 | templates_path = ['_templates'] 41 | 42 | # The suffix(es) of source filenames. 43 | # You can specify multiple suffix as a list of string: 44 | # 45 | # source_suffix = ['.rst', '.md'] 46 | source_suffix = '.rst' 47 | 48 | # The master toctree document. 49 | master_doc = 'index' 50 | 51 | # General information about the project. 52 | project = 'freenect2' 53 | copyright = '2017, Rich Wareham' 54 | author = 'Rich Wareham' 55 | 56 | # The version info for the project you're documenting, acts as replacement for 57 | # |version| and |release|, also used in various other places throughout the 58 | # built documents. 59 | # 60 | # The short X.Y version. 61 | version = '0.2.3' 62 | # The full version, including alpha/beta/rc tags. 63 | release = '0.2.3' 64 | 65 | # The language for content autogenerated by Sphinx. Refer to documentation 66 | # for a list of supported languages. 67 | # 68 | # This is also used if you do content translation via gettext catalogs. 69 | # Usually you set "language" from the command line for these cases. 70 | language = None 71 | 72 | # List of patterns, relative to source directory, that match files and 73 | # directories to ignore when looking for source files. 74 | # This patterns also effect to html_static_path and html_extra_path 75 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 76 | 77 | # The name of the Pygments (syntax highlighting) style to use. 78 | pygments_style = 'sphinx' 79 | 80 | # If true, `todo` and `todoList` produce output, else they produce nothing. 81 | todo_include_todos = False 82 | 83 | 84 | # -- Options for HTML output ---------------------------------------------- 85 | 86 | # The theme to use for HTML and HTML Help pages. See the documentation for 87 | # a list of builtin themes. 88 | # 89 | html_theme = 'alabaster' 90 | 91 | # Theme options are theme-specific and customize the look and feel of a theme 92 | # further. For a list of options available for each theme, see the 93 | # documentation. 94 | # 95 | # html_theme_options = {} 96 | 97 | # Add any paths that contain custom static files (such as style sheets) here, 98 | # relative to this directory. They are copied after the builtin static files, 99 | # so a file named "default.css" will overwrite the builtin "default.css". 100 | html_static_path = ['_static'] 101 | 102 | 103 | # -- Options for HTMLHelp output ------------------------------------------ 104 | 105 | # Output file base name for HTML help builder. 106 | htmlhelp_basename = 'freenect2doc' 107 | 108 | 109 | # -- Options for LaTeX output --------------------------------------------- 110 | 111 | latex_elements = { 112 | # The paper size ('letterpaper' or 'a4paper'). 113 | # 114 | # 'papersize': 'letterpaper', 115 | 116 | # The font size ('10pt', '11pt' or '12pt'). 117 | # 118 | # 'pointsize': '10pt', 119 | 120 | # Additional stuff for the LaTeX preamble. 121 | # 122 | # 'preamble': '', 123 | 124 | # Latex figure (float) alignment 125 | # 126 | # 'figure_align': 'htbp', 127 | } 128 | 129 | # Grouping the document tree into LaTeX files. List of tuples 130 | # (source start file, target name, title, 131 | # author, documentclass [howto, manual, or own class]). 132 | latex_documents = [ 133 | (master_doc, 'freenect2.tex', 'freenect2 Documentation', 134 | 'Rich Wareham', 'manual'), 135 | ] 136 | 137 | 138 | # -- Options for manual page output --------------------------------------- 139 | 140 | # One entry per manual page. List of tuples 141 | # (source start file, name, description, authors, manual section). 142 | man_pages = [ 143 | (master_doc, 'freenect2', 'freenect2 Documentation', 144 | [author], 1) 145 | ] 146 | 147 | 148 | # -- Options for Texinfo output ------------------------------------------- 149 | 150 | # Grouping the document tree into Texinfo files. List of tuples 151 | # (source start file, target name, title, author, 152 | # dir menu entry, description, category) 153 | texinfo_documents = [ 154 | (master_doc, 'freenect2', 'freenect2 Documentation', 155 | author, 'freenect2', 'One line description of project.', 156 | 'Miscellaneous'), 157 | ] 158 | 159 | 160 | 161 | -------------------------------------------------------------------------------- /binding/freenect2_build.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import os 3 | import subprocess 4 | from cffi import FFI 5 | 6 | this_dir = os.path.abspath(os.path.dirname(__file__)) 7 | 8 | with open(os.path.join(this_dir, 'freenect2-c.cpp')) as fobj: 9 | binding_source = fobj.read() 10 | 11 | extra_compile_args = codecs.decode(subprocess.check_output( 12 | 'pkg-config freenect2 --cflags'.split()), 'ascii').split() 13 | extra_link_args = codecs.decode(subprocess.check_output( 14 | 'pkg-config freenect2 --libs'.split()), 'ascii').split() 15 | 16 | ffibuilder = FFI() 17 | 18 | ffibuilder.set_source( 19 | "freenect2._freenect2", binding_source, 20 | libraries=['freenect2'], source_extension='.cpp', 21 | extra_compile_args=extra_compile_args, 22 | extra_link_args=extra_link_args) 23 | 24 | ffibuilder.cdef(r''' 25 | typedef enum { 26 | FRAME_TYPE_COLOR, FRAME_TYPE_IR, FRAME_TYPE_DEPTH, ... 27 | } Freenect2FrameType; 28 | 29 | typedef enum { 30 | FRAME_FORMAT_INVALID, FRAME_FORMAT_RAW, FRAME_FORMAT_FLOAT, 31 | FRAME_FORMAT_BGRX, FRAME_FORMAT_RGBX, FRAME_FORMAT_GRAY, ... 32 | } Freenect2FrameFormat; 33 | 34 | typedef void *Freenect2Ref; 35 | typedef void *Freenect2DeviceRef; 36 | typedef void *Freenect2FrameRef; 37 | typedef void *Freenect2FrameListenerRef; 38 | typedef void *Freenect2RegistrationRef; 39 | 40 | typedef struct { 41 | float fx, fy, cx, cy, k1, k2, k3, p1, p2; 42 | ...; 43 | } IrCameraParams; 44 | 45 | typedef struct { 46 | float fx, fy, cx, cy; 47 | float shift_d, shift_m; 48 | float mx_x3y0; // xxx 49 | float mx_x0y3; // yyy 50 | float mx_x2y1; // xxy 51 | float mx_x1y2; // yyx 52 | float mx_x2y0; // xx 53 | float mx_x0y2; // yy 54 | float mx_x1y1; // xy 55 | float mx_x1y0; // x 56 | float mx_x0y1; // y 57 | float mx_x0y0; // 1 58 | 59 | float my_x3y0; // xxx 60 | float my_x0y3; // yyy 61 | float my_x2y1; // xxy 62 | float my_x1y2; // yyx 63 | float my_x2y0; // xx 64 | float my_x0y2; // yy 65 | float my_x1y1; // xy 66 | float my_x1y0; // x 67 | float my_x0y1; // y 68 | float my_x0y0; // 1 69 | ...; 70 | } ColorCameraParams; 71 | 72 | Freenect2Ref freenect2_create(void); 73 | void freenect2_dispose(Freenect2Ref fn2_ref); 74 | int freenect2_enumerate_devices(Freenect2Ref fn2_ref); 75 | 76 | Freenect2DeviceRef freenect2_open_default_device(Freenect2Ref fn2_ref); 77 | Freenect2DeviceRef freenect2_open_device_by_index( 78 | Freenect2Ref fn2_ref, int index); 79 | Freenect2DeviceRef freenect2_open_device_by_serial( 80 | Freenect2Ref fn2_ref, const char* serial); 81 | 82 | int freenect2_device_start(Freenect2DeviceRef device_ref); 83 | int freenect2_device_stop(Freenect2DeviceRef device_ref); 84 | int freenect2_device_close(Freenect2DeviceRef device_ref); 85 | void freenect2_device_set_color_frame_listener( 86 | Freenect2DeviceRef device_ref, Freenect2FrameListenerRef fl_ref); 87 | void freenect2_device_set_ir_and_depth_frame_listener( 88 | Freenect2DeviceRef device_ref, Freenect2FrameListenerRef fl_ref); 89 | IrCameraParams freenect2_device_get_ir_camera_params( 90 | Freenect2DeviceRef device_ref); 91 | ColorCameraParams freenect2_device_get_color_camera_params( 92 | Freenect2DeviceRef device_ref); 93 | 94 | typedef int (*Freenect2FrameListenerFunc) ( 95 | Freenect2FrameType type, Freenect2FrameRef frame, void *user_data); 96 | Freenect2FrameListenerRef freenect2_frame_listener_create( 97 | Freenect2FrameListenerFunc func, void* user_data); 98 | void freenect2_frame_listener_dispose(Freenect2FrameListenerRef fl_ref); 99 | 100 | extern "Python" int frame_listener_callback( 101 | Freenect2FrameType type, Freenect2FrameRef frame, void *user_data); 102 | 103 | Freenect2FrameRef freenect2_frame_create( 104 | size_t width, size_t height, size_t bytes_per_pixel); 105 | void freenect2_frame_dispose(Freenect2FrameRef frame_ref); 106 | 107 | size_t freenect2_frame_get_width(Freenect2FrameRef frame_ref); 108 | size_t freenect2_frame_get_height(Freenect2FrameRef frame_ref); 109 | size_t freenect2_frame_get_bytes_per_pixel(Freenect2FrameRef frame_ref); 110 | void* freenect2_frame_get_data(Freenect2FrameRef frame_ref); 111 | uint32_t freenect2_frame_get_timestamp(Freenect2FrameRef frame_ref); 112 | uint32_t freenect2_frame_get_sequence(Freenect2FrameRef frame_ref); 113 | float freenect2_frame_get_exposure(Freenect2FrameRef frame_ref); 114 | float freenect2_frame_get_gain(Freenect2FrameRef frame_ref); 115 | float freenect2_frame_get_gamma(Freenect2FrameRef frame_ref); 116 | uint32_t freenect2_frame_get_status(Freenect2FrameRef frame_ref); 117 | Freenect2FrameFormat freenect2_frame_get_format(Freenect2FrameRef frame_ref); 118 | 119 | void freenect2_frame_set_width(Freenect2FrameRef frame_ref, size_t value); 120 | void freenect2_frame_set_height(Freenect2FrameRef frame_ref, size_t value); 121 | void freenect2_frame_set_bytes_per_pixel(Freenect2FrameRef frame_ref, size_t value); 122 | void freenect2_frame_set_timestamp(Freenect2FrameRef frame_ref, uint32_t value); 123 | void freenect2_frame_set_sequence(Freenect2FrameRef frame_ref, uint32_t value); 124 | void freenect2_frame_set_exposure(Freenect2FrameRef frame_ref, float value); 125 | void freenect2_frame_set_gain(Freenect2FrameRef frame_ref, float value); 126 | void freenect2_frame_set_gamma(Freenect2FrameRef frame_ref, float value); 127 | void freenect2_frame_set_status(Freenect2FrameRef frame_ref, uint32_t value); 128 | void freenect2_frame_set_format(Freenect2FrameRef frame_ref, Freenect2FrameFormat value); 129 | 130 | Freenect2RegistrationRef freenect2_registration_create( 131 | IrCameraParams depth_p, ColorCameraParams rgb_p); 132 | void freenect2_registration_dispose(Freenect2RegistrationRef reg_ref); 133 | void freenect2_registration_apply( 134 | Freenect2RegistrationRef reg_ref, Freenect2FrameRef rgb_ref, 135 | Freenect2FrameRef depth_ref, Freenect2FrameRef undistorted_ref, 136 | Freenect2FrameRef registered_ref, int enable_filter, 137 | Freenect2FrameRef big_depth_ref); 138 | void freenect2_registration_get_points_xyz( 139 | Freenect2RegistrationRef reg_ref, Freenect2FrameRef undistorted_ref, 140 | const int32_t* rows, const int32_t* cols, size_t n_points, 141 | float* out_xs, float* out_ys, float* out_zs); 142 | ''') 143 | 144 | if __name__ == "__main__": 145 | ffibuilder.compile(verbose=True) 146 | -------------------------------------------------------------------------------- /binding/freenect2-c.cpp: -------------------------------------------------------------------------------- 1 | // C binding for libfreenect2 2 | 3 | #include 4 | #include 5 | 6 | typedef void *Freenect2Ref; 7 | typedef void *Freenect2DeviceRef; 8 | typedef void *Freenect2FrameListenerRef; 9 | typedef void *Freenect2FrameRef; 10 | typedef void *Freenect2RegistrationRef; 11 | 12 | class Freenect2FrameListener; 13 | 14 | using namespace libfreenect2; 15 | 16 | typedef enum { 17 | FRAME_TYPE_COLOR = Frame::Color, 18 | FRAME_TYPE_IR = Frame::Ir, 19 | FRAME_TYPE_DEPTH = Frame::Depth, 20 | } Freenect2FrameType; 21 | 22 | typedef enum { 23 | FRAME_FORMAT_INVALID = Frame::Invalid, 24 | FRAME_FORMAT_RAW = Frame::Raw, 25 | FRAME_FORMAT_FLOAT = Frame::Float, 26 | FRAME_FORMAT_BGRX = Frame::BGRX, 27 | FRAME_FORMAT_RGBX = Frame::RGBX, 28 | FRAME_FORMAT_GRAY = Frame::Gray, 29 | } Freenect2FrameFormat; 30 | 31 | typedef Freenect2Device::IrCameraParams IrCameraParams; 32 | typedef Freenect2Device::ColorCameraParams ColorCameraParams; 33 | 34 | static Freenect2Ref freenect2_create(void) 35 | { 36 | return reinterpret_cast(new Freenect2()); 37 | } 38 | 39 | static void freenect2_dispose(Freenect2Ref fn2) 40 | { 41 | delete reinterpret_cast(fn2); 42 | } 43 | 44 | static int freenect2_enumerate_devices(Freenect2Ref fn2_ref) 45 | { 46 | Freenect2* fn2 = reinterpret_cast(fn2_ref); 47 | return fn2->enumerateDevices(); 48 | } 49 | 50 | static Freenect2DeviceRef freenect2_open_default_device(Freenect2Ref fn2_ref) 51 | { 52 | Freenect2* fn2 = reinterpret_cast(fn2_ref); 53 | return reinterpret_cast(fn2->openDefaultDevice()); 54 | } 55 | 56 | static Freenect2DeviceRef freenect2_open_device_by_index(Freenect2Ref fn2_ref, int index) 57 | { 58 | Freenect2* fn2 = reinterpret_cast(fn2_ref); 59 | return reinterpret_cast(fn2->openDevice(index)); 60 | } 61 | 62 | static Freenect2DeviceRef freenect2_open_device_by_serial( 63 | Freenect2Ref fn2_ref, const char* serial) 64 | { 65 | Freenect2* fn2 = reinterpret_cast(fn2_ref); 66 | return reinterpret_cast( 67 | fn2->openDevice(std::string(serial))); 68 | } 69 | 70 | static int freenect2_device_start(Freenect2DeviceRef device_ref) { 71 | Freenect2Device* device = reinterpret_cast(device_ref); 72 | return device->start(); 73 | } 74 | 75 | static int freenect2_device_stop(Freenect2DeviceRef device_ref) { 76 | Freenect2Device* device = reinterpret_cast(device_ref); 77 | return device->stop(); 78 | } 79 | 80 | static int freenect2_device_close(Freenect2DeviceRef device_ref) { 81 | Freenect2Device* device = reinterpret_cast(device_ref); 82 | return device->close(); 83 | } 84 | 85 | static void freenect2_device_set_color_frame_listener( 86 | Freenect2DeviceRef device_ref, Freenect2FrameListenerRef fl_ref) 87 | { 88 | Freenect2Device* device = reinterpret_cast(device_ref); 89 | FrameListener* fl = reinterpret_cast(fl_ref); 90 | device->setColorFrameListener(fl); 91 | } 92 | 93 | static void freenect2_device_set_ir_and_depth_frame_listener( 94 | Freenect2DeviceRef device_ref, Freenect2FrameListenerRef fl_ref) 95 | { 96 | Freenect2Device* device = reinterpret_cast(device_ref); 97 | FrameListener* fl = reinterpret_cast(fl_ref); 98 | device->setIrAndDepthFrameListener(fl); 99 | } 100 | 101 | static IrCameraParams freenect2_device_get_ir_camera_params( 102 | Freenect2DeviceRef device_ref) 103 | { 104 | Freenect2Device* device = reinterpret_cast(device_ref); 105 | return device->getIrCameraParams(); 106 | } 107 | 108 | static ColorCameraParams freenect2_device_get_color_camera_params( 109 | Freenect2DeviceRef device_ref) 110 | { 111 | Freenect2Device* device = reinterpret_cast(device_ref); 112 | return device->getColorCameraParams(); 113 | } 114 | 115 | typedef int (*Freenect2FrameListenerFunc) ( 116 | Freenect2FrameType type, Freenect2FrameRef frame, void *user_data); 117 | 118 | class Freenect2FrameListener : public FrameListener 119 | { 120 | public: 121 | Freenect2FrameListener( 122 | Freenect2FrameListenerFunc func, 123 | void* user_data=NULL) 124 | : func_(func), user_data_(user_data) { } 125 | 126 | virtual ~Freenect2FrameListener() { } 127 | 128 | virtual bool onNewFrame(Frame::Type type, Frame *frame) 129 | { 130 | return func_( 131 | static_cast(type), 132 | reinterpret_cast(frame), 133 | user_data_); 134 | } 135 | 136 | protected: 137 | Freenect2FrameListenerFunc func_; 138 | void *user_data_; 139 | }; 140 | 141 | static Freenect2FrameListenerRef freenect2_frame_listener_create( 142 | Freenect2FrameListenerFunc func, void* user_data) 143 | { 144 | return reinterpret_cast( 145 | new Freenect2FrameListener(func, user_data)); 146 | } 147 | 148 | static void freenect2_frame_listener_dispose(Freenect2FrameListenerRef fl_ref) 149 | { 150 | Freenect2FrameListener* fl = reinterpret_cast(fl_ref); 151 | delete fl; 152 | } 153 | 154 | static Freenect2FrameRef freenect2_frame_create( 155 | size_t width, size_t height, size_t bytes_per_pixel) 156 | { 157 | return reinterpret_cast( 158 | new Frame(width, height, bytes_per_pixel)); 159 | } 160 | 161 | static void freenect2_frame_dispose(Freenect2FrameRef frame_ref) 162 | { 163 | Frame* frame = reinterpret_cast(frame_ref); 164 | delete frame; 165 | } 166 | 167 | static size_t freenect2_frame_get_width(Freenect2FrameRef frame_ref) 168 | { 169 | Frame* frame = reinterpret_cast(frame_ref); 170 | return frame->width; 171 | } 172 | 173 | static size_t freenect2_frame_get_height(Freenect2FrameRef frame_ref) 174 | { 175 | Frame* frame = reinterpret_cast(frame_ref); 176 | return frame->height; 177 | } 178 | 179 | static size_t freenect2_frame_get_bytes_per_pixel(Freenect2FrameRef frame_ref) 180 | { 181 | Frame* frame = reinterpret_cast(frame_ref); 182 | return frame->bytes_per_pixel; 183 | } 184 | 185 | static void* freenect2_frame_get_data(Freenect2FrameRef frame_ref) 186 | { 187 | Frame* frame = reinterpret_cast(frame_ref); 188 | return frame->data; 189 | } 190 | 191 | static uint32_t freenect2_frame_get_timestamp(Freenect2FrameRef frame_ref) 192 | { 193 | Frame* frame = reinterpret_cast(frame_ref); 194 | return frame->timestamp; 195 | } 196 | 197 | static uint32_t freenect2_frame_get_sequence(Freenect2FrameRef frame_ref) 198 | { 199 | Frame* frame = reinterpret_cast(frame_ref); 200 | return frame->sequence; 201 | } 202 | 203 | static float freenect2_frame_get_exposure(Freenect2FrameRef frame_ref) 204 | { 205 | Frame* frame = reinterpret_cast(frame_ref); 206 | return frame->exposure; 207 | } 208 | 209 | static float freenect2_frame_get_gain(Freenect2FrameRef frame_ref) 210 | { 211 | Frame* frame = reinterpret_cast(frame_ref); 212 | return frame->gain; 213 | } 214 | 215 | static float freenect2_frame_get_gamma(Freenect2FrameRef frame_ref) 216 | { 217 | Frame* frame = reinterpret_cast(frame_ref); 218 | return frame->gamma; 219 | } 220 | 221 | static uint32_t freenect2_frame_get_status(Freenect2FrameRef frame_ref) 222 | { 223 | Frame* frame = reinterpret_cast(frame_ref); 224 | return frame->status; 225 | } 226 | 227 | static void freenect2_frame_set_width(Freenect2FrameRef frame_ref, size_t value) 228 | { 229 | Frame* frame = reinterpret_cast(frame_ref); 230 | frame->width = value; 231 | } 232 | 233 | static void freenect2_frame_set_height(Freenect2FrameRef frame_ref, size_t value) 234 | { 235 | Frame* frame = reinterpret_cast(frame_ref); 236 | frame->height = value; 237 | } 238 | 239 | static void freenect2_frame_set_bytes_per_pixel(Freenect2FrameRef frame_ref, size_t value) 240 | { 241 | Frame* frame = reinterpret_cast(frame_ref); 242 | frame->bytes_per_pixel = value; 243 | } 244 | 245 | static void freenect2_frame_set_timestamp(Freenect2FrameRef frame_ref, uint32_t value) 246 | { 247 | Frame* frame = reinterpret_cast(frame_ref); 248 | frame->timestamp = value; 249 | } 250 | 251 | static void freenect2_frame_set_sequence(Freenect2FrameRef frame_ref, uint32_t value) 252 | { 253 | Frame* frame = reinterpret_cast(frame_ref); 254 | frame->sequence = value; 255 | } 256 | 257 | static void freenect2_frame_set_exposure(Freenect2FrameRef frame_ref, float value) 258 | { 259 | Frame* frame = reinterpret_cast(frame_ref); 260 | frame->exposure = value; 261 | } 262 | 263 | static void freenect2_frame_set_gain(Freenect2FrameRef frame_ref, float value) 264 | { 265 | Frame* frame = reinterpret_cast(frame_ref); 266 | frame->gain = value; 267 | } 268 | 269 | static void freenect2_frame_set_gamma(Freenect2FrameRef frame_ref, float value) 270 | { 271 | Frame* frame = reinterpret_cast(frame_ref); 272 | frame->gamma = value; 273 | } 274 | 275 | static void freenect2_frame_set_status(Freenect2FrameRef frame_ref, uint32_t value) 276 | { 277 | Frame* frame = reinterpret_cast(frame_ref); 278 | frame->status = value; 279 | } 280 | 281 | static void freenect2_frame_set_format(Freenect2FrameRef frame_ref, Freenect2FrameFormat value) 282 | { 283 | Frame* frame = reinterpret_cast(frame_ref); 284 | frame->format = static_cast(value); 285 | } 286 | 287 | 288 | static Freenect2FrameFormat freenect2_frame_get_format(Freenect2FrameRef frame_ref) 289 | { 290 | Frame* frame = reinterpret_cast(frame_ref); 291 | return static_cast(frame->format); 292 | } 293 | 294 | static Freenect2RegistrationRef freenect2_registration_create( 295 | IrCameraParams depth_p, ColorCameraParams rgb_p) 296 | { 297 | return reinterpret_cast( 298 | new Registration(depth_p, rgb_p)); 299 | } 300 | 301 | static void freenect2_registration_dispose(Freenect2RegistrationRef reg_ref) 302 | { 303 | Registration* reg = reinterpret_cast(reg_ref); 304 | delete reg; 305 | } 306 | 307 | static void freenect2_registration_apply( 308 | Freenect2RegistrationRef reg_ref, Freenect2FrameRef rgb_ref, 309 | Freenect2FrameRef depth_ref, Freenect2FrameRef undistorted_ref, 310 | Freenect2FrameRef registered_ref, int enable_filter, 311 | Freenect2FrameRef big_depth_ref) 312 | { 313 | Registration* reg = reinterpret_cast(reg_ref); 314 | Frame* rgb = reinterpret_cast(rgb_ref); 315 | Frame* depth = reinterpret_cast(depth_ref); 316 | Frame* undistorted = reinterpret_cast(undistorted_ref); 317 | Frame* registered = reinterpret_cast(registered_ref); 318 | Frame* big_depth = reinterpret_cast(big_depth_ref); 319 | reg->apply(rgb, depth, undistorted, registered, 320 | (enable_filter != 0) ? true : false, big_depth); 321 | } 322 | 323 | static void freenect2_registration_get_points_xyz( 324 | Freenect2RegistrationRef reg_ref, Freenect2FrameRef undistorted_ref, 325 | const int32_t* rows, const int32_t* cols, size_t n_points, 326 | float* out_xs, float* out_ys, float* out_zs) 327 | { 328 | Registration* reg = reinterpret_cast(reg_ref); 329 | Frame* undistorted = reinterpret_cast(undistorted_ref); 330 | for(size_t p_idx=0; p_idxgetPointXYZ( 333 | undistorted, rows[p_idx], cols[p_idx], 334 | out_xs[p_idx], out_ys[p_idx], out_zs[p_idx]); 335 | } 336 | } 337 | -------------------------------------------------------------------------------- /freenect2/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from contextlib import contextmanager 4 | import enum 5 | from queue import Queue, Empty 6 | 7 | import numpy as np 8 | from PIL import Image 9 | 10 | from ._freenect2 import lib, ffi 11 | 12 | __all__ = ( 13 | 'NoDeviceError', 14 | 'NoFrameReceivedError', 15 | 'Device', 16 | 'FrameType', 17 | 'FrameFormat', 18 | 'Frame', 19 | 'Registration', 20 | 'IrCameraParams', 21 | 'ColorCameraParams' 22 | ) 23 | 24 | _FREENECT2_SINGLETON = None 25 | 26 | def _get_freenect2(): 27 | global _FREENECT2_SINGLETON 28 | if _FREENECT2_SINGLETON is None: 29 | _FREENECT2_SINGLETON = ffi.gc( 30 | lib.freenect2_create(), lib.freenect2_dispose) 31 | lib.freenect2_enumerate_devices(_FREENECT2_SINGLETON) 32 | return _FREENECT2_SINGLETON 33 | 34 | class NoDeviceError(RuntimeError): 35 | """Raised by :py:class:`.Device` when there is no default device to open.""" 36 | pass 37 | 38 | class FrameType(enum.Enum): 39 | """Available types of frames.""" 40 | 41 | #: 1920x1080. BGRX or RGBX 42 | Color = lib.FRAME_TYPE_COLOR 43 | 44 | #: 512x424 float. Range is [0.0, 65535.0] 45 | Ir = lib.FRAME_TYPE_IR 46 | 47 | #: 512x424 float, unit: millimeter. Non-positive, NaN, and infinity are 48 | #: invalid or missing data 49 | Depth = lib.FRAME_TYPE_DEPTH 50 | 51 | class FrameFormat(enum.Enum): 52 | """Pixel format.""" 53 | 54 | #: Invalid format. 55 | Invalid = lib.FRAME_FORMAT_INVALID 56 | 57 | #: Raw bitstream. 'bytes_per_pixel' defines the number of bytes 58 | Raw = lib.FRAME_FORMAT_RAW 59 | 60 | #: A 4-byte float per pixel 61 | Float = lib.FRAME_FORMAT_FLOAT 62 | 63 | #: 4 bytes of B, G, R, and unused per pixel 64 | BGRX = lib.FRAME_FORMAT_BGRX 65 | 66 | #: 4 bytes of R, G, B, and unused per pixel 67 | RGBX = lib.FRAME_FORMAT_RGBX 68 | 69 | #: 1 byte of gray per pixel 70 | Gray = lib.FRAME_FORMAT_GRAY 71 | 72 | @ffi.def_extern() 73 | def frame_listener_callback(type_, frame_ref, user_data): 74 | callable_ = ffi.from_handle(user_data) 75 | assert callable(callable_) 76 | frame = Frame(ffi.gc(frame_ref, lib.freenect2_frame_dispose)) 77 | callable_(FrameType(type_), frame) 78 | return 1 79 | 80 | def _callable_to_frame_listener(callable_): 81 | assert callable(callable_) 82 | handle = ffi.new_handle(callable_) 83 | return handle, ffi.gc( 84 | lib.freenect2_frame_listener_create(lib.frame_listener_callback, handle), 85 | lib.freenect2_frame_listener_dispose 86 | ) 87 | 88 | class NoFrameReceivedError(RuntimeError): 89 | """With the default frame listener this is raised when no frame has been 90 | received from the device within a set time.""" 91 | pass 92 | 93 | class QueueFrameListener(object): 94 | def __init__(self, maxsize=16): 95 | self.queue = Queue(maxsize=maxsize) 96 | 97 | def __call__(self, frame_type, frame): 98 | self.queue.put_nowait((frame_type, frame)) 99 | 100 | def get(self, timeout=False): 101 | return self.queue.get(True, timeout) 102 | 103 | class ColorCameraParams(object): 104 | """ 105 | Color camera intrinsic calibration. 106 | 107 | .. py:attribute:: fx 108 | 109 | Focal length for x-axis (pixels) 110 | 111 | .. py:attribute:: fy 112 | 113 | Focal length for y-axis (pixels) 114 | 115 | .. py:attribute:: cx 116 | 117 | Principal point for x-axis (pixels) 118 | 119 | .. py:attribute:: cy 120 | 121 | Principal point for y-axis (pixels) 122 | """ 123 | 124 | class IrCameraParams(object): 125 | """ 126 | IR/depth camera intrinsic calibration. 127 | 128 | .. py:attribute:: fx 129 | 130 | Focal length for x-axis (pixels) 131 | 132 | .. py:attribute:: fy 133 | 134 | Focal length for y-axis (pixels) 135 | 136 | .. py:attribute:: cx 137 | 138 | Principal point for x-axis (pixels) 139 | 140 | .. py:attribute:: cy 141 | 142 | Principal point for y-axis (pixels) 143 | 144 | .. py:attribute:: k1 145 | 146 | Radial distortion co-efficient, 1st-order 147 | 148 | .. py:attribute:: k2 149 | 150 | Radial distortion co-efficient, 2nd-order 151 | 152 | .. py:attribute:: k3 153 | 154 | Radial distortion co-efficient, 3rd-order 155 | 156 | .. py:attribute:: p1 157 | 158 | Tangential distortion co-efficient 159 | 160 | .. py:attribute:: p2 161 | 162 | Tangential distortion co-efficient 163 | """ 164 | 165 | class Device(object): 166 | """Control a single device. 167 | 168 | If called with no arguments, the default device is opened. 169 | 170 | Raises: 171 | :py:class:`.NoDeviceError` if there is no default device to open. 172 | 173 | .. py:attribute:: color_camera_params 174 | 175 | (:py:class:`.ColorCameraParams` or None) A structure describing the RGB 176 | camera factory calibration. Before the :py:func:`.start` is called, this 177 | is *None* since the device only reports calibration when capture begins. 178 | 179 | .. py:attribute:: ir_camera_params 180 | 181 | (:py:class:`.IrCameraParams` or None) A structure describing the IR 182 | camera factory calibration. Before the :py:func:`.start` is called, this 183 | is *None* since the device only reports calibration when capture begins. 184 | 185 | """ 186 | 187 | def __init__(self, c_object=None): 188 | if c_object is None: 189 | c_object = lib.freenect2_open_default_device(_get_freenect2()) 190 | self._c_object = c_object 191 | if self._c_object == ffi.NULL: 192 | raise NoDeviceError() 193 | 194 | self._registration = None 195 | 196 | self._default_listener = QueueFrameListener() 197 | self.color_frame_listener = self._default_listener 198 | self.ir_and_depth_frame_listener = self._default_listener 199 | 200 | self.color_camera_params = None 201 | self.ir_camera_params = None 202 | 203 | def start(self, frame_listener=None): 204 | """Start depth, IR and RGB streams. 205 | 206 | Args: 207 | frame_listener (callable or None): if not-None, this is a callable 208 | which is assigned to both :py:attr:`.color_frame_listener` and 209 | :py:attr:`.ir_and_depth_frame_listener` before the device is 210 | started. 211 | 212 | """ 213 | if frame_listener is not None: 214 | self.color_frame_listener = frame_listener 215 | self.ir_and_depth_frame_listener = frame_listener 216 | lib.freenect2_device_start(self._c_object) 217 | 218 | self.color_camera_params = lib.freenect2_device_get_color_camera_params(self._c_object) 219 | self.ir_camera_params = lib.freenect2_device_get_ir_camera_params(self._c_object) 220 | 221 | def stop(self): 222 | """Stop any running streams.""" 223 | lib.freenect2_device_stop(self._c_object) 224 | 225 | def close(self): 226 | """Close the device and free any associated resources.""" 227 | lib.freenect2_device_close(self._c_object) 228 | 229 | def get_next_frame(self, timeout=None): 230 | """Get the next frame type and frame from the device. 231 | 232 | Args: 233 | timeout (number or None): If not-None, a positive number of seconds 234 | to wait for a frame before raising a 235 | :py:class:`.NoFrameReceivedError` exception. 236 | 237 | Returns: 238 | A :py:class:`.FrameType`, :py:class:`.Frame` tuple representing the 239 | type of received frame and the frame itself. 240 | 241 | .. note:: 242 | 243 | This method only works if the default listener is being used. That 244 | is to say that :py:attr:`.color_frame_listener` and 245 | :py:attr:`.ir_and_depth_frame_listener` have not been changed from 246 | their default values. 247 | 248 | """ 249 | try: 250 | frame_type, frame = self._default_listener.get(timeout) 251 | except Empty: 252 | raise NoFrameReceivedError() 253 | 254 | return frame_type, frame 255 | 256 | @property 257 | def registration(self): 258 | """An instance of :py:class:`.Registration` which can be used to 259 | undistort the raw depth image and register the RGB image with it. 260 | 261 | """ 262 | if self._registration is None: 263 | self._registration = Registration( 264 | self.ir_camera_params, self.color_camera_params) 265 | return self._registration 266 | 267 | @property 268 | def color_frame_listener(self): 269 | """A callable called whenever a new color frame arrives from the 270 | device. The callable should take two positional arguments, the frame 271 | type (an instance of :py:class:`.FrameType`) and the frame itself (an 272 | instance of :py:class:`.Frame`).""" 273 | return self._color_frame_listener[0] 274 | 275 | @color_frame_listener.setter 276 | def color_frame_listener(self, value): 277 | if value is None: 278 | self._color_frame_listener = (None, None, None) 279 | return 280 | handle, fl = _callable_to_frame_listener(value) 281 | lib.freenect2_device_set_color_frame_listener(self._c_object, fl) 282 | self._color_frame_listener = value, handle, fl 283 | 284 | @property 285 | def ir_and_depth_frame_listener(self): 286 | """A callable called whenever a new IR or depth frame arrives from the 287 | device. The callable should take two positional arguments, the frame 288 | type (an instance of :py:class:`.FrameType`) and the frame itself (an 289 | instance of :py:class:`.Frame`).""" 290 | return self._ir_and_depth_frame_listener[0] 291 | 292 | @ir_and_depth_frame_listener.setter 293 | def ir_and_depth_frame_listener(self, value): 294 | if value is None: 295 | self._ir_and_depth_frame_listener = (None, None, None) 296 | return 297 | handle, fl = _callable_to_frame_listener(value) 298 | lib.freenect2_device_set_ir_and_depth_frame_listener(self._c_object, fl) 299 | self._ir_and_depth_frame_listener = value, handle, fl 300 | 301 | @contextmanager 302 | def running(self, *args, **kwargs): 303 | """A context manager which can be used to ensure that the device's 304 | streams are stopped. Any arguments are passed to :py:meth:`.start`. 305 | 306 | .. code:: 307 | 308 | from freenect2 import Device 309 | 310 | # Open default device 311 | device = Device() 312 | 313 | # Start depth and color frames 314 | with device.running(): 315 | # ... frame listener callbacks will be called ... 316 | 317 | # Device is now stopped 318 | 319 | """ 320 | self.start(*args, **kwargs) 321 | yield self 322 | self.stop() 323 | 324 | def __iter__(self): 325 | def iterator(): 326 | while True: 327 | yield self.get_next_frame() 328 | return iterator() 329 | 330 | class Frame(object): 331 | """A single frame received from the device. 332 | 333 | These should not be constructed directly since they are usually created by 334 | the freenect2 library itself. However you may need to construct "blank" 335 | frames for use with :py:class:`.Registration`. In which case, you should use 336 | the :py:meth:`.Frame.create` class method. 337 | 338 | """ 339 | def __init__(self, frame_ref): 340 | self._c_object = frame_ref 341 | 342 | @classmethod 343 | def create(self, width, height, bytes_per_pixel): 344 | """Create a blank frame with the specified width, height and bytes per 345 | pixel. Memory for the frame is automatically allocated. No other 346 | attributes are initialised. 347 | 348 | """ 349 | return Frame(lib.freenect2_frame_create(width, height, bytes_per_pixel)) 350 | 351 | def to_image(self): 352 | """Convert the Frame to a PIL :py:class:`Image` instance.""" 353 | if self.format is FrameFormat.BGRX: 354 | return Image.frombuffer( 355 | 'RGB', (self.width, self.height), self.data, 'raw', 'BGRX') 356 | elif self.format is FrameFormat.RGBX: 357 | return Image.frombuffer( 358 | 'RGB', (self.width, self.height), self.data, 'raw', 'RGBX') 359 | elif self.format is FrameFormat.Gray: 360 | return Image.frombuffer( 361 | 'L', (self.width, self.height), self.data, 'raw', 'L') 362 | elif self.format is FrameFormat.Float: 363 | return Image.frombuffer( 364 | 'F', (self.width, self.height), self.data, 'raw', 'F') 365 | else: 366 | raise NotImplementedError() 367 | 368 | def to_array(self): 369 | """Convert the image to a numpy :py:class:`array` instance. 370 | 371 | The memory is not copied so be careful performing any operations which 372 | modify the contents of the frame. 373 | 374 | """ 375 | if self.format is FrameFormat.BGRX or self.format is FrameFormat.RGBX: 376 | return np.frombuffer( 377 | self.data, dtype='uint8').reshape( 378 | (self.height, self.width, 4), order='C') 379 | elif self.format is FrameFormat.Gray: 380 | return np.frombuffer( 381 | self.data, dtype='uint8').reshape( 382 | (self.height, self.width), order='C') 383 | elif self.format is FrameFormat.Float: 384 | return np.frombuffer( 385 | self.data, dtype='float32').reshape( 386 | (self.height, self.width), order='C') 387 | else: 388 | raise NotImplementedError() 389 | 390 | @property 391 | def width(self): 392 | """Length of a line (in pixels)""" 393 | return lib.freenect2_frame_get_width(self._c_object) 394 | 395 | @width.setter 396 | def width(self, value): 397 | lib.freenect2_frame_set_width(self._c_object, value) 398 | 399 | @property 400 | def height(self): 401 | """Number of lines in the frame""" 402 | return lib.freenect2_frame_get_height(self._c_object) 403 | 404 | @height.setter 405 | def height(self, value): 406 | lib.freenect2_frame_set_height(self._c_object, value) 407 | 408 | @property 409 | def bytes_per_pixel(self): 410 | """Number of bytes in a pixel. If :py:attr:`.format` is 411 | :py:attr:`.FrameFormat.Raw`, this is the buffer size.""" 412 | return lib.freenect2_frame_get_bytes_per_pixel(self._c_object) 413 | 414 | @bytes_per_pixel.setter 415 | def bytes_per_pixel(self, value): 416 | lib.freenect2_frame_set_bytes_per_pixel(self._c_object, value) 417 | 418 | @property 419 | def data(self): 420 | """A buffer object pointing to the raw memory contents of the frame.""" 421 | data_ptr = lib.freenect2_frame_get_data(self._c_object) 422 | return ffi.buffer( 423 | data_ptr, self.width * self.height * self.bytes_per_pixel) 424 | 425 | @property 426 | def timestamp(self): 427 | """Unit: roughly or exactly 0.1 millisecond""" 428 | return lib.freenect2_frame_get_timestamp(self._c_object) 429 | 430 | @timestamp.setter 431 | def timestamp(self, value): 432 | lib.freenect2_frame_set_timestamp(self._c_object, value) 433 | 434 | @property 435 | def sequence(self): 436 | """Increasing frame sequence number""" 437 | return lib.freenect2_frame_get_sequence(self._c_object) 438 | 439 | @sequence.setter 440 | def sequence(self, value): 441 | lib.freenect2_frame_set_sequence(self._c_object, value) 442 | 443 | @property 444 | def exposure(self): 445 | """From 0.5 (very bright) to ~60.0 (fully covered)""" 446 | return lib.freenect2_frame_get_exposure(self._c_object) 447 | 448 | @exposure.setter 449 | def exposure(self, value): 450 | lib.freenect2_frame_set_exposure(self._c_object, value) 451 | 452 | @property 453 | def gain(self): 454 | """From 1.0 (bright) to 1.5 (covered)""" 455 | return lib.freenect2_frame_get_gain(self._c_object) 456 | 457 | @gain.setter 458 | def gain(self, value): 459 | lib.freenect2_frame_set_gain(self._c_object, value) 460 | 461 | @property 462 | def gamma(self): 463 | """From 1.0 (bright) to 6.4 (covered)""" 464 | return lib.freenect2_frame_get_gamma(self._c_object) 465 | 466 | @gamma.setter 467 | def gamma(self, value): 468 | lib.freenect2_frame_set_gamma(self._c_object, value) 469 | 470 | @property 471 | def status(self): 472 | """zero if ok; non-zero for errors""" 473 | return lib.freenect2_frame_get_status(self._c_object) 474 | 475 | @status.setter 476 | def status(self, value): 477 | lib.freenect2_frame_set_status(self._c_object, value) 478 | 479 | @property 480 | def format(self): 481 | """Byte format. Informative only, doesn't indicate errors. An instance 482 | of :py:class:`.FrameFormat`.""" 483 | return FrameFormat(lib.freenect2_frame_get_format(self._c_object)) 484 | 485 | @format.setter 486 | def format(self, value): 487 | lib.freenect2_frame_set_format(self._c_object, value.value) 488 | 489 | def __repr__(self): 490 | return ( 491 | 'Frame(width={0.width}, height={0.height}, sequence={0.sequence}, ' 492 | 'timestamp={0.timestamp}, format={0.format})').format(self) 493 | 494 | class Registration(object): 495 | """Information required to undistort raw depth frames and register RGB 496 | frames onto depth. 497 | 498 | Do not construct this directly. Instead use the 499 | :py:attr:`.Device.registration` attribute. 500 | 501 | """ 502 | def __init__(self, depth_p, rgb_p): 503 | self.depth_p = depth_p 504 | self.rgb_p = rgb_p 505 | self._c_object = ffi.gc( 506 | lib.freenect2_registration_create(depth_p, rgb_p), 507 | lib.freenect2_registration_dispose) 508 | 509 | def apply(self, rgb, depth, enable_filter=True, with_big_depth=False): 510 | """Take an RGB and Depth image and return tuple with the undistorted 511 | depth image and color image rectified onto depth. 512 | 513 | Args: 514 | rgb (:py:class:`.Frame`): RGB frame received from device 515 | depth (:py:class:`.Frame`): Depth frame received from device 516 | enable_filter (bool): If true, filter out pixels not visible in 517 | both cameras. 518 | with_big_depth (bool): If true, also return a 1920x1082 mapping of 519 | depth onto the color map. The top and bottom rows are blank. 520 | 521 | Returns: 522 | A :py:class:`Frame` pair representing the undistorted depth and 523 | registered RGB frames. 524 | 525 | """ 526 | undistorted = Frame.create(512, 424, 4) 527 | undistorted.format = depth.format 528 | registered = Frame.create(512, 424, 4) 529 | registered.format = rgb.format 530 | 531 | big_depth, big_depth_ref = None, ffi.NULL 532 | if with_big_depth: 533 | big_depth = Frame.create(1920, 1082, 4) 534 | big_depth.format = depth.format 535 | big_depth_ref = big_depth._c_object 536 | 537 | lib.freenect2_registration_apply( 538 | self._c_object, 539 | rgb._c_object, depth._c_object, undistorted._c_object, 540 | registered._c_object, 1 if enable_filter else 0, 541 | big_depth_ref 542 | ) 543 | 544 | rvs = [undistorted, registered] 545 | if with_big_depth: 546 | rvs.append(big_depth) 547 | 548 | return tuple(rvs) 549 | 550 | def get_points_xyz(self, undistorted, rows, cols): 551 | """Retrieve real-world co-ordinates corresponding to points in the 552 | undistorted depth image. Units are millimetres. 553 | 554 | Args: 555 | undistorted (:py:class:`Frame`): the undistorted depth frame 556 | rows (numpy array): integer row indices of points in the depth frame 557 | cols (numpy array): integer column indices of points in the depth 558 | frame. Must be the same shape as *rows*. 559 | 560 | Returns: 561 | A three element tuple containing numpy arrays for the x-, y- and 562 | z-co-ordinates of the points. Each array has the same shape as 563 | *rows*. 564 | """ 565 | rows = np.atleast_1d(rows).astype(np.int32) 566 | cols = np.atleast_1d(cols).astype(np.int32) 567 | assert rows.shape == cols.shape 568 | xs = np.ones(rows.shape, dtype=np.float32) 569 | ys = np.ones(rows.shape, dtype=np.float32) 570 | zs = np.ones(rows.shape, dtype=np.float32) 571 | lib.freenect2_registration_get_points_xyz( 572 | self._c_object, undistorted._c_object, 573 | ffi.cast('int32_t*', rows.ctypes.data), 574 | ffi.cast('int32_t*', cols.ctypes.data), 575 | int(np.product(rows.shape)), 576 | ffi.cast('float*', xs.ctypes.data), 577 | ffi.cast('float*', ys.ctypes.data), 578 | ffi.cast('float*', zs.ctypes.data) 579 | ) 580 | return xs, ys, -zs 581 | 582 | def get_points_xyz_array(self, undistorted): 583 | """Return a 3D array of x, y, z points for each point in an undistorted 584 | frame. Invalid points are Nan-ed. 585 | 586 | Args: 587 | undistorted (:py:class:`.Frame`): the undistorted depth frame 588 | 589 | Returns: 590 | A 424x512x3 array of 3D points. The last dimension corresponding to 591 | x, y and z. 592 | 593 | """ 594 | cols, rows = np.meshgrid( 595 | np.arange(undistorted.width), np.arange(undistorted.height)) 596 | return np.dstack(self.get_points_xyz(undistorted, rows, cols)) 597 | 598 | def get_big_points_xyz_array(self, big_depth): 599 | """Like :py:meth:`.get_points_xyz_array` but operates on the "big" depth 600 | map which can be returned from :py:meth:`.apply`. 601 | 602 | Args: 603 | big_depth (:py:class:`.Frame`): big 1920x1082 frame returned from 604 | :py:meth:`.apply`. 605 | 606 | Returns: 607 | A 1082x1920x3 array of 3D points. The last dimension corresponding 608 | to x, y and z. 609 | 610 | """ 611 | points = np.ones((big_depth.height, big_depth.width, 3), dtype=np.float32) 612 | points[..., 2] = 1e-3 * big_depth.to_array() 613 | points[..., 0], points[..., 1] = np.meshgrid( 614 | (np.arange(1920) - self.rgb_p.cx) / self.rgb_p.fx, 615 | (1080-np.arange(-1, 1081) - self.rgb_p.cy) / self.rgb_p.fy) 616 | points[..., 0] *= points[..., 2] 617 | points[..., 1] *= points[..., 2] 618 | 619 | return points 620 | 621 | def write_pcd(self, file_object, undistorted, registered=None): 622 | """Write depth map and (optionally) RGB data to libpcl-compatible PCD 623 | format file. If the registered RGB frame is present, each point is 624 | coloured according to the image otherwise the points are left 625 | uncoloured. 626 | 627 | .. note:: 628 | 629 | Under Python 3 the file object *must* be opened in binary mode. 630 | 631 | Args: 632 | file_object (file): A file object to write PCD data to 633 | undistorted (:py:class:`Frame`): the undistorted depth frame 634 | registered (:py:class:`Frame`): if not-None, the RGB data corresponding to 635 | the depth frame. 636 | """ 637 | write_pcd( 638 | file_object, self.get_points_xyz_array(undistorted), registered) 639 | 640 | def write_big_pcd(self, file_object, big_depth, rgb=None): 641 | """Write depth map and (optionally) RGB data to libpcl-compatible PCD 642 | format file. Works like :py:meth:`.write_pcd` except that it works on 643 | the "big" depth map which can be returned from :py:meth:`apply`. If the 644 | RGB frame is present, each point is coloured according to the image 645 | otherwise the points are left uncoloured. 646 | 647 | .. note:: 648 | 649 | Under Python 3 the file object *must* be opened in binary mode. 650 | 651 | Args: 652 | file_object (file): A file object to write PCD data to 653 | big_depth (:py:class:`Frame`): the 1920x1082 de[th frame 654 | registered (:py:class:`Frame`): if not-None, the RGB data from the 655 | color camera 656 | """ 657 | write_pcd( 658 | file_object, 659 | self.get_big_points_xyz_array(big_depth)[1:-1, ...], rgb) 660 | 661 | def write_pcd(file_object, points, rgb=None): 662 | """Write 3d points and (optionally) RGB data to libpcl-compatible PCD 663 | format file. If the registered RGB frame is present, each point is 664 | coloured according to the image otherwise the points are left 665 | uncoloured. 666 | 667 | .. note:: 668 | 669 | Under Python 3 the file object *must* be opened in binary mode. 670 | 671 | Args: 672 | file_object (file): A file object to write PCD data to 673 | points (array): A NxMx3 array of 3d points. 674 | rgb (:py:class:`Frame`): if not-None, the RGB frame corresponding to 675 | the points array. Assumed to be NxM. 676 | """ 677 | assert len(points.shape) == 3 678 | xs, ys, zs = points[..., 0], points[..., 1], points[..., 2] 679 | n_points = int(np.product(points.shape[:-1])) 680 | 681 | file_object.write(b'VERSION .7\n') 682 | 683 | if rgb is None: 684 | file_object.write( 685 | b'FIELDS x y z\nSIZE 4 4 4\nTYPE F F F\nCOUNT 1 1 1\n') 686 | data = np.zeros((n_points, 3), order='C', dtype=np.float32) 687 | data[:, 0] = xs.flatten() 688 | data[:, 1] = ys.flatten() 689 | data[:, 2] = zs.flatten() 690 | else: 691 | file_object.write( 692 | b'FIELDS x y z rgb\nSIZE 4 4 4 4\nTYPE F F F F\nCOUNT 1 1 1 1\n') 693 | bgrx = rgb.to_array().astype(np.uint32) 694 | rgbs = ( 695 | bgrx[..., 0] + (bgrx[..., 1] << 8) + (bgrx[..., 2] << 16) 696 | ).view(np.float32) 697 | data = np.zeros((n_points, 4), order='C', dtype=np.float32) 698 | data[:, 0] = xs.flatten() 699 | data[:, 1] = ys.flatten() 700 | data[:, 2] = zs.flatten() 701 | data[:, 3] = rgbs.flatten() 702 | 703 | file_object.write('WIDTH {}\n'.format(points.shape[1]).encode()) 704 | file_object.write('HEIGHT {}\n'.format(points.shape[0]).encode()) 705 | file_object.write(b'VIEWPOINT 0 0 0 1 0 0 0\n') 706 | file_object.write('POINTS {}\n'.format(n_points).encode()) 707 | file_object.write(b'DATA binary\n') 708 | file_object.write(data.tobytes()) 709 | --------------------------------------------------------------------------------