├── .gitignore ├── README.md ├── examples └── simple.py └── projectionmapping ├── __init__.py └── data ├── mirefullhd.jpg └── white.png /.gitignore: -------------------------------------------------------------------------------- 1 | calibration.json 2 | __pycache__ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Projection Mapping 2 | 3 | Kivy widget that reproject its content according to a calibration grid. 4 | It includes a calibration tool accessible from F2. 5 | 6 | screenshot_projectionmapping 7 | 8 | Projection to a curved wall: 9 | 10 | ![projection_to_curved_wall](https://user-images.githubusercontent.com/37904/55559503-a3186180-56ee-11e9-8832-1b145110666f.png) 11 | 12 | ## Usage 13 | 14 | ```python 15 | from kivy.app import App 16 | from projectionmapping import ProjectionMapping 17 | from kivy.uix.image import Image 18 | 19 | class SimpleProjectionMapping(App): 20 | def build(self): 21 | self.root = ProjectionMapping(filename="calibration.json") 22 | self.root.add_widget( 23 | Image(source="projectionmapping/data/mirefullhd.jpg")) 24 | ``` 25 | 26 | ## Keybinding 27 | 28 | - F2: Toggle calibration 29 | - space: Toggle help 30 | - r: Reset the calibration grid 31 | - s: Save the current calibration 32 | - l: Load latest calibration 33 | - x/c: Remove/add a column (current calibration is lost) 34 | - v/b: Remove/add a row (current calibration is lost) 35 | 36 | ## Resources 37 | 38 | - http://www.reedbeta.com/blog/quadrilateral-interpolation-part-2/ 39 | - http://iquilezles.org/www/articles/ibilinear/ibilinear.htm 40 | -------------------------------------------------------------------------------- /examples/simple.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path += ["."] 3 | from kivy.app import App 4 | from projectionmapping import ProjectionMapping 5 | from kivy.uix.image import Image 6 | 7 | 8 | class SimpleProjectionMapping(App): 9 | def build(self): 10 | self.root = ProjectionMapping(filename="calibration.json") 11 | self.root.add_widget( 12 | Image(source="projectionmapping/data/mirefullhd.jpg")) 13 | 14 | if __name__ == "__main__": 15 | SimpleProjectionMapping().run() -------------------------------------------------------------------------------- /projectionmapping/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Projection mapping 3 | ================== 4 | 5 | .. author:: Mathieu Virbel 6 | 7 | Grid-based Inverse Bilinear Projection 8 | """ 9 | 10 | from kivy.factory import Factory as F 11 | from kivy.properties import StringProperty, BooleanProperty 12 | from kivy.graphics import ( 13 | Fbo, Rectangle, Color, Mesh, PushMatrix, PopMatrix, Scale, 14 | Canvas, RenderContext, Translate) 15 | from kivy.graphics.transformation import Matrix 16 | from kivy.lang import Builder 17 | from kivy.vector import Vector 18 | import json 19 | 20 | FS = ''' 21 | #ifdef GL_ES 22 | precision highp float; 23 | #endif 24 | 25 | /* Outputs from the vertex shader */ 26 | varying vec4 frag_color; 27 | varying vec2 q, q0, b1, b2, b3, vsize; 28 | 29 | /* uniform texture samplers */ 30 | uniform sampler2D texture0; 31 | uniform mat4 frag_modelview_mat; 32 | 33 | float Wedge2D(vec2 v, vec2 w) { 34 | return v.x * w.y - v.y * w.x; 35 | } 36 | 37 | void main (void){ 38 | // Set up quadratic formula 39 | float A = Wedge2D(b2, b3); 40 | float B = Wedge2D(b3, q) - Wedge2D(b1, b2); 41 | float C = Wedge2D(b1, q); 42 | 43 | // Solve for v 44 | vec2 uv; 45 | if (abs(A) < 0.001) { 46 | // Linear form 47 | uv.y = -C / B; 48 | } else { 49 | // Quadratic form. Take positive root for CCW winding with V-up 50 | float discrim = B * B - 4. * A *C; 51 | uv.y = 0.5 * (-B + sqrt(discrim)) / A; 52 | } 53 | 54 | // Solve for u, using largest-magnitude component 55 | vec2 denom = b1 + uv.y * b3; 56 | if (abs(denom.x) > abs(denom.y)) 57 | uv.x = (q.x - b2.x * uv.y) / denom.x; 58 | else 59 | uv.x = (q.y - b2.y * uv.y) / denom.y; 60 | 61 | uv.x /= vsize.y; 62 | uv.y /= vsize.x; 63 | uv.x += q0.y; 64 | uv.y += q0.x; 65 | 66 | gl_FragColor = frag_color * texture2D(texture0, vec2(uv.y, uv.x)); 67 | } 68 | ''' 69 | 70 | VS = ''' 71 | #ifdef GL_ES 72 | precision highp float; 73 | #endif 74 | 75 | /* Outputs to the fragment shader */ 76 | varying vec4 frag_color; 77 | varying vec2 q; 78 | varying vec2 q0; 79 | varying vec2 b1; 80 | varying vec2 b2; 81 | varying vec2 b3; 82 | varying vec2 vsize; 83 | 84 | /* vertex attributes */ 85 | attribute vec2 vPosition; 86 | attribute vec2 vQuad0; 87 | attribute vec2 vQuad1; 88 | attribute vec2 vQuad2; 89 | attribute vec2 vQuad3; 90 | attribute vec2 vTex0; 91 | attribute vec2 vSize; 92 | 93 | /* uniform variables */ 94 | uniform mat4 modelview_mat; 95 | uniform mat4 projection_mat; 96 | uniform vec4 color; 97 | uniform float opacity; 98 | 99 | void main (void) { 100 | q = vPosition - vQuad0; 101 | q0 = vTex0; 102 | b1 = vQuad1 - vQuad0; 103 | b2 = vQuad2 - vQuad0; 104 | b3 = vQuad0 - vQuad1 - vQuad2 + vQuad3; 105 | vsize = vSize; 106 | frag_color = color * vec4(1.0, 1.0, 1.0, opacity); 107 | gl_Position = projection_mat * modelview_mat * vec4(vPosition.xy, 0.0, 1.0); 108 | } 109 | ''' 110 | 111 | 112 | 113 | Builder.load_string(""" 114 | : 115 | Label: 116 | size_hint: None, None 117 | size: self.texture_size[0] + dp(20), self.texture_size[1] + dp(20) 118 | text: root.informations 119 | markup: True 120 | opacity: int(root.show_help) 121 | font_name: "data/fonts/RobotoMono-Regular.ttf" 122 | canvas.before: 123 | Color: 124 | rgba: 0, 0, 0, 0.7 125 | Rectangle: 126 | pos: self.pos 127 | size: self.size 128 | 129 | : 130 | ProjectionMappingGrid: 131 | id: container 132 | ProjectionMappingCalibration: 133 | id: calibration 134 | """) 135 | 136 | 137 | class ProjectionMappingGrid(F.RelativeLayout): 138 | 139 | def __init__(self, **kwargs): 140 | self.cols = self.rows = 2 141 | self.canvas = RenderContext( 142 | fs=FS, vs=VS, 143 | use_parent_projection=True, 144 | use_parent_modelview=True) 145 | super(ProjectionMappingGrid, self).__init__(**kwargs) 146 | self.build_mapping() 147 | self.init_fbo() 148 | self.bind(size=self.rebuild_fbo) 149 | 150 | def add_widget(self, widget): 151 | if widget not in self.children: 152 | self.children.append(widget) 153 | self.g_fbo.add(widget.canvas) 154 | 155 | def remove_widget(self, widget): 156 | if widget in self.children: 157 | self.children.remove(widget) 158 | self.g_fbo.remove(widget.canvas) 159 | 160 | def init_fbo(self): 161 | with self.canvas: 162 | Color(1, 1, 1) 163 | self.g_fbo = Fbo(size=self.size) 164 | self.g_fbo_texture = self.g_fbo.texture 165 | Color(1, 1, 1, 1) 166 | PushMatrix() 167 | self.g_scale = Scale(self.width / 2, self.height / 2, 1.) 168 | self.build_grid() 169 | PopMatrix() 170 | 171 | def rebuild_fbo(self, *largs): 172 | asp = self.width / float(self.height) 173 | self.g_fbo.size = self.size 174 | self.g_fbo_texture = self.g_fbo.texture 175 | self.update_grid() 176 | 177 | def build_mapping(self, calibration=None): 178 | rows = self.rows 179 | cols = self.cols 180 | line_vertices = [] 181 | line_indices = [] 182 | ncols = float(cols) 183 | nrows = float(rows) 184 | i = 0 185 | for row in range(rows + 1): 186 | for col in range(cols + 1): 187 | if calibration: 188 | i = 2 * (col + row * (cols + 1)) 189 | line_vertices += calibration[i:i + 2] 190 | line_vertices += [0, 0] 191 | else: 192 | line_vertices += [col / ncols, row / nrows, 0, 0] 193 | for row in range(rows): 194 | for col in range(cols): 195 | i = col + row * (cols + 1) 196 | line_indices += [i, i + 1] 197 | line_indices += [i, i + (cols + 1)] 198 | self.line_vertices = line_vertices 199 | self.line_indices = line_indices 200 | 201 | def get_calibration(self): 202 | calibration = [] 203 | v = self.line_vertices 204 | for i in range(0, len(v), 4): 205 | calibration += v[i:i + 2] 206 | return calibration 207 | 208 | def build_grid(self): 209 | rows = self.rows 210 | cols = self.cols 211 | vertices = [] 212 | indices = [] 213 | 214 | dx = 1. / float(cols) 215 | dy = 1. / float(rows) 216 | for col in range(cols): 217 | x = col / float(cols) 218 | for row in range(rows): 219 | y = row / float(rows) 220 | 221 | # use line 222 | corners = [] 223 | i = 4 * (col + row * (cols + 1)) 224 | corners += self.line_vertices[i:i + 2] 225 | i = 4 * (col + (row + 1) * (cols + 1)) 226 | corners += self.line_vertices[i:i + 2] 227 | i = 4 * (1 + col + row * (cols + 1)) 228 | corners += self.line_vertices[i:i + 2] 229 | i = 4 * (1 + col + (row + 1) * (cols + 1)) 230 | corners += self.line_vertices[i:i + 2] 231 | 232 | data = [ 233 | x, y, cols, rows 234 | ] 235 | 236 | vertices.extend(corners[0:2]) 237 | vertices.extend(corners) 238 | vertices.extend(data) 239 | vertices.extend(corners[2:4]) 240 | vertices.extend(corners) 241 | vertices.extend(data) 242 | vertices.extend(corners[4:6]) 243 | vertices.extend(corners) 244 | vertices.extend(data) 245 | vertices.extend(corners[6:8]) 246 | vertices.extend(corners) 247 | vertices.extend(data) 248 | 249 | i = 0 250 | for col in range(cols): 251 | for row in range(rows): 252 | indices.extend(( 253 | i, i + 3, i + 1, 254 | i, i + 2, i + 3)) 255 | i += 4 256 | 257 | self.indices = indices 258 | self.vertices = vertices 259 | fmt = [ 260 | (b'vPosition', 2, 'float'), 261 | (b'vQuad0', 2, 'float'), 262 | (b'vQuad1', 2, 'float'), 263 | (b'vQuad2', 2, 'float'), 264 | (b'vQuad3', 2, 'float'), 265 | (b'vTex0', 2, 'float'), 266 | (b'vSize', 2, 'float') 267 | ] 268 | fmtsize = 14 269 | 270 | if not hasattr(self, "g_mesh"): 271 | self.g_mesh = Mesh( 272 | indices=indices, vertices=vertices, mode="triangles", 273 | texture=self.g_fbo_texture, 274 | fmt=fmt 275 | ) 276 | else: 277 | self.g_mesh.indices = indices 278 | self.g_mesh.vertices = vertices 279 | 280 | def update_grid(self): 281 | self.g_scale.x = self.width 282 | self.g_scale.y = self.height 283 | self.g_mesh.texture = self.g_fbo_texture 284 | 285 | def set_vertice(self, i, sx, sy): 286 | line_vertices = self.line_vertices 287 | line_vertices[i * 4] = sx 288 | line_vertices[i * 4 + 1] = sy 289 | self.build_grid() 290 | 291 | 292 | class ProjectionMappingCalibration(F.RelativeLayout): 293 | informations = StringProperty() 294 | show_help = BooleanProperty(True) 295 | 296 | def __init__(self, **kwargs): 297 | super(ProjectionMappingCalibration, self).__init__(**kwargs) 298 | self.g_canvas = None 299 | 300 | def rebuild_informations(self): 301 | self.informations = "\n".join([ 302 | "[b]Projection Mapping[/b]", 303 | "Cols: {} - Rows: {}", 304 | "", 305 | "[b]Help[/b]", 306 | "F2: Toggle calibration", 307 | "space: Toggle help", 308 | "r: Reset the calibration grid", 309 | "s: Save the current calibration", 310 | "l: Load latest calibration", 311 | "x/c: Remove/add a column (current calibration is lost)", 312 | "v/b: Remove/add a row (current calibration is lost)" 313 | ]).format( 314 | self.grid.cols, 315 | self.grid.rows) 316 | 317 | def show_lines(self): 318 | indices = [] 319 | grid = self.grid 320 | cols = grid.cols 321 | rows = grid.rows 322 | for col in range(grid.cols + 1): 323 | indices.extend(( 324 | col * (rows + 1), col * (rows + 1) + rows, 325 | )) 326 | for row in range(grid.rows + 1): 327 | indices.extend(( 328 | row, row + (cols * (rows + 1)), 329 | )) 330 | 331 | with self.canvas: 332 | self.g_canvas = Canvas() 333 | 334 | with self.g_canvas: 335 | Color(1, 0, 0, 0.5) 336 | PushMatrix() 337 | Scale(self.width, self.height, 1.) 338 | self.g_mesh = Mesh( 339 | vertices=self.grid.line_vertices, 340 | indices=self.grid.line_indices, 341 | mode="lines", 342 | source="projectionmapping/data/white.png") 343 | PopMatrix() 344 | 345 | self.rebuild_informations() 346 | 347 | def hide_lines(self): 348 | if self.g_canvas: 349 | self.canvas.remove(self.g_canvas) 350 | self.g_canvas = None 351 | 352 | def update_mesh(self): 353 | self.hide_lines() 354 | self.show_lines() 355 | 356 | def on_touch_down(self, touch): 357 | cols = self.grid.cols 358 | rows = self.grid.rows 359 | 360 | # select the nearest point 361 | v = self.grid.line_vertices 362 | vt = Vector(touch.sx, touch.sy) 363 | min_i = -1 364 | min_dist = float("inf") 365 | for i4 in range(0, len(v), 4): 366 | d = Vector(v[i4:i4 + 2]).distance(vt) 367 | if min_dist > d: 368 | min_dist = d 369 | min_i = i4 / 4 370 | touch.ud["i"] = int(min_i) 371 | touch.grab(self) 372 | return super(ProjectionMappingCalibration, self).on_touch_down(touch) 373 | 374 | def on_touch_move(self, touch): 375 | if touch.grab_current is self: 376 | self.grid.set_vertice(touch.ud["i"], touch.sx, touch.sy) 377 | self.update_mesh() 378 | return True 379 | return super(ProjectionMappingCalibration, self).on_touch_move(touch) 380 | 381 | def on_touch_up(self, touch): 382 | if touch.grab_current is self: 383 | self.grid.set_vertice(touch.ud["i"], touch.sx, touch.sy) 384 | self.update_mesh() 385 | return True 386 | return super(ProjectionMappingCalibration, self).on_touch_up(touch) 387 | 388 | 389 | class ProjectionMapping(F.RelativeLayout): 390 | def __init__(self, **kwargs): 391 | self.wid_container = self.wid_calibration = None 392 | self.filename = kwargs.pop("filename", "calibration.json") 393 | super(ProjectionMapping, self).__init__(**kwargs) 394 | self.wid_container = self.ids.container.__self__ 395 | self.wid_calibration = self.ids.calibration.__self__ 396 | self.remove_widget(self.wid_calibration) 397 | self.bind_keyboard() 398 | self.load_calibration() 399 | 400 | def save_calibration(self): 401 | data = { 402 | "rows": self.wid_container.rows, 403 | "cols": self.wid_container.cols, 404 | "calibration": self.wid_container.get_calibration() 405 | } 406 | with open(self.filename, "w") as fd: 407 | json.dump(data, fd) 408 | print("Calibration saved to {}".format(self.filename)) 409 | 410 | def load_calibration(self): 411 | try: 412 | with open(self.filename, "r") as fd: 413 | data = json.load(fd) 414 | except Exception as e: 415 | print("ERROR: Unable to load {}: {!r}".format( 416 | self.filename, e)) 417 | return 418 | self.wid_container.rows = data["rows"] 419 | self.wid_container.cols = data["cols"] 420 | self.wid_container.build_mapping(calibration=data["calibration"]) 421 | self.wid_container.build_grid() 422 | if self.wid_calibration.parent: 423 | self.hide_projection() 424 | self.show_projection() 425 | 426 | def add_widget(self, widget): 427 | if self.wid_container: 428 | return self.wid_container.add_widget(widget) 429 | return super(ProjectionMapping, self).add_widget(widget) 430 | 431 | def remove_widget(self, widget): 432 | if widget in self.wid_container.children: 433 | return self.wid_container.remove_widget(widget) 434 | return super(ProjectionMapping, self).remove_widget(widget) 435 | 436 | def bind_keyboard(self): 437 | from kivy.core.window import Window 438 | 439 | def on_key_down(window, scancode, *largs): 440 | if scancode == 283: 441 | self.toggle_projection() 442 | return True 443 | if not self.wid_calibration.parent: 444 | return 445 | if scancode == 32: # space 446 | self.wid_calibration.show_help = not self.wid_calibration.show_help 447 | return True 448 | elif scancode in (120, 99, 118, 98, 114): # x, c, v, b, r 449 | if scancode == 120: 450 | self.wid_container.rows = max(1, self.wid_container.rows - 1) 451 | elif scancode == 99: 452 | self.wid_container.rows = self.wid_container.rows + 1 453 | elif scancode == 118: 454 | self.wid_container.cols = max(1, self.wid_container.cols - 1) 455 | elif scancode == 98: 456 | self.wid_container.cols = self.wid_container.cols + 1 457 | self.wid_container.build_mapping() 458 | self.wid_container.build_grid() 459 | self.hide_projection() 460 | self.show_projection() 461 | return True 462 | elif scancode == 115: 463 | self.save_calibration() 464 | return True 465 | elif scancode == 108: 466 | self.load_calibration() 467 | return True 468 | 469 | Window.bind(on_key_down=on_key_down) 470 | 471 | def toggle_projection(self): 472 | if self.wid_calibration.parent: 473 | self.hide_projection() 474 | else: 475 | self.show_projection() 476 | 477 | def hide_projection(self): 478 | super(ProjectionMapping, self).remove_widget(self.wid_calibration) 479 | self.wid_calibration.hide_lines() 480 | 481 | def show_projection(self): 482 | super(ProjectionMapping, self).add_widget(self.wid_calibration) 483 | self.wid_calibration.grid = self.wid_container 484 | self.wid_calibration.size = self.size 485 | self.wid_calibration.show_lines() -------------------------------------------------------------------------------- /projectionmapping/data/mirefullhd.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tito/projectionmapping/990162ac8dd3020c93517e7bc23730b401171752/projectionmapping/data/mirefullhd.jpg -------------------------------------------------------------------------------- /projectionmapping/data/white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tito/projectionmapping/990162ac8dd3020c93517e7bc23730b401171752/projectionmapping/data/white.png --------------------------------------------------------------------------------