├── README.md
└── planar_filter.ipynb


/README.md:
--------------------------------------------------------------------------------
1 | A toy numpy reimplementation of the "Planar Filter" from Section 7.1 (Algorithms 1 and 2) of "Depth from motion for smartphone AR", Valentin et al., SIGGRAPH Asia 2018
2 | 
3 | 


--------------------------------------------------------------------------------
/planar_filter.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |   "nbformat": 4,
  3 |   "nbformat_minor": 0,
  4 |   "metadata": {
  5 |     "colab": {
  6 |       "name": "planar_filter.ipynb",
  7 |       "provenance": [],
  8 |       "collapsed_sections": [],
  9 |       "authorship_tag": "ABX9TyNtj810FNvcHH8gCXjIJTfZ",
 10 |       "include_colab_link": true
 11 |     },
 12 |     "kernelspec": {
 13 |       "name": "python3",
 14 |       "display_name": "Python 3"
 15 |     }
 16 |   },
 17 |   "cells": [
 18 |     {
 19 |       "cell_type": "markdown",
 20 |       "metadata": {
 21 |         "id": "view-in-github",
 22 |         "colab_type": "text"
 23 |       },
 24 |       "source": [
 25 |         "<a href=\"https://colab.research.google.com/github/jonbarron/planar_filter/blob/main/planar_filter.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
 26 |       ]
 27 |     },
 28 |     {
 29 |       "cell_type": "markdown",
 30 |       "metadata": {
 31 |         "id": "CjGhKZLDthj3"
 32 |       },
 33 |       "source": [
 34 |         "A reimplementation of the \"Planar Filter\" from Section 7.1 (Algorithms 1 and 2) of [Depth from motion for smartphone AR\n",
 35 |         "](https://research.google/pubs/pub48288/), Valentin et al., SIGGRAPH Asia 2018"
 36 |       ]
 37 |     },
 38 |     {
 39 |       "cell_type": "code",
 40 |       "metadata": {
 41 |         "id": "vjvpIdQxo918"
 42 |       },
 43 |       "source": [
 44 |         "import numpy as np\n",
 45 |         "import matplotlib.pyplot as plt"
 46 |       ],
 47 |       "execution_count": null,
 48 |       "outputs": []
 49 |     },
 50 |     {
 51 |       "cell_type": "code",
 52 |       "metadata": {
 53 |         "id": "uKrRdHtXwOWZ"
 54 |       },
 55 |       "source": [
 56 |         "def solve_image_ldl3(A11, A12, A13, A22, A23, A33, b1, b2, b3):\n",
 57 |         "  # An unrolled LDL solver for a 3x3 symmetric linear system.\n",
 58 |         "  d1 = A11\n",
 59 |         "  L12 = A12/d1\n",
 60 |         "  d2 = A22 - L12*A12\n",
 61 |         "  L13 = A13/d1\n",
 62 |         "  L23 = (A23 - L13*A12)/d2\n",
 63 |         "  d3 = A33 - L13*A13 - L23*L23*d2\n",
 64 |         "  y1 = b1\n",
 65 |         "  y2 = b2 - L12*y1\n",
 66 |         "  y3 = b3 - L13*y1 - L23*y2\n",
 67 |         "  x3 = y3/d3\n",
 68 |         "  x2 = y2/d2 - L23*x3\n",
 69 |         "  x1 = y1/d1 - L12*x2 - L13*x3\n",
 70 |         "  return x1, x2, x3\n",
 71 |         "\n",
 72 |         "def planar_filter(Z, filt, eps):\n",
 73 |         "  # Solve for the plane at each pixel in `Z`, where the plane fit is computed\n",
 74 |         "  # by using `filt` (a function that blurs something of the same size and shape\n",
 75 |         "  # as `Z` by taking a linear non-negative combination of inputs) to weight\n",
 76 |         "  # pixels in Z, and `eps` regularizes the output to be fronto-parallel.\n",
 77 |         "  # Returns (Zx, Zy, Zz), which is a plane parameterization for each pixel:\n",
 78 |         "  # the derivative wrt x and y, and the offset (which can itself be used as\n",
 79 |         "  # \"the\" filtered output).\n",
 80 |         "\n",
 81 |         "  # Note: This isn't the same code as in the paper. I flipped x and y to match\n",
 82 |         "  # a more pythonic (x, y) convention, and I had to flip a sign on the output\n",
 83 |         "  # slopes to make the unit tests pass(this may be a bug in the paper's math).\n",
 84 |         "  # Also, I decided to not regularize the \"offset\" component of the plane fit,\n",
 85 |         "  # which means that setting eps -> infinity gives the output (0, 0, filt(Z)).\n",
 86 |         "  xy_shape = np.array(Z.shape[-2:])\n",
 87 |         "  xy_scale = 2 / np.mean(xy_shape-1)  # Scaling the x, y coords to be in ~[0, 1]\n",
 88 |         "  x, y = np.meshgrid(*[(np.arange(s) - (s-1)/2) * xy_scale for s in xy_shape], indexing='ij')\n",
 89 |         "  [F1, Fx, Fy, Fz, Fxx, Fxy, Fxz, Fyy, Fyz] = [\n",
 90 |         "    filt(t) for t in [\n",
 91 |         "    np.ones_like(x), x, y, Z, x**2, x*y, x*Z, y**2, y*Z]]\n",
 92 |         "  A11 = F1*x**2 - 2*x*Fx + Fxx + eps**2\n",
 93 |         "  A22 = F1*y**2 - 2*y*Fy + Fyy + eps**2\n",
 94 |         "  A12 = F1*y*x - x*Fy - y*Fx + Fxy\n",
 95 |         "  A13 = F1*x - Fx\n",
 96 |         "  A23 = F1*y - Fy\n",
 97 |         "  A33 = F1# + eps**2\n",
 98 |         "  b1 = Fz*x - Fxz\n",
 99 |         "  b2 = Fz*y - Fyz\n",
100 |         "  b3 = Fz\n",
101 |         "  Zx, Zy, Zz = solve_image_ldl3(A11, A12, A13, A22, A23, A33, b1, b2, b3)\n",
102 |         "  return -Zx*xy_scale, -Zy*xy_scale, Zz"
103 |       ],
104 |       "execution_count": null,
105 |       "outputs": []
106 |     },
107 |     {
108 |       "cell_type": "code",
109 |       "metadata": {
110 |         "id": "hpkhkvJ0Sf6o"
111 |       },
112 |       "source": [
113 |         "# A simple linear blur filter. This can be whatever, provided it averages the\n",
114 |         "# input images by averaging its inputs with non-negative weights.\n",
115 |         "def blur(X, alpha):\n",
116 |         "  # Do an exponential decay filter on the outermost two dimensions of X.\n",
117 |         "  # Equivalent to convolving an image with a Laplacian blur.\n",
118 |         "  Y = X.copy()\n",
119 |         "  for i in range(Y.shape[-1]-1):\n",
120 |         "    Y[...,i+1] += alpha * Y[...,i]\n",
121 |         "\n",
122 |         "  for i in range(Y.shape[-1]-1)[::-1]:\n",
123 |         "    Y[...,i] += alpha * Y[...,i+1]\n",
124 |         "\n",
125 |         "  for i in range(Y.shape[-2]-1):\n",
126 |         "    Y[...,i+1,:] += alpha * Y[...,i,:]\n",
127 |         "\n",
128 |         "  for i in range(Y.shape[-2]-1)[::-1]:\n",
129 |         "    Y[...,i,:] += alpha * Y[...,i+1,:]\n",
130 |         "  return Y"
131 |       ],
132 |       "execution_count": null,
133 |       "outputs": []
134 |     },
135 |     {
136 |       "cell_type": "code",
137 |       "metadata": {
138 |         "id": "20IQWVBJpY3r"
139 |       },
140 |       "source": [
141 |         "# Test that planar_filter'ing correctly recovers planes on single images.\n",
142 |         "np.random.seed(0)\n",
143 |         "for i_test in range(10):\n",
144 |         "\n",
145 |         "  # Make a random plane.\n",
146 |         "  x, y = np.meshgrid(range(int(32 + 32*np.random.uniform())), range(int(32 + 32*np.random.uniform())), indexing='ij')\n",
147 |         "  sx, sy, shift = np.random.normal(size=(3))\n",
148 |         "  Z_true = sx * x + sy * y + shift\n",
149 |         "\n",
150 |         "  # Mask out most of the pixels\n",
151 |         "  mask = (np.mod(x, 4) == 0) & (np.mod(y, 4) == 0)\n",
152 |         "  Z = mask * Z_true\n",
153 |         "  W = np.float32(mask)\n",
154 |         "\n",
155 |         "  # Define a blur function.\n",
156 |         "  alpha = 0.2\n",
157 |         "  filt = lambda x : blur(x * W, alpha) / blur(W, alpha)\n",
158 |         "\n",
159 |         "  # normal filteirng, and planar_filter'ing\n",
160 |         "  Zf = filt(Z)\n",
161 |         "  Zx, Zy, Zz = planar_filter(Z, filt, 1e-4)\n",
162 |         "\n",
163 |         "  basic_max_error = np.max(np.abs(Zf - Z_true))\n",
164 |         "  planar_max_error = np.max(np.abs(Zz - Z_true))\n",
165 |         "  print(f'Errors = {basic_max_error:0.5f} | {planar_max_error:0.5f}')\n",
166 |         "\n",
167 |         "  # Plane fitting correctly recovers the true plane values.\n",
168 |         "  assert(planar_max_error < 0.01)\n",
169 |         "\n",
170 |         "  # Plane fitting correctly recovers the slope of the plane.\n",
171 |         "  assert(np.max(np.abs(np.median(Zx) - sx)) < 0.001)\n",
172 |         "  assert(np.max(np.abs(np.median(Zy) - sy)) < 0.001)\n",
173 |         "\n",
174 |         "  # Setting `eps` -> infinity behaves as expected.\n",
175 |         "  Zx0, Zy0, Zf_recon = planar_filter(Z, filt, 1e10)\n",
176 |         "  assert(np.max(np.abs(Zx0)) < 0.001)\n",
177 |         "  assert(np.max(np.abs(Zy0)) < 0.001)\n",
178 |         "  assert(np.max(np.abs(Zf_recon - Zf)) < 0.001)\n",
179 |         "\n",
180 |         "  plt.figure(i_test)\n",
181 |         "  _, ax = plt.subplots(1, 3, figsize=(12, 4))\n",
182 |         "  ax[0].imshow(Zf)\n",
183 |         "  ax[1].imshow(Zz)\n",
184 |         "  ax[2].imshow(Z_true)"
185 |       ],
186 |       "execution_count": null,
187 |       "outputs": []
188 |     },
189 |     {
190 |       "cell_type": "code",
191 |       "metadata": {
192 |         "id": "AwB_xXKqUI4c"
193 |       },
194 |       "source": [
195 |         "# Test that planar_filter'ing works correctly on batches of data.\n",
196 |         "np.random.seed(0)\n",
197 |         "x, y = np.meshgrid(range(32), range(48), indexing='ij')\n",
198 |         "mask = (np.mod(x, 4) == 0) & (np.mod(y, 4) == 0)\n",
199 |         "W = np.float32(mask)\n",
200 |         "\n",
201 |         "Zs = []\n",
202 |         "Zs_true = []\n",
203 |         "s_true = []\n",
204 |         "for i_test in range(10):\n",
205 |         "\n",
206 |         "  sx, sy, shift = np.random.normal(size=(3))\n",
207 |         "\n",
208 |         "  Z_true = sx * x + sy * y + shift\n",
209 |         "  Z = mask * Z_true\n",
210 |         "\n",
211 |         "  Zs_true.append(Z_true)\n",
212 |         "  Zs.append(Z)\n",
213 |         "  s_true.append((sx, sy, shift))\n",
214 |         "\n",
215 |         "Zs = np.stack(Zs, 0)\n",
216 |         "Zs_true = np.stack(Zs_true, 0)\n",
217 |         "\n",
218 |         "alpha = 0.2\n",
219 |         "filt = lambda x : blur(x * W, alpha) / blur(W, alpha)\n",
220 |         "\n",
221 |         "Zsf = filt(Zs)\n",
222 |         "Zsx, Zsy, Zsz = planar_filter(Zs, filt, 1e-4)\n",
223 |         "\n",
224 |         "basic_max_error = np.max(np.abs(Zsf - Zs_true))\n",
225 |         "planar_max_error = np.max(np.abs(Zsz - Zs_true))\n",
226 |         "print(f'Errors = {basic_max_error:0.5f} | {planar_max_error:0.5f}')\n",
227 |         "assert(planar_max_error < 0.01)\n",
228 |         "\n",
229 |         "assert np.all(np.abs(np.array([s[0] for s in s_true])[:,None,None] - Zsx) < 1e-3)\n",
230 |         "assert np.all(np.abs(np.array([s[1] for s in s_true])[:,None,None] - Zsy) < 1e-3)\n",
231 |         "\n",
232 |         "Zsx0, Zsy0, Zsf_recon = planar_filter(Zs, filt, 1e10)\n",
233 |         "assert(np.max(np.abs(Zsx0)) < 0.001)\n",
234 |         "assert(np.max(np.abs(Zsy0)) < 0.001)\n",
235 |         "assert(np.max(np.abs(Zsf_recon - Zsf)) < 0.001)\n",
236 |         "\n",
237 |         "plt.figure(figsize=(20,20))\n",
238 |         "plt.imshow(np.concatenate([np.reshape(Zsf, [-1, Zsf.shape[-1]]), np.reshape(Zsz, [-1, Zsz.shape[-1]]), np.reshape(Zs_true, [-1, Zs_true.shape[-1]])], 1))"
239 |       ],
240 |       "execution_count": null,
241 |       "outputs": []
242 |     }
243 |   ]
244 | }


--------------------------------------------------------------------------------