├── .gitignore ├── LICENSE ├── README.md ├── shaders ├── copy.frag.glsl ├── drawCanvas.frag.glsl ├── expandKnown.frag.glsl ├── gathering.frag.glsl ├── localSmooth.frag.glsl ├── quad.vert.glsl └── refinement.frag.glsl ├── shared.js └── utils.js /.gitignore: -------------------------------------------------------------------------------- 1 | **/.DS_Store 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Shared Matting WebGL implementation 2 | Implement Shared Sampling for Real-Time Alpha Matting using webgl shader language(glsl). [Paper](http://inf.ufrgs.br/%7Eeslgastal/SharedMatting/) 3 | 4 | ## Setups && Run 5 | ***Note: all JS are written in ES6 standard*** 6 | 1. A modern browser 7 | 2. turn on webgl. for new version chrome, it is automatically enabled. 8 | 3. A webgl wrapper dependency [`Igloo`](https://github.com/kingfolk/igloojs/blob/master/igloo.js). Do 9 | ``` 10 | npm install igloo-ext --save 11 | ``` 12 | or equaivalent to install this dependency. 13 | 4. Copy code into your project along with all the shaders and `shared.js` 14 | 15 | ```js 16 | let canvas = document.getElementById(YourCanvasID); 17 | let runner = new SharedGL(canvas); 18 | runner.setImage(ImageURL); 19 | runner.setTrimap(TrimapURL); 20 | 21 | // getScale is in the utils.js file 22 | // image is the input image for matting 23 | // scale is used to draw a centered result on canvas 24 | let scale = utils.getScale(canvas.width, canvas.height, image.naturalWidth, image.naturalHeight); 25 | 26 | // will draw the result to canvas 27 | runner.run(scale); 28 | ``` 29 | 30 | ## Demo 31 | 32 | go http://kingfolk.github.io, find project "Shared Matting WebGL" and click demo. 33 | -------------------------------------------------------------------------------- /shaders/copy.frag.glsl: -------------------------------------------------------------------------------- 1 | #ifdef GL_ES 2 | precision mediump float; 3 | #endif 4 | 5 | uniform sampler2D state; 6 | uniform vec2 scale; 7 | 8 | 9 | void main() { 10 | vec2 coord = vec2(gl_FragCoord.x/scale.x, 1.0 - gl_FragCoord.y/scale.y); 11 | gl_FragColor = texture2D(state, coord); 12 | } 13 | -------------------------------------------------------------------------------- /shaders/drawCanvas.frag.glsl: -------------------------------------------------------------------------------- 1 | #ifdef GL_ES 2 | precision mediump float; 3 | #endif 4 | 5 | uniform sampler2D state; 6 | uniform vec2 scale; 7 | uniform vec2 offset; 8 | 9 | 10 | void main() { 11 | if (gl_FragCoord.y < offset.y || gl_FragCoord.x < offset.x 12 | || gl_FragCoord.y > scale.y - offset.y || gl_FragCoord.x > scale.x - offset.x) { 13 | gl_FragColor = vec4(0.0, 0.0, 0.0, 0.0); 14 | } 15 | else { 16 | vec2 coord = vec2((gl_FragCoord.x - offset.x)/(scale.x - 2.0*offset.x), 17 | 1.0 - (gl_FragCoord.y - offset.y)/(scale.y - 2.0*offset.y)); 18 | gl_FragColor = texture2D(state, coord); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /shaders/expandKnown.frag.glsl: -------------------------------------------------------------------------------- 1 | #ifdef GL_ES 2 | precision mediump float; 3 | #endif 4 | 5 | uniform sampler2D oriState; 6 | uniform sampler2D triState; 7 | uniform vec2 scale; 8 | uniform float kC; 9 | 10 | vec3 getOri() { 11 | vec4 color = texture2D(oriState, (gl_FragCoord.xy) / scale); 12 | return color.xyz; 13 | } 14 | 15 | vec3 getOri(vec2 coord) { 16 | vec4 color = texture2D(oriState, coord); 17 | return color.xyz; 18 | } 19 | 20 | vec3 getTri() { 21 | vec4 color = texture2D(triState, (gl_FragCoord.xy) / scale); 22 | return color.xyz; 23 | } 24 | 25 | vec4 getTri(vec2 coord) { 26 | return texture2D(triState, coord); 27 | } 28 | 29 | void main() { 30 | float tri = getTri().x; 31 | gl_FragColor = texture2D(triState, (gl_FragCoord.xy) / scale); 32 | 33 | // unknown region 34 | if (tri > 0.0 && tri < 1.0) { 35 | float x = gl_FragCoord.x, 36 | y = gl_FragCoord.y; 37 | 38 | const float kI = float( %%kI%% ); 39 | 40 | vec3 curColor = getOri(); 41 | for (float i = -kI; i < kI; i += 1.0) { 42 | for (float j = -kI; j < kI; j += 1.0) { 43 | float w = gl_FragCoord.x + i; 44 | float h = gl_FragCoord.y + j; 45 | if (w > 0.0 && w < scale.x 46 | && h > 0.0 && h < scale.y) { 47 | vec2 coord = vec2(w, h); 48 | vec3 color = getOri(coord / scale); 49 | vec4 triColor = getTri(coord / scale); 50 | if ((triColor.x == 0.0 || triColor.x == 1.0) 51 | && distance(coord, gl_FragCoord.xy) < kI 52 | && distance(color, curColor) < kC) { 53 | gl_FragColor = triColor; 54 | } 55 | } 56 | } 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /shaders/gathering.frag.glsl: -------------------------------------------------------------------------------- 1 | #extension GL_EXT_draw_buffers : require 2 | #ifdef GL_ES 3 | precision mediump float; 4 | #endif 5 | 6 | #define i0_ 0 7 | #define i1_ 1 8 | #define i2_ 2 9 | #define i3_ 3 10 | 11 | uniform sampler2D oriState; 12 | uniform sampler2D triState; 13 | uniform int isTest; 14 | uniform vec2 scale; 15 | uniform int kG; 16 | 17 | vec2 FSamples[%%kG%%]; 18 | vec2 BSamples[%%kG%%]; 19 | 20 | struct FBPair { 21 | vec3 F; 22 | vec3 B; 23 | vec2 FCoord; 24 | vec2 BCoord; 25 | }; 26 | 27 | vec2 arrIdx(int which, int idx) { 28 | vec2 res; 29 | if (which == 0) { 30 | if (idx == 0) { res = FSamples[i0_]; } 31 | else if (idx == 1) { res = FSamples[i1_]; } 32 | else if (idx == 2) { res = FSamples[i2_]; } 33 | else if (idx == 3) { res = FSamples[i3_]; } 34 | } 35 | else if (which == 1) { 36 | if (idx == 0) { res = BSamples[i0_]; } 37 | else if (idx == 1) { res = BSamples[i1_]; } 38 | else if (idx == 2) { res = BSamples[i2_]; } 39 | else if (idx == 3) { res = BSamples[i3_]; } 40 | } 41 | 42 | return res; 43 | } 44 | 45 | void arrIdx(int which, int idx, vec2 value) { 46 | if (which == 0) { 47 | if (idx == 0) { FSamples[i0_] = value; } 48 | else if (idx == 1) { FSamples[i1_] = value; } 49 | else if (idx == 2) { FSamples[i2_] = value; } 50 | else if (idx == 3) { FSamples[i3_] = value; } 51 | } 52 | else if (which == 1) { 53 | if (idx == 0) { BSamples[i0_] = value; } 54 | else if (idx == 1) { BSamples[i1_] = value; } 55 | else if (idx == 2) { BSamples[i2_] = value; } 56 | else if (idx == 3) { BSamples[i3_] = value; } 57 | } 58 | } 59 | 60 | vec3 getOri() { 61 | vec4 color = texture2D(oriState, (gl_FragCoord.xy) / scale); 62 | return color.xyz; 63 | } 64 | 65 | vec3 getOri(vec2 coord) { 66 | vec4 color = texture2D(oriState, coord); 67 | return color.xyz; 68 | } 69 | 70 | vec3 getTri() { 71 | vec4 color = texture2D(triState, (gl_FragCoord.xy) / scale); 72 | return color.xyz; 73 | } 74 | 75 | vec3 getTri(vec2 coord) { 76 | return texture2D(triState, coord).xyz; 77 | } 78 | 79 | void sample() { 80 | int fIdx = 0, bIdx = 0; 81 | float x = gl_FragCoord.x, 82 | y = gl_FragCoord.y; 83 | 84 | float inc = 360.0 / float(kG); 85 | float ca = inc / 9.0; 86 | float angle = (x + y) * ca; 87 | for (int k = 0; k < %%kG%%; k ++) { 88 | bool flagF = false; 89 | bool flagB = false; 90 | 91 | float z = (angle + float(k) * inc) / 180.0 * 3.1415926; 92 | float ex = sin(z); 93 | float ey = cos(z); 94 | float step = min(1.0 / (abs(ex) + 1e-10), 1.0 / (abs(ey) + 1e-10)); 95 | 96 | for (int i = 0; i < %%searchRange%%; i ++) { 97 | float t = step * float(i); 98 | float tx = x + ex * t, 99 | ty = y + ey * t; 100 | 101 | if (tx > 0.0 && tx < scale.x 102 | && ty > 0.0 && ty < scale.y 103 | && (!flagF || !flagB) ) { 104 | vec2 coord = vec2(tx, ty) / scale; 105 | float triColor = getTri(coord).x; 106 | if (!flagF && triColor == 1.0) { 107 | arrIdx(0, fIdx++, coord); 108 | flagF = true; 109 | } 110 | else if (!flagB && triColor == 0.0) { 111 | arrIdx(1, bIdx++, coord); 112 | flagB = true; 113 | } 114 | } 115 | } 116 | } 117 | for (int i = 0; i < %%kG%%; i ++) { 118 | if (i >= fIdx) { 119 | arrIdx(0, i, vec2(-1.0, -1.0)); 120 | // arrIdx(0, i, gl_FragCoord.xy / scale); 121 | } 122 | } 123 | for (int i = 0; i < %%kG%%; i ++) { 124 | if (i >= bIdx) { 125 | arrIdx(1, i, vec2(-1.0, -1.0)); 126 | // arrIdx(1, i, gl_FragCoord.xy / scale); 127 | } 128 | } 129 | // FSamples[0] = gl_FragCoord.xy / scale; 130 | // arrIdx(0, 0, gl_FragCoord.xy / scale); 131 | } 132 | 133 | 134 | float e_p(vec2 coord) { 135 | float lx = coord.x * scale.x - gl_FragCoord.x, 136 | ly = coord.y * scale.y - gl_FragCoord.y, 137 | l = length(vec2(lx, ly)); 138 | 139 | float ex = lx / (l + 1e-10), 140 | ey = ly / (l + 1e-10), 141 | step = min(1.0/(abs(ex) + 1e-10), 1.0/(abs(ey) + 1e-10)), 142 | res = 0.0; 143 | vec3 color = getOri(); 144 | 145 | for (int i = 0; i < %%searchRange%%; i ++) { 146 | float t = step * float(i); 147 | if (t < l) { 148 | float x = gl_FragCoord.x + t * ex, 149 | y = gl_FragCoord.y + t * ey; 150 | vec2 coord = vec2(x, y) / scale; 151 | vec3 color_ = getOri(coord); 152 | float d = distance(color, color_); 153 | res += d * d; 154 | } 155 | } 156 | return res; 157 | } 158 | 159 | float pf_p() { 160 | float fMin = 1e10, 161 | bMin = 1e10; 162 | for (int i = 0; i < %%kG%%; i ++) { 163 | float f = e_p(arrIdx(0, i)), 164 | b = e_p(arrIdx(1, i)); 165 | if (f < fMin) { 166 | fMin = f; 167 | } 168 | if (b < bMin) { 169 | bMin = b; 170 | } 171 | } 172 | 173 | return bMin / (fMin + bMin + 1e-10); 174 | } 175 | 176 | float estimAlpha(vec3 color, vec3 fColor, vec3 bColor) { 177 | float d = distance(fColor, bColor); 178 | return dot((color - bColor), (fColor - bColor)) / (d * d); 179 | } 180 | 181 | float a_p(vec2 FCoord, vec2 BCoord, float pfp) { 182 | vec3 color = getOri(); 183 | float alpha = estimAlpha(color, getOri(FCoord), getOri(BCoord)); 184 | 185 | return pfp + (1.0 - 2.0 * pfp) * alpha; 186 | } 187 | 188 | float d_p(vec2 coord) { 189 | return distance(gl_FragCoord.xy, coord); 190 | } 191 | 192 | float m_p(vec2 coord, vec2 FCoord, vec2 BCoord) { 193 | vec3 color = getOri(coord), 194 | fColor = getOri(FCoord), 195 | bColor = getOri(BCoord); 196 | float alpha = estimAlpha(color, fColor, bColor); 197 | return length(color - alpha * fColor - (1.0 - alpha) * bColor); 198 | } 199 | 200 | float n_p(vec2 FCoord, vec2 BCoord) { 201 | float res = 0.0; 202 | for (int i = -1; i < 2; i ++) { 203 | for (int j = -1; j < 2; j ++) { 204 | float m = m_p((gl_FragCoord.xy + vec2(float(i), float(j)))/scale, FCoord, BCoord); 205 | res += m * m; 206 | } 207 | } 208 | 209 | return res; 210 | } 211 | 212 | float g_p(vec2 FCoord, vec2 BCoord, float pfp) { 213 | float np = pow(n_p(FCoord, BCoord), 3.0), 214 | ap = pow(a_p(FCoord, BCoord, pfp), 2.0), 215 | dpf = d_p(FCoord), 216 | dpb = pow(d_p(BCoord), 4.0); 217 | 218 | return np * ap * dpf * dpb; 219 | } 220 | 221 | FBPair gathering() { 222 | float gpMin = 1.0e10; 223 | float pfp = pf_p(); 224 | int idxI = 0, idxJ = 0; 225 | for (int i = 0; i < %%kG%%; i ++) { 226 | for (int j = 0; j < %%kG%%; j ++) { 227 | vec2 fs = arrIdx(0, i), 228 | bs = arrIdx(1, j); 229 | if (fs.x != -1.0 && bs.x != -1.0) { 230 | float gp = g_p(fs, bs, pfp); 231 | if (gp < gpMin) { 232 | gpMin = gp; 233 | idxI = i; 234 | idxJ = j; 235 | } 236 | } 237 | } 238 | } 239 | 240 | FBPair p; 241 | p.FCoord = arrIdx(0,idxI); p.BCoord = arrIdx(1,idxJ); 242 | p.F = getOri(p.FCoord); p.B = getOri(p.BCoord); 243 | 244 | return p; 245 | } 246 | 247 | void testAlpha() { 248 | if (getTri().x > 0.0 && getTri().x < 1.0) { 249 | sample(); 250 | FBPair p = gathering(); 251 | float alpha = estimAlpha(getOri(), p.F, p.B); 252 | // float alpha = estimAlpha(getOri(), getOri(FSamples[3]), getOri(BSamples[3])); 253 | gl_FragColor = vec4(alpha, alpha, alpha, 1.0); 254 | // gl_FragData[0] = vec4(p.F, 1.0); 255 | // gl_FragColor = vec4(alpha, alpha, alpha, 1.0); 256 | } 257 | else { 258 | gl_FragColor = vec4(getTri(), 1.0); 259 | // gl_FragData[0] = vec4(0.0, 0.0, 0.0, 1.0); 260 | // gl_FragColor = vec4(getTri(), 1.0); 261 | } 262 | } 263 | 264 | void writeFB() { 265 | if (getTri().x > 0.0 && getTri().x < 1.0) { 266 | vec2 FSamples[%%kG%%]; 267 | vec2 BSamples[%%kG%%]; 268 | sample(); 269 | FBPair p = gathering(); 270 | // F B component 271 | gl_FragColor = vec4(p.FCoord, p.BCoord); 272 | } 273 | else { 274 | gl_FragColor = vec4(1.0, 1.0, 1.0, 1.0); 275 | } 276 | } 277 | 278 | void main() { 279 | if (isTest == 0) { 280 | writeFB(); 281 | } 282 | else { 283 | testAlpha(); 284 | } 285 | } 286 | -------------------------------------------------------------------------------- /shaders/localSmooth.frag.glsl: -------------------------------------------------------------------------------- 1 | #ifdef GL_ES 2 | precision mediump float; 3 | #endif 4 | 5 | uniform sampler2D oriState; 6 | uniform sampler2D triState; 7 | uniform sampler2D foreState; 8 | uniform sampler2D backState; 9 | uniform sampler2D acState; 10 | uniform vec2 scale; 11 | 12 | vec3 getOri() { 13 | vec4 color = texture2D(oriState, (gl_FragCoord.xy) / scale); 14 | return color.xyz; 15 | } 16 | 17 | vec3 getOri(vec2 coord) { 18 | vec4 color = texture2D(oriState, coord); 19 | return color.xyz; 20 | } 21 | 22 | vec3 getTri() { 23 | vec4 color = texture2D(triState, (gl_FragCoord.xy) / scale); 24 | return color.xyz; 25 | } 26 | 27 | vec3 getTri(vec2 coord) { 28 | return texture2D(triState, coord).xyz; 29 | } 30 | 31 | float getAlpha(vec2 coord) { 32 | return texture2D(acState, coord).x; 33 | } 34 | 35 | float getConfidence(vec2 coord) { 36 | return texture2D(acState, coord).y; 37 | } 38 | 39 | vec3 getFore(vec2 coord) { 40 | return texture2D(foreState, coord).xyz; 41 | } 42 | 43 | vec3 getBack(vec2 coord) { 44 | return texture2D(backState, coord).xyz; 45 | } 46 | 47 | float estimAlpha(vec3 color, vec3 fColor, vec3 bColor) { 48 | float d = distance(fColor, bColor); 49 | return dot((color - bColor), (fColor - bColor)) / (d * d); 50 | } 51 | 52 | float m_p(vec2 coord, vec3 fColor, vec3 bColor) { 53 | vec3 color = getOri(coord); 54 | float alpha = estimAlpha(color, fColor, bColor); 55 | return length(color - alpha * fColor - (1.0 - alpha) * bColor); 56 | } 57 | 58 | float smooth() { 59 | float sigma2 = 100.0 / (9.0 * 3.1415926); 60 | vec3 accumWcUpF = vec3(0.0, 0.0, 0.0), accumWcUpB = vec3(0.0, 0.0, 0.0); 61 | float accumWcDownF = 0.0, accumWcDownB = 0.0; 62 | float accumWfbUp = 0.0, accumWfbDown = 0.0; 63 | float confidence00 = getConfidence(gl_FragCoord.xy / scale); 64 | float alpha00 = getAlpha(gl_FragCoord.xy / scale); 65 | float accumWaUp = 0.0, accumWaDown = 0.0; 66 | for (int i = -10; i < 11; i ++) { 67 | for (int j = -10; j < 11; j ++) { 68 | vec2 offset = vec2(float(i), float(j)); 69 | vec2 coord = (gl_FragCoord.xy + offset) / scale; 70 | float d = distance(offset, vec2(0.0, 0.0)); 71 | if (coord.x > 0.0 && coord.x < 1.0 72 | && coord.y > 0.0 && coord.y < 1.0 73 | && d <= 3.0 * sigma2) { 74 | // aqr 75 | float alpha = getAlpha(coord), 76 | // fqr 77 | confidence = getConfidence(coord); 78 | float g = exp(- d*d / 2.0 / sigma2); 79 | float wc = 0.0; 80 | if (i == 0 && j == 0) { 81 | wc = g * confidence; 82 | } 83 | else { 84 | wc = g * confidence * abs(alpha - alpha00); 85 | } 86 | vec3 foreColor = getFore(coord), 87 | backColor = getBack(coord); 88 | float wca = wc * alpha; 89 | accumWcUpF += wca * foreColor; 90 | accumWcUpB += (wc - wca) * backColor; 91 | accumWcDownF += wc * alpha; 92 | accumWcDownB += wc - wca; 93 | 94 | float wfbq = confidence * alpha * (1.0 - alpha); 95 | accumWfbUp += wfbq * distance(foreColor, backColor); 96 | accumWfbDown += wfbq; 97 | 98 | float theta = 0.0; 99 | if (getTri(coord).x == 0.0 || getTri(coord).x == 1.0) { 100 | theta = 1.0; 101 | } 102 | float wa = confidence * g + theta; 103 | accumWaUp += wa * alpha; 104 | accumWaDown += wa; 105 | } 106 | } 107 | } 108 | vec3 fp = accumWcUpF / (accumWcDownF + 1e-10), 109 | bp = accumWcUpB / (accumWcDownB + 1e-10); 110 | float dfb = accumWfbUp / (accumWfbDown + 1e-10); 111 | float confidenceP = min(1.0, distance(fp, bp) / dfb) * exp(-10.0 * m_p(gl_FragCoord.xy/scale, fp, bp)); 112 | float alphaP = accumWaUp / (accumWaDown + 1e-10); 113 | alphaP = max(0.0, min(1.0, alphaP)); 114 | float alphaFinal = confidenceP * estimAlpha(getOri(), fp, bp) + (1.0 - confidenceP) * alphaP; 115 | 116 | return alphaFinal; 117 | } 118 | 119 | void getAlpha() { 120 | if (getTri().x == 0.0 || getTri().x == 1.0) { 121 | gl_FragColor = vec4(getTri(), 1.0); 122 | } 123 | else { 124 | float alpha = smooth(); 125 | gl_FragColor = vec4(alpha, alpha, alpha, 1.0); 126 | // gl_FragColor = vec4(getBack(gl_FragCoord.xy / scale), 1.0); 127 | } 128 | // gl_FragColor = vec4(getBack(gl_FragCoord.xy / scale), 1.0); 129 | } 130 | 131 | void main() { 132 | getAlpha(); 133 | } 134 | -------------------------------------------------------------------------------- /shaders/quad.vert.glsl: -------------------------------------------------------------------------------- 1 | #ifdef GL_ES 2 | precision mediump float; 3 | #endif 4 | 5 | attribute vec2 quad; 6 | 7 | void main() { 8 | gl_Position = vec4(quad, 0, 1.0); 9 | } 10 | -------------------------------------------------------------------------------- /shaders/refinement.frag.glsl: -------------------------------------------------------------------------------- 1 | #extension GL_EXT_draw_buffers : require 2 | #ifdef GL_ES 3 | precision mediump float; 4 | #endif 5 | 6 | uniform sampler2D oriState; 7 | uniform sampler2D triState; 8 | uniform sampler2D fbState; 9 | uniform sampler2D fbSigmaState; 10 | uniform vec2 scale; 11 | uniform int isTest; 12 | 13 | struct point { 14 | float x; 15 | float y; 16 | float mp; 17 | }; 18 | 19 | struct rTuple { 20 | vec3 fColor; 21 | vec3 bColor; 22 | float alpha; 23 | float confidence; 24 | }; 25 | 26 | vec3 getOri() { 27 | vec4 color = texture2D(oriState, (gl_FragCoord.xy) / scale); 28 | return color.xyz; 29 | } 30 | 31 | vec3 getOri(vec2 coord) { 32 | vec4 color = texture2D(oriState, coord); 33 | return color.xyz; 34 | } 35 | 36 | vec3 getTri() { 37 | vec4 color = texture2D(triState, (gl_FragCoord.xy) / scale); 38 | return color.xyz; 39 | } 40 | 41 | vec3 getTri(vec2 coord) { 42 | return texture2D(triState, coord).xyz; 43 | } 44 | 45 | vec2 getFCoord() { 46 | return texture2D(fbState, (gl_FragCoord.xy) / scale).xy; 47 | } 48 | 49 | vec2 getBCoord() { 50 | return texture2D(fbState, (gl_FragCoord.xy) / scale).zw; 51 | } 52 | 53 | vec2 getFCoord(vec2 coord) { 54 | return texture2D(fbState, coord).xy; 55 | } 56 | 57 | vec2 getBCoord(vec2 coord) { 58 | return texture2D(fbState, coord).zw; 59 | } 60 | 61 | float getFSigma() { 62 | return texture2D(fbSigmaState, (gl_FragCoord.xy) / scale).x; 63 | } 64 | 65 | float getBSigma() { 66 | return texture2D(fbSigmaState, (gl_FragCoord.xy) / scale).y; 67 | } 68 | 69 | float getFSigma(vec2 coord) { 70 | return texture2D(fbSigmaState, coord).x; 71 | } 72 | 73 | float getBSigma(vec2 coord) { 74 | return texture2D(fbSigmaState, coord).y; 75 | } 76 | 77 | float estimAlpha(vec3 color, vec3 fColor, vec3 bColor) { 78 | float d = distance(fColor, bColor); 79 | return dot((color - bColor), (fColor - bColor)) / (d * d); 80 | } 81 | 82 | float m_p(vec2 coord, vec3 fColor, vec3 bColor) { 83 | vec3 color = getOri(coord); 84 | float alpha = estimAlpha(color, fColor, bColor); 85 | return length(color - alpha * fColor - (1.0 - alpha) * bColor); 86 | } 87 | 88 | float m_p(vec3 color, vec3 fColor, vec3 bColor) { 89 | float alpha = estimAlpha(color, fColor, bColor); 90 | return length(color - alpha * fColor - (1.0 - alpha) * bColor); 91 | } 92 | 93 | float sigma2(vec2 coord) { 94 | float res = 0.0; 95 | int neiCount = 0; 96 | vec3 color = getOri(coord); 97 | for (int i = -2; i < 3; i ++) { 98 | for (int j = -2; j < 3; j ++) { 99 | float xn = coord.x + (float(i) / scale.x), 100 | yn = coord.y + (float(j) / scale.y); 101 | if (xn > 0.0 && xn < 1.0 102 | && yn > 0.0 && yn < 1.0) { 103 | vec3 colorn = getOri(vec2(xn, yn)); 104 | float d = distance(colorn, color); 105 | res += d * d; 106 | neiCount ++; 107 | } 108 | } 109 | } 110 | return res / (float(neiCount) + 1e-10); 111 | } 112 | 113 | rTuple refine() { 114 | int count = 0; 115 | point minMp0, minMp1, minMp2; 116 | minMp0.mp = 1e10; minMp1.mp = 1e10; minMp2.mp = 1e10; 117 | for (int i = -5; i < 6; i ++) { 118 | for (int j = -5; j < 6; j ++) { 119 | float x = gl_FragCoord.x + float(i), 120 | y = gl_FragCoord.y + float(j); 121 | vec2 coord = vec2(x, y) / scale; 122 | 123 | if (x > 0.0 && x < scale.x 124 | && y > 0.0 && y < scale.y 125 | && getTri(coord).x > 0.0 && getTri(coord).x < 1.0) { 126 | 127 | vec3 fColor = getOri(getFCoord(coord)); 128 | vec3 bColor = getOri(getBCoord(coord)); 129 | float mp = m_p(coord, fColor, bColor); 130 | if (mp < minMp0.mp) { 131 | minMp2 = minMp1; 132 | minMp1 = minMp0; 133 | // minMp2.x = minMp1.x; 134 | // minMp2.y = minMp1.y; 135 | // minMp2.mp = minMp1.mp; 136 | // minMp1.x = minMp0.x; 137 | // minMp1.y = minMp0.y; 138 | // minMp1.mp = minMp0.mp; 139 | 140 | minMp0.x = coord.x; 141 | minMp0.y = coord.y; 142 | minMp0.mp = mp; 143 | count ++; 144 | } 145 | else if (mp < minMp1.mp) { 146 | minMp2 = minMp1; 147 | // minMp2.x = minMp1.x; 148 | // minMp2.y = minMp1.y; 149 | // minMp2.mp = minMp1.mp; 150 | 151 | minMp1.x = coord.x; 152 | minMp1.y = coord.y; 153 | minMp1.mp = mp; 154 | count ++; 155 | } 156 | else if (mp < minMp2.mp) { 157 | minMp2.x = coord.x; 158 | minMp2.y = coord.y; 159 | minMp2.mp = mp; 160 | count ++; 161 | } 162 | } 163 | } 164 | } 165 | 166 | // count must > 0 because self is included 167 | count = count > 3 ? 3 : count; 168 | 169 | vec2 coord = vec2(minMp0.x, minMp0.y); 170 | vec2 fCoord = getFCoord(coord), 171 | bCoord = getBCoord(coord); 172 | vec3 fg = getOri(fCoord), 173 | bg = getOri(bCoord); 174 | float sf = sigma2(fCoord), 175 | sb = sigma2(bCoord); 176 | 177 | if (count > 1) { 178 | coord = vec2(minMp1.x, minMp1.y); 179 | fCoord = getFCoord(coord); 180 | bCoord = getBCoord(coord); 181 | fg += getOri(fCoord); 182 | bg += getOri(bCoord); 183 | sf += sigma2(fCoord); 184 | sb += sigma2(bCoord); 185 | } 186 | if (count > 2) { 187 | coord = vec2(minMp2.x, minMp2.y); 188 | fCoord = getFCoord(coord); 189 | bCoord = getBCoord(coord); 190 | fg += getOri(fCoord); 191 | bg += getOri(bCoord); 192 | sf += sigma2(fCoord); 193 | sb += sigma2(bCoord); 194 | } 195 | fg = fg / float(count); 196 | bg = bg / float(count); 197 | sf = sf / float(count); 198 | sb = sb / float(count); 199 | 200 | vec3 color = getOri(); 201 | rTuple r; 202 | float d = distance(color, fg); 203 | float df = d * d; 204 | d = distance(color, bg); 205 | float db = d * d; 206 | 207 | if (df < sf) { r.fColor = color; } 208 | else { r.fColor = fg; } 209 | if (db < sb) { r.bColor = color; } 210 | else { r.bColor = bg; } 211 | 212 | if (r.fColor != r.bColor) { r.confidence = exp(-10.0 * m_p(color, fg, bg)); } 213 | else { r.confidence = 1e-8; } 214 | r.alpha = max(0.0, min(1.0, estimAlpha(color, r.fColor, r.bColor))); 215 | // r.alpha = estimAlpha(color, fg, bg); 216 | // r.alpha = estimAlpha(getOri(), getOri(getFCoord()), getOri(getBCoord())); 217 | 218 | return r; 219 | } 220 | 221 | void writeFB() { 222 | float triColor = getTri().x; 223 | if (triColor == 0.0 || triColor == 1.0) { 224 | // fore color texture 225 | gl_FragData[0] = vec4(getOri(), 1.0); 226 | // back color texture 227 | gl_FragData[1] = vec4(getOri(), 1.0); 228 | // alpha and confidence 229 | gl_FragData[2] = vec4(triColor, 1.0, 1.0, 1.0); 230 | } 231 | else { 232 | rTuple r = refine(); 233 | // fore color texture 234 | gl_FragData[0] = vec4(r.fColor, 1.0); 235 | // back color texture 236 | gl_FragData[1] = vec4(r.bColor, 1.0); 237 | // alpha and confidence 238 | gl_FragData[2] = vec4(r.alpha, r.confidence, 1.0, 1.0); 239 | } 240 | } 241 | 242 | void testAlpha() { 243 | float triColor = getTri().x; 244 | if (triColor == 0.0 || triColor == 1.0) { 245 | gl_FragData[0] = vec4(getTri(), 1.0); 246 | } 247 | else { 248 | rTuple r = refine(); 249 | // gl_FragData[0] = vec4(getOri(getBCoord()), 1.0); 250 | // gl_FragData[0] = vec4(r.fColor, 1.0); 251 | // float s = sigma2(getBCoord()); 252 | // gl_FragData[0] = vec4(s, s, s, 1.0); 253 | gl_FragData[0] = vec4(r.alpha, r.alpha, r.alpha, 1.0); 254 | } 255 | } 256 | 257 | void main() { 258 | if (isTest == 0) { 259 | writeFB(); 260 | } 261 | else { 262 | testAlpha(); 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /shared.js: -------------------------------------------------------------------------------- 1 | import Igloo from 'igloo-ext'; 2 | 3 | import quadSL from './shaders/quad.vert.glsl'; 4 | import drawSL from './shaders/drawCanvas.frag.glsl'; 5 | import expandKnownSL from './shaders/expandKnown.frag.glsl'; 6 | import gatheringSL from './shaders/gathering.frag.glsl'; 7 | import refinementSL from './shaders/refinement.frag.glsl'; 8 | import localSmoothSL from './shaders/localSmooth.frag.glsl'; 9 | 10 | export default class Shared { 11 | constructor(canvas) { 12 | this.viewSize = new Float32Array([canvas.width, canvas.height]); 13 | this.igloo = new Igloo(canvas); 14 | let gl = this.gl = this.igloo.gl; 15 | let ext = this.ext = gl.getExtension('WEBGL_draw_buffers'); 16 | if (!gl) { 17 | alert('Could not initialize WebGL!'); 18 | throw new Error('No WebGL'); 19 | } 20 | if (!ext) { 21 | alert('Could not initialize WebGL Ext!'); 22 | throw new Error('No WEBGL_draw_buffers'); 23 | } 24 | gl.disable(gl.DEPTH_TEST); 25 | 26 | this.textures = this.textures || {}; 27 | } 28 | setImage(oriUrl) { 29 | this.loadImage(oriUrl, (img) => { 30 | let gl = this.gl; 31 | this.textures.ori = this.igloo.texture(img, gl.RGBA, gl.CLAMP_TO_EDGE, gl.NEAREST) 32 | this.stateSize = new Float32Array([img.naturalWidth, img.naturalHeight]); 33 | let maxPixels = 640000; 34 | if (this.stateSize[0] * this.stateSize[1] > maxPixels) { 35 | let ratio = Math.sqrt(maxPixels / (this.stateSize[0] * this.stateSize[1]) ); 36 | this.stateSize[0] = ratio * this.stateSize[0]; 37 | this.stateSize[1] = ratio * this.stateSize[1]; 38 | } 39 | }) 40 | } 41 | setTrimap(triUrl) { 42 | this.loadImage(triUrl, (img) => { 43 | let gl = this.gl; 44 | this.textures.tri = this.igloo.texture(img, gl.RGBA, gl.CLAMP_TO_EDGE, gl.NEAREST); 45 | }) 46 | } 47 | glslSetup() { 48 | let igloo = this.igloo, 49 | gl = this.gl; 50 | 51 | this.framebuffers = { 52 | back: igloo.framebuffer(), 53 | refine: igloo.framebuffer(), 54 | }; 55 | this.params = { 56 | kI: 10.0, 57 | kC: 5.0 / 255.0, 58 | kG: 4, 59 | searchRange: 500.0 60 | } 61 | 62 | this.programs = { 63 | copy: igloo.program(quadSL, drawSL), 64 | expandKnown: igloo.program(quadSL, expandKnownSL, 65 | this.replacer({kI: this.params.kI})), 66 | gathering: igloo.program(quadSL, gatheringSL, 67 | this.replacer({kG: this.params.kG, searchRange: this.params.searchRange})), 68 | refinement: igloo.program(quadSL, refinementSL), 69 | localSmooth: igloo.program(quadSL, localSmoothSL) 70 | }; 71 | this.buffers = { 72 | quad: igloo.array(Igloo.QUAD2) 73 | }; 74 | this.textures = { 75 | ...this.textures, 76 | triExpand: this.igloo.texture(null, gl.RGBA, gl.CLAMP_TO_EDGE, gl.NEAREST) 77 | .blank(this.stateSize[0], this.stateSize[1]), 78 | gatherFB: this.igloo.texture(null, gl.RGBA, gl.CLAMP_TO_EDGE, gl.NEAREST) 79 | .blank(this.stateSize[0], this.stateSize[1]), 80 | gatherFBSigma: this.igloo.texture(null, gl.RGBA, gl.CLAMP_TO_EDGE, gl.NEAREST) 81 | .blank(this.stateSize[0], this.stateSize[1]), 82 | refineF: this.igloo.texture(null, gl.RGBA, gl.CLAMP_TO_EDGE, gl.NEAREST) 83 | .blank(this.stateSize[0], this.stateSize[1]), 84 | refineB: this.igloo.texture(null, gl.RGBA, gl.CLAMP_TO_EDGE, gl.NEAREST) 85 | .blank(this.stateSize[0], this.stateSize[1]), 86 | refineAC: this.igloo.texture(null, gl.RGBA, gl.CLAMP_TO_EDGE, gl.NEAREST) 87 | .blank(this.stateSize[0], this.stateSize[1]), 88 | alpha: this.igloo.texture(null, gl.RGBA, gl.CLAMP_TO_EDGE, gl.NEAREST) 89 | .blank(this.stateSize[0], this.stateSize[1]) 90 | } 91 | } 92 | run(canvasScale) { 93 | console.log('run shared matting') 94 | if (!this.textures.ori || !this.textures.tri) return; 95 | 96 | var t1 = new Date(); 97 | this.glslSetup(); 98 | var t2 = new Date(); 99 | this 100 | .expandKnown() 101 | .gathering() 102 | .refinement() 103 | .localSmooth() 104 | .draw(canvasScale); 105 | var t3 = new Date(); 106 | console.log(`Setup time: ${t2.getTime() - t1.getTime()}`) 107 | console.log(`Render time: ${t3.getTime() - t2.getTime()}`) 108 | } 109 | expandKnown() { 110 | let gl = this.gl; 111 | this.framebuffers.back.attach(this.textures.triExpand); 112 | this.textures.ori.bind(0); 113 | this.textures.tri.bind(1); 114 | // this.textures.triTest.bind(1); 115 | gl.viewport(0, 0, this.stateSize[0], this.stateSize[1]); 116 | 117 | this.programs.expandKnown.use() 118 | .attrib('quad', this.buffers.quad, 2) 119 | .uniformi('oriState', 0) 120 | .uniformi('triState', 1) 121 | .uniform('kC', this.params.kC) 122 | .uniform('scale', this.stateSize) 123 | .draw(gl.TRIANGLE_STRIP, 4); 124 | 125 | return this; 126 | } 127 | gathering(mode) { 128 | let gl = this.gl; 129 | 130 | if (mode == 'test') { 131 | this.framebuffers.back.attach(this.textures.alpha); 132 | } 133 | else { 134 | this.framebuffers.back.attach(this.textures.gatherFB); 135 | } 136 | if (gl.checkFramebufferStatus(gl.FRAMEBUFFER) !== gl.FRAMEBUFFER_COMPLETE) { 137 | alert('Could not initialize Buffer array!'); 138 | throw new Error('No WebGL'); 139 | } 140 | 141 | this.textures.ori.bind(0); 142 | // this.textures.triExpand.bind(1); 143 | this.textures.triExpand.bind(1); 144 | gl.viewport(0, 0, this.stateSize[0], this.stateSize[1]); 145 | 146 | this.programs.gathering.use() 147 | .attrib('quad', this.buffers.quad, 2) 148 | .uniformi('oriState', 0) 149 | .uniformi('triState', 1) 150 | .uniformi('isTest', mode == 'test' ? 1 : 0) 151 | .uniformi('kG', this.params.kG) 152 | .uniform('scale', this.stateSize) 153 | .draw(gl.TRIANGLE_STRIP, 4); 154 | 155 | return this; 156 | } 157 | refinement(mode) { 158 | let gl = this.gl; 159 | let arr = []; 160 | if (mode == 'test') { arr = [this.textures.alpha]; } 161 | else { arr = [this.textures.refineF, this.textures.refineB, this.textures.refineAC]; } 162 | 163 | this.framebuffers.refine.attachArr(arr); 164 | if (gl.checkFramebufferStatus(gl.FRAMEBUFFER) !== gl.FRAMEBUFFER_COMPLETE) { 165 | alert('Could not initialize Buffer array!'); 166 | throw new Error('No WebGL'); 167 | } 168 | this.textures.ori.bind(0); 169 | // this.textures.triExpand.bind(1); 170 | this.textures.triExpand.bind(1); 171 | this.textures.gatherFB.bind(2); 172 | this.textures.gatherFBSigma.bind(3); 173 | gl.viewport(0, 0, this.stateSize[0], this.stateSize[1]); 174 | 175 | this.programs.refinement.use() 176 | .attrib('quad', this.buffers.quad, 2) 177 | .uniformi('oriState', 0) 178 | .uniformi('triState', 1) 179 | .uniformi('fbState', 2) 180 | .uniformi('fbSigmaState', 3) 181 | .uniformi('isTest', mode == 'test' ? 1 : 0) 182 | .uniform('scale', this.stateSize) 183 | .draw(gl.TRIANGLE_STRIP, 4); 184 | 185 | return this; 186 | } 187 | localSmooth() { 188 | let gl = this.gl; 189 | this.framebuffers.back.attach(this.textures.alpha); 190 | 191 | this.textures.ori.bind(0); 192 | // this.textures.triExpand.bind(1); 193 | this.textures.triExpand.bind(1); 194 | this.textures.refineAC.bind(2); 195 | this.textures.refineF.bind(3); 196 | this.textures.refineB.bind(4); 197 | gl.viewport(0, 0, this.stateSize[0], this.stateSize[1]); 198 | 199 | this.programs.localSmooth.use() 200 | .attrib('quad', this.buffers.quad, 2) 201 | .uniformi('oriState', 0) 202 | .uniformi('triState', 1) 203 | .uniformi('acState', 2) 204 | .uniformi('foreState', 3) 205 | .uniformi('backState', 4) 206 | .uniform('scale', this.stateSize) 207 | .draw(gl.TRIANGLE_STRIP, 4); 208 | 209 | return this; 210 | } 211 | draw(canvasScale) { 212 | let gl = this.gl; 213 | this.igloo.defaultFramebuffer.bind(); 214 | this.textures.alpha.bind(0); 215 | gl.viewport(0, 0, this.viewSize[0], this.viewSize[1]); 216 | 217 | let offset = new Float32Array([canvasScale.ol, canvasScale.ot]); 218 | this.programs.copy.use() 219 | .attrib('quad', this.buffers.quad, 2) 220 | .uniformi('state', 0) 221 | .uniform('scale', this.viewSize) 222 | .uniform('offset', offset) 223 | .draw(gl.TRIANGLE_STRIP, 4); 224 | 225 | return this; 226 | } 227 | 228 | replacer(map) { 229 | var keys = Object.keys(map); 230 | return function(source) { 231 | for (var i = 0; i < keys.length; i++) { 232 | var key = keys[i], regex = new RegExp('%%' + key + '%%', 'g'); 233 | source = source.replace(regex, map[key]); 234 | } 235 | return source; 236 | }; 237 | } 238 | 239 | loadImage(image, callback) { 240 | let img = image; 241 | if (typeof img === 'string') { 242 | img = new Image(); 243 | img.src = image; 244 | } 245 | img.onload = () => { 246 | img.style.display = 'none'; 247 | if (callback) { 248 | callback(img); 249 | } 250 | } 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /utils.js: -------------------------------------------------------------------------------- 1 | function getScale(cw, ch, iw, ih) { 2 | let ot, ol, iw_, ih_, ratio; 3 | if (iw / ih > cw / ch) { 4 | ratio = cw / iw; 5 | iw_ = iw * ratio; 6 | ih_ = ih * ratio; 7 | ot = (ch - ih_) / 2; 8 | ol = 0; 9 | } 10 | else { 11 | ratio = ch / ih; 12 | iw_ = iw * ratio; 13 | ih_ = ih * ratio; 14 | ot = 0; 15 | ol = (cw - iw_) / 2; 16 | } 17 | return { 18 | ot, 19 | ol, 20 | iw: iw_, 21 | ih: ih_, 22 | } 23 | } 24 | 25 | export {getScale}; 26 | --------------------------------------------------------------------------------