├── .eslintignore ├── Procfile ├── .gitignore ├── server ├── .eslintrc.js └── index.js ├── src ├── .eslintrc.js ├── images │ ├── sudoku-1.png │ └── sudoku-2.png ├── styles.css ├── index.html ├── hello.cpp └── index.js ├── test ├── .eslintrc.js ├── tests.html └── tests.js ├── .eslintrc.js ├── webpack.config.js ├── package.json ├── CMakeLists.txt └── README.md /.eslintignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | build/ -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: node server 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules/ 3 | build/ 4 | dist/ 5 | -------------------------------------------------------------------------------- /server/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | node: true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | browser: true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/images/sudoku-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/taylorjg/emscripten-opencv/HEAD/src/images/sudoku-1.png -------------------------------------------------------------------------------- /src/images/sudoku-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/taylorjg/emscripten-opencv/HEAD/src/images/sudoku-2.png -------------------------------------------------------------------------------- /test/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | node: true, 4 | browser: true, 5 | mocha: true 6 | }, 7 | globals: { 8 | chai: 'readonly' 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: 'eslint:recommended', 3 | env: { 4 | es6: true 5 | }, 6 | parserOptions: { 7 | ecmaVersion: 2018, 8 | sourceType: 'module' 9 | }, 10 | rules: { 11 | 'no-console': 'off' 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /server/index.js: -------------------------------------------------------------------------------- 1 | const express = require('express') 2 | const path = require('path') 3 | 4 | const PORT = process.env.PORT || 3092 5 | const DIST_FOLDER = path.resolve(__dirname, '..', 'dist') 6 | 7 | const app = express() 8 | app.use(express.static(DIST_FOLDER)) 9 | app.listen(PORT, () => console.log(`Listening on http://localhost:${PORT}`)) 10 | -------------------------------------------------------------------------------- /src/styles.css: -------------------------------------------------------------------------------- 1 | .container { 2 | margin-top: 20px; 3 | } 4 | 5 | .version { 6 | font-size: small; 7 | font-style: italic; 8 | } 9 | 10 | .row-gap { 11 | margin-top: 5px; 12 | } 13 | 14 | .image-wrapper { 15 | position: relative; 16 | } 17 | 18 | .image-overlay { 19 | position: absolute; 20 | top: 0; 21 | left: 0; 22 | } 23 | 24 | .cell { 25 | margin: 2px; 26 | } 27 | -------------------------------------------------------------------------------- /test/tests.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Tests 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |
15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | /* eslint-env node */ 2 | 3 | const CopyWebpackPlugin = require('copy-webpack-plugin') 4 | const HtmlWebpackPlugin = require('html-webpack-plugin') 5 | const path = require('path') 6 | const { version } = require('./package.json') 7 | 8 | const DIST_FOLDER = path.join(__dirname, 'dist') 9 | 10 | module.exports = { 11 | mode: 'production', 12 | entry: './src/index.js', 13 | output: { 14 | path: DIST_FOLDER, 15 | filename: 'bundle.js', 16 | }, 17 | plugins: [ 18 | new CopyWebpackPlugin({ 19 | patterns: [ 20 | { context: './src', from: '*.css' }, 21 | { context: './src', from: 'images/*.png' }, 22 | { context: './build', from: 'hello.js' }, 23 | { context: './build', from: 'hello.wasm' }, 24 | { context: './test', from: 'tests.html' }, 25 | { context: './test', from: 'tests.js' } 26 | ] 27 | }), 28 | new HtmlWebpackPlugin({ 29 | template: './src/index.html', 30 | inject: false, 31 | version 32 | }) 33 | ], 34 | devtool: 'source-map', 35 | devServer: { 36 | contentBase: DIST_FOLDER 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "emscripten-opencv", 3 | "version": "0.0.18", 4 | "description": "Web app calling into C++ WebAssembly code that uses OpenCV", 5 | "scripts": { 6 | "eslint": "eslint .", 7 | "build:opencv": "docker run --rm -t -w /src -v \"$PWD\"/../opencv:/src/opencv emscripten/emsdk sh -c 'EMSCRIPTEN=/emsdk/upstream/emscripten python3 opencv/platforms/js/build_js.py opencv/build_wasm --build_wasm'", 8 | "build:wasm": "docker run --rm -t -w /src -v \"$PWD\"/../opencv:/src/opencv -v \"$PWD\":/src/emscripten-opencv emscripten/emsdk sh emscripten-opencv/build-wasm.sh", 9 | "build": "npm run build:wasm && webpack", 10 | "start": "node server", 11 | "start:dev": "npm run build:wasm && webpack-dev-server", 12 | "heroku-postbuild": "echo use contents of dist folder", 13 | "test": "mocha --experimental-wasm-threads --no-warnings" 14 | }, 15 | "dependencies": { 16 | "express": "^4.17.1" 17 | }, 18 | "devDependencies": { 19 | "canvas": "^2.8.0", 20 | "chai": "^4.3.4", 21 | "chai-almost": "^1.0.1", 22 | "copy-webpack-plugin": "^9.0.0", 23 | "eslint": "^7.27.0", 24 | "html-webpack-plugin": "^5.3.1", 25 | "mocha": "^8.4.0", 26 | "webpack": "^5.38.1", 27 | "webpack-cli": "^4.7.0", 28 | "webpack-dev-server": "^3.11.2" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(OPENCV_DIR "../opencv") 2 | 3 | cmake_minimum_required(VERSION 3.1) 4 | project(HelloCV) 5 | 6 | # Use C++ 11 by default 7 | set(CMAKE_CXX_STANDARD 11) 8 | 9 | # Set Release as default build type 10 | if(NOT CMAKE_BUILD_TYPE) 11 | set(CMAKE_BUILD_TYPE Release) 12 | endif(NOT CMAKE_BUILD_TYPE) 13 | 14 | # Does not work 15 | # find_package(OpenCV REQUIRED PATHS "${OPENCV_DIR}/build_wasm" NO_DEFAULT_PATH) 16 | 17 | # Needed for opencv2/opencv.hpp 18 | include_directories("${OPENCV_DIR}/include") 19 | 20 | # Needed by opencv.hpp for opencv2/opencv_modules.hpp 21 | include_directories("${OPENCV_DIR}/build_wasm") 22 | 23 | # Needed by opencv_modules.hpp for every module 24 | file(GLOB opencv_include_modules "${OPENCV_DIR}/modules/*/include") 25 | include_directories(${opencv_include_modules}) 26 | 27 | # Our hello world executable 28 | add_executable(hello src/hello.cpp) 29 | 30 | # Link to opencv.js precompiled libraries 31 | file(GLOB opencv_libs "${OPENCV_DIR}/build_wasm/lib/*.a") 32 | target_link_libraries(hello ${opencv_libs}) 33 | 34 | # There is an issue regarding the order in which libraries 35 | # are passed to the compiler - pass libopencv_core.a last 36 | # https://answers.opencv.org/question/186124/undefined-reference-to-cvsoftdoubleoperator/ 37 | file(GLOB opencv_lib_core "${OPENCV_DIR}/build_wasm/lib/libopencv_core.a") 38 | target_link_libraries(hello ${opencv_lib_core}) 39 | 40 | # Specify linker arguments 41 | set_target_properties(hello PROPERTIES LINK_FLAGS "-s EXPORTED_RUNTIME_METHODS=cwrap -s EXPORTED_FUNCTIONS=_free -s MODULARIZE=1 -s EXPORT_NAME=createHelloModule") 42 | -------------------------------------------------------------------------------- /src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | emscripten-opencv 8 | 9 | 10 | 15 | 16 | 17 | 18 |
19 |
20 |
21 |
(version: <%= htmlWebpackPlugin.options.version %>)
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 | 33 | 34 |
35 |
36 |
37 |
38 | 39 | 40 |
41 |
42 |
43 |
44 | 45 | 46 |
47 |
48 |
49 |
50 |
51 | 52 | 53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
Elapsed time:
64 |
65 |
66 |
67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /test/tests.js: -------------------------------------------------------------------------------- 1 | let expect 2 | let nodeCanvas 3 | let helloModule 4 | 5 | const IS_NODE = typeof process === 'object' && typeof require === 'function' 6 | 7 | const main = async () => { 8 | if (IS_NODE) { 9 | expect = require('chai').expect 10 | nodeCanvas = require('canvas') 11 | } else { 12 | expect = chai.expect 13 | mocha.setup('bdd') 14 | helloModule = await window.createHelloModule() 15 | mocha.run() 16 | } 17 | } 18 | 19 | main() 20 | 21 | const getImageDataNode = async () => { 22 | const fs = require('fs').promises 23 | const png = await fs.readFile('src/images/sudoku-1.png') 24 | return new Promise(resolve => { 25 | const img = new nodeCanvas.Image() 26 | img.onload = () => { 27 | const canvas = nodeCanvas.createCanvas(img.width, img.height) 28 | const ctx = canvas.getContext('2d') 29 | ctx.drawImage(img, 0, 0) 30 | const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height) 31 | resolve(imageData) 32 | } 33 | img.src = png 34 | }) 35 | } 36 | 37 | const getImageDataBrowser = async () => 38 | new Promise(resolve => { 39 | const img = new Image() 40 | img.onload = () => { 41 | const canvas = document.createElement('canvas') 42 | canvas.width = img.width 43 | canvas.height = img.height 44 | const ctx = canvas.getContext('2d') 45 | ctx.drawImage(img, 0, 0) 46 | const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height) 47 | resolve(imageData) 48 | } 49 | img.src = '/images/sudoku-1.png' 50 | }) 51 | 52 | const getImageData = () => 53 | IS_NODE ? getImageDataNode() : getImageDataBrowser() 54 | 55 | describe('tests', () => { 56 | 57 | before(async () => { 58 | if (IS_NODE) { 59 | const createHelloModule = require('../build/hello.js') 60 | helloModule = await createHelloModule() 61 | } 62 | }) 63 | 64 | const expectWithinTolerance = (actual, expected) => { 65 | const TOLERANCE = 1 66 | const lowerBound = expected - TOLERANCE 67 | const upperBound = expected + TOLERANCE 68 | expect(actual).to.be.within(lowerBound, upperBound) 69 | } 70 | 71 | it('processImage', async () => { 72 | 73 | const ident = 'processImage' 74 | const returnType = 'number' 75 | const argTypes = ['array', 'number', 'number'] 76 | const processImage = helloModule.cwrap(ident, returnType, argTypes) 77 | 78 | const imageData = await getImageData() 79 | const { data, width, height } = imageData 80 | 81 | const addr = processImage(data, width, height) 82 | 83 | try { 84 | const addr32 = addr / helloModule.HEAP32.BYTES_PER_ELEMENT 85 | const data32 = helloModule.HEAP32.slice(addr32, addr32 + 22) 86 | 87 | const [bbx, bby, bbw, bbh, imgw, imgh, imgd] = data32 88 | 89 | expectWithinTolerance(bbx, 20) 90 | expectWithinTolerance(bby, 30) 91 | expectWithinTolerance(bbw, 185) 92 | expectWithinTolerance(bbh, 185) 93 | 94 | expect(imgw).to.equal(224) 95 | expect(imgh).to.equal(224) 96 | expect(imgd).to.equal(1) 97 | } finally { 98 | helloModule._free(addr) 99 | } 100 | }) 101 | }) 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Description 2 | 3 | > I meant to fork https://github.com/mpizenberg/emscripten-opencv.git but I cloned it instead and I don't know how to correct it. 4 | 5 | I have another repo ([sudoku-buster](https://github.com/taylorjg/sudoku-buster)) that uses OpenCV.js 6 | to detect the bounding box of a Sudoku puzzle given an image from a newspaper or similar. It then 7 | goes on to solve the puzzle and show the solution. 8 | 9 | It works pretty well but the OpenCV.js file is very big. I have managed to get it down to about 4 MB from 8 MB by 10 | excluding some bits that I am not using. However, it occurred to me that if I could move the bounding box code 11 | from JavaScript to C++ and then build it as a WebAssembly, the overall result might be smaller. 12 | 13 | > **UPDATE:** This isn't the final version of the code but results so far are encouraging - my 14 | current C++ WebAssembly is about 1 MB in size. 15 | 16 | # Setup 17 | 18 | ## Directory Structure 19 | 20 | These instructions assume the following directory structure: 21 | 22 | ``` 23 | some-root 24 | ├── emscripten-opencv 25 | └── opencv 26 | ``` 27 | 28 | `` could be any directory. 29 | The key point is that the `opencv` and `emscripten-opencv` directories are at the same level. 30 | 31 | Clone `opencv` from https://github.com/opencv/opencv.git. 32 | You may want to checkout to a particular tag e.g. 33 | 34 | ``` 35 | cd /opencv 36 | git checkout 4.1.2 37 | ``` 38 | 39 | ## Ensure Docker is installed 40 | 41 | ~~I use the [trzeci/emscripten](https://hub.docker.com/r/trzeci/emscripten/) Docker image to build WebAssemblies.~~ 42 | 43 | According to https://hub.docker.com/r/trzeci/emscripten/: 44 | 45 | > THIS DOCKER IMAGE IS TO BE DEPRECATED 46 | > 47 | > In favour of the official image: https://hub.docker.com/repository/r/emscripten/emsdk Please investigate transition to those images. In case of missing features or bugs, please report them to https://github.com/emscripten-core/emsdk/ 48 | 49 | Hence, I am now using [emscripten/emsdk](https://hub.docker.com/r/emscripten/emsdk). 50 | 51 | ## Build the OpenCV WebAssembly Libraries 52 | 53 | You only need to do this once. 54 | 55 | ``` 56 | cd /emscripten-opencv 57 | npm run build:opencv 58 | ``` 59 | 60 | This builds the OpenCV WebAssembly libraries which you will find in `../opencv/build_wasm/lib` 61 | when it has finished building: 62 | 63 | ``` 64 | $ ls -l ../opencv/build_wasm/lib 65 | 66 | total 33208 67 | -rw-r--r-- 1 jontaylor staff 2153518 29 May 08:02 libopencv_calib3d.a 68 | -rw-r--r-- 1 jontaylor staff 2434882 29 May 08:00 libopencv_core.a 69 | -rw-r--r-- 1 jontaylor staff 5858586 29 May 08:03 libopencv_dnn.a 70 | -rw-r--r-- 1 jontaylor staff 665434 29 May 08:01 libopencv_features2d.a 71 | -rw-r--r-- 1 jontaylor staff 549828 29 May 08:00 libopencv_flann.a 72 | -rw-r--r-- 1 jontaylor staff 3836412 29 May 08:01 libopencv_imgproc.a 73 | -rw-r--r-- 1 jontaylor staff 455288 29 May 08:03 libopencv_objdetect.a 74 | -rw-r--r-- 1 jontaylor staff 621346 29 May 08:01 libopencv_photo.a 75 | -rw-r--r-- 1 jontaylor staff 407924 29 May 08:03 libopencv_video.a 76 | ``` 77 | 78 | ## Build this project's WebAssembly 79 | 80 | Run this whenever a change is made to this project's C++ source files. 81 | This will build: 82 | 83 | * `/emscripten-opencv/build/hello.js` 84 | * `/emscripten-opencv/build/hello.wasm` 85 | 86 | ``` 87 | cd /emscripten-opencv 88 | npm run build:wasm 89 | ``` 90 | 91 | # Running 92 | 93 | The following sections assume that you have already cloned and built OpenCV as described above. 94 | 95 | ## Running a local server 96 | 97 | This builds this repo's WebAssembly and bundles everything using `webpack` and then launches 98 | an Express-based web server: 99 | 100 | ``` 101 | cd /emscripten-opencv 102 | npm run build 103 | npm start 104 | ``` 105 | 106 | To run on a specific port e.g. 3434: 107 | 108 | ``` 109 | PORT=3434 npm start 110 | ``` 111 | 112 | ## Running a local server in dev mode 113 | 114 | This builds this repo's WebAssembly then launches `webpack-dev-server`: 115 | 116 | ``` 117 | cd /emscripten-opencv 118 | npm run start:dev 119 | ``` 120 | 121 | This will automatically rebundle when a change is made to files of type .js, .html, .css etc. 122 | 123 | If you change a C++ source file, you will have to explicitly re-run `npm run build:wasm`. 124 | The resulting WebAssembly should then be automatically rebundled. 125 | 126 | # Unit Tests 127 | 128 | I have added a single unit test so far. This can be run from the command line or a web browser 129 | (inspired by chapter 13 of [WebAssembly in Action](https://www.manning.com/books/webassembly-in-action)). 130 | 131 | The following sections assume that you have already cloned and built OpenCV as described above. 132 | 133 | ## Running unit tests from the command line 134 | 135 | ``` 136 | npm run build 137 | npm test 138 | ``` 139 | 140 | > **NOTE:** this is currently failing since switching from `trzeci/emscripten` to `emscripten/emsdk`: 141 | 142 | ``` 143 | RuntimeError: abort(RuntimeError: abort(CompileError: WebAssembly.instantiate(): Compiling function #16 failed: Invalid opcode (enable with --experimental-wasm-threads) @+9038). Build with -s ASSERTIONS=1 for more info.). Build with -s ASSERTIONS=1 for more info. 144 | at process.abort (build/hello.js:1:9914) 145 | at processPromiseRejections (internal/process/promises.js:245:33) 146 | at processTicksAndRejections (internal/process/task_queues.js:94:32) { 147 | uncaught: true 148 | } 149 | ``` 150 | 151 | ## Running unit tests from a web browser 152 | 153 | ``` 154 | npm run build 155 | PORT=3434 npm start 156 | open http://localhost:3434/tests.html 157 | ``` 158 | 159 | # Links 160 | 161 | * https://github.com/mpizenberg/emscripten-opencv.git 162 | * https://docs.opencv.org/4.1.2/d4/da1/tutorial_js_setup.html 163 | * https://hub.docker.com/r/trzeci/emscripten/ 164 | * https://hub.docker.com/r/emscripten/emsdk 165 | -------------------------------------------------------------------------------- /src/hello.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace std; 11 | using namespace cv; 12 | 13 | int distanceSquared(const Point &p1, const Point& p2) { 14 | auto dx = p1.x - p2.x; 15 | auto dy = p1.y - p2.y; 16 | return dx * dx + dy * dy; 17 | } 18 | 19 | int distance(const Point &p1, const Point& p2) { 20 | return sqrt(distanceSquared(p1, p2)); 21 | } 22 | 23 | int findQuadrant(const Point ¢re, const Point &p) { 24 | if (p.x < centre.x && p.y < centre.y) return 0; 25 | if (p.x > centre.x && p.y < centre.y) return 1; 26 | if (p.x > centre.x && p.y > centre.y) return 2; 27 | if (p.x < centre.x && p.y > centre.y) return 3; 28 | return -1; 29 | } 30 | 31 | vector findCorners(const vector &contour) { 32 | auto M = moments(contour, true); 33 | auto centre = Point(M.m10 / M.m00, M.m01 / M.m00); 34 | auto corners = vector(4); 35 | int maxDistances[] = {0, 0, 0, 0}; 36 | for_each( 37 | contour.cbegin(), 38 | contour.cend(), 39 | [&](const Point &p){ 40 | auto q = findQuadrant(centre, p); 41 | if (q >= 0) { 42 | auto d = distanceSquared(centre, p); 43 | if (d > maxDistances[q]) { 44 | maxDistances[q] = d; 45 | corners[q] = p; 46 | } 47 | } 48 | }); 49 | return corners; 50 | } 51 | 52 | void applyWarpPerspective(const Mat &matIn, Mat &matOut, const vector &corners) { 53 | auto widthTop = distance(corners[0], corners[1]); 54 | auto widthBottom = distance(corners[2], corners[3]); 55 | auto heightLeft = distance(corners[0], corners[3]); 56 | auto heightRight = distance(corners[1], corners[2]); 57 | auto unwarpedSize = Size(max(widthTop, widthBottom), max(heightLeft, heightRight)); 58 | auto unwarpedCorners = vector(4); 59 | unwarpedCorners[0] = Point2f(0, 0); 60 | unwarpedCorners[1] = Point2f(unwarpedSize.width, 0); 61 | unwarpedCorners[2] = Point2f(unwarpedSize.width, unwarpedSize.height); 62 | unwarpedCorners[3] = Point2f(0, unwarpedSize.height); 63 | auto warpedCorners = vector(4); 64 | warpedCorners[0] = Point2f(corners[0].x, corners[0].y); 65 | warpedCorners[1] = Point2f(corners[1].x, corners[1].y); 66 | warpedCorners[2] = Point2f(corners[2].x, corners[2].y); 67 | warpedCorners[3] = Point2f(corners[3].x, corners[3].y); 68 | auto transform = getPerspectiveTransform(warpedCorners.data(), unwarpedCorners.data()); 69 | warpPerspective(matIn, matOut, transform, unwarpedSize); 70 | } 71 | 72 | #ifdef __cplusplus 73 | extern "C" { 74 | #endif 75 | 76 | EMSCRIPTEN_KEEPALIVE 77 | int *processImage(uchar *array, int width, int height) { 78 | 79 | Mat matIn(height, width, CV_8UC4, array); 80 | Mat matNormalised; 81 | Mat matBinary; 82 | 83 | cvtColor(matIn, matNormalised, COLOR_RGBA2GRAY); 84 | 85 | auto newSize = Size(224, 224); 86 | resize(matNormalised, matNormalised, newSize); 87 | 88 | auto ksize = Size(5, 5); 89 | auto sigmaX = 0; 90 | GaussianBlur(matNormalised, matBinary, ksize, sigmaX); 91 | 92 | auto maxValue = 255; 93 | auto adaptiveMethod = ADAPTIVE_THRESH_GAUSSIAN_C; 94 | auto thresholdType = THRESH_BINARY_INV; 95 | auto blockSize = 19; 96 | auto C = 3; 97 | adaptiveThreshold(matBinary, matBinary, maxValue, adaptiveMethod, thresholdType, blockSize, C); 98 | 99 | vector> contours; 100 | vector hierarchy; 101 | auto mode = RETR_EXTERNAL; 102 | auto method = CHAIN_APPROX_SIMPLE; 103 | findContours(matBinary, contours, hierarchy, mode, method); 104 | 105 | if (contours.empty()) { 106 | return nullptr; 107 | } 108 | 109 | auto areas = vector(contours.size()); 110 | transform( 111 | contours.cbegin(), 112 | contours.cend(), 113 | areas.begin(), 114 | [](const vector &contour){ return contourArea(contour); }); 115 | 116 | auto itMaxArea = max_element(areas.cbegin(), areas.cend()); 117 | auto indexMaxArea = distance(areas.cbegin(), itMaxArea); 118 | auto contour = contours[indexMaxArea]; // vector 119 | auto bb = boundingRect(contour); 120 | 121 | auto corners = findCorners(contour); 122 | Mat matUnwarped; 123 | applyWarpPerspective(matNormalised, matUnwarped, corners); 124 | 125 | int return_image_1_width = matNormalised.cols; 126 | int return_image_1_height = matNormalised.rows; 127 | int return_image_1_channels = matNormalised.channels(); 128 | int return_image_1_size = return_image_1_width * return_image_1_height * return_image_1_channels; 129 | 130 | int return_image_2_width = matUnwarped.cols; 131 | int return_image_2_height = matUnwarped.rows; 132 | int return_image_2_channels = matUnwarped.channels(); 133 | int return_image_2_size = return_image_2_width * return_image_2_height * return_image_2_channels; 134 | 135 | int numContourPoints = contour.size(); 136 | int return_contour_size = numContourPoints * 2 * sizeof(int); 137 | 138 | int return_data_size = 22 * sizeof(int) + return_image_1_size + return_image_2_size + return_contour_size; 139 | int *return_data = static_cast(malloc(return_data_size)); 140 | uchar *return_image_1_addr = reinterpret_cast(&return_data[22]); 141 | uchar *return_image_2_addr = return_image_1_addr + return_image_1_size; 142 | int *return_contour_addr = reinterpret_cast(return_image_2_addr + return_image_2_size); 143 | memcpy(return_image_1_addr, matNormalised.data, return_image_1_size); 144 | memcpy(return_image_2_addr, matUnwarped.data, return_image_2_size); 145 | 146 | for (auto i = 0; i < numContourPoints; i++) { 147 | return_contour_addr[i * 2] = contour[i].x; 148 | return_contour_addr[i * 2 + 1] = contour[i].y; 149 | } 150 | 151 | return_data[0] = bb.x; 152 | return_data[1] = bb.y; 153 | return_data[2] = bb.width; 154 | return_data[3] = bb.height; 155 | return_data[4] = return_image_1_width; 156 | return_data[5] = return_image_1_height; 157 | return_data[6] = return_image_1_channels; 158 | return_data[7] = reinterpret_cast(return_image_1_addr); 159 | return_data[8] = return_image_2_width; 160 | return_data[9] = return_image_2_height; 161 | return_data[10] = return_image_2_channels; 162 | return_data[11] = reinterpret_cast(return_image_2_addr); 163 | return_data[12] = corners[0].x; 164 | return_data[13] = corners[0].y; 165 | return_data[14] = corners[1].x; 166 | return_data[15] = corners[1].y; 167 | return_data[16] = corners[2].x; 168 | return_data[17] = corners[2].y; 169 | return_data[18] = corners[3].x; 170 | return_data[19] = corners[3].y; 171 | return_data[20] = numContourPoints; 172 | return_data[21] = reinterpret_cast(return_contour_addr); 173 | 174 | return return_data; 175 | } 176 | 177 | #ifdef __cplusplus 178 | } 179 | #endif 180 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | /* global R */ 2 | 3 | const range = n => 4 | Array.from(Array(n).keys()) 5 | 6 | export const inset = (x, y, w, h, dx, dy) => 7 | [x + dx, y + dy, w - 2 * dx, h - 2 * dy] 8 | 9 | const getImageData = () => { 10 | console.log('[getImageData]') 11 | const canvas = document.getElementById('input-image') 12 | const ctx = canvas.getContext('2d') 13 | const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height) 14 | return imageData 15 | } 16 | 17 | const imageDataFrom1Channel = (data, width, height) => { 18 | console.log('[imageDataFrom1Channel]') 19 | const cb = width * height * 4 20 | const array = new Uint8ClampedArray(cb) 21 | data.forEach((pixelValue, index) => { 22 | const base = index * 4 23 | array[base] = pixelValue 24 | array[base + 1] = pixelValue 25 | array[base + 2] = pixelValue 26 | array[base + 3] = 255 27 | }) 28 | const imageData = new ImageData(array, width, height) 29 | return imageData 30 | } 31 | 32 | const imageDataFrom4Channels = (data, width, height) => { 33 | console.log('[imageDataFrom4Channels]') 34 | const array = new Uint8ClampedArray(data) 35 | const imageData = new ImageData(array, width, height) 36 | return imageData 37 | } 38 | 39 | const drawOutputImage = (imageData, canvasId) => { 40 | console.log('[drawOutputImage]') 41 | const canvas = document.getElementById(canvasId) 42 | canvas.width = imageData.width 43 | canvas.height = imageData.height 44 | const ctx = canvas.getContext('2d') 45 | ctx.putImageData(imageData, 0, 0) 46 | const outputImageOverlay = document.getElementById(`${canvasId}-overlay`) 47 | outputImageOverlay.width = imageData.width 48 | outputImageOverlay.height = imageData.height 49 | } 50 | 51 | const drawBoundingBox = (boundingBox, canvasId) => { 52 | console.log('[drawBoundingBox]') 53 | const canvas = document.getElementById(canvasId) 54 | const ctx = canvas.getContext('2d') 55 | ctx.strokeStyle = 'blue' 56 | ctx.lineWidth = 1 57 | ctx.strokeRect(...inset(...boundingBox, 2, 2)) 58 | } 59 | 60 | const drawPoints = (points, canvasId, colour) => { 61 | console.log('[drawPoints]') 62 | const canvas = document.getElementById(canvasId) 63 | const ctx = canvas.getContext('2d') 64 | ctx.strokeStyle = colour 65 | ctx.lineWidth = 1 66 | const path = new Path2D() 67 | points.forEach(({ x, y }, index) => index === 0 ? path.moveTo(x, y) : path.lineTo(x, y)) 68 | path.closePath() 69 | ctx.stroke(path) 70 | } 71 | 72 | const drawCorners = (corners, canvasId) => { 73 | console.log('[drawCorners]') 74 | drawPoints(corners, canvasId, 'magenta') 75 | } 76 | 77 | export const drawContour = (points, canvasId) => { 78 | console.log('[drawContour]') 79 | drawPoints(points, canvasId, 'red') 80 | } 81 | 82 | const cropCells = (canvasId, cellsId, boundingBox) => { 83 | console.log('[cropCells]') 84 | 85 | const canvas = document.getElementById(canvasId) 86 | const ctx = canvas.getContext('2d') 87 | 88 | const cellsElement = document.getElementById(cellsId) 89 | 90 | const [bbx, bby, bbw, bbh] = inset(...boundingBox, 2, 2) 91 | const cellw = bbw / 9 92 | const cellh = bbh / 9 93 | for (const y of range(9)) { 94 | const row = document.createElement('div') 95 | const celly = bby + y * cellh 96 | for (const x of range(9)) { 97 | const cellx = bbx + x * cellw 98 | const imageData = ctx.getImageData(...inset(cellx, celly, cellw, cellh, 2, 2)) 99 | const cellCanvas = document.createElement('canvas') 100 | cellCanvas.setAttribute('class', 'cell') 101 | cellCanvas.width = imageData.width 102 | cellCanvas.height = imageData.height 103 | cellCanvas.getContext('2d').putImageData(imageData, 0, 0) 104 | row.appendChild(cellCanvas) 105 | } 106 | cellsElement.appendChild(row) 107 | } 108 | } 109 | 110 | const clearCanvas = canvasId => { 111 | const canvas = document.getElementById(canvasId) 112 | const ctx = canvas.getContext('2d') 113 | ctx.clearRect(0, 0, canvas.width, canvas.height) 114 | } 115 | 116 | const deleteChildren = elementId => { 117 | const parent = document.getElementById(elementId) 118 | while (parent.firstChild) { 119 | parent.removeChild(parent.firstChild) 120 | } 121 | } 122 | 123 | const reset = () => { 124 | clearCanvas('output-image-1') 125 | clearCanvas('output-image-1-overlay') 126 | clearCanvas('output-image-2') 127 | clearCanvas('output-image-2-overlay') 128 | deleteChildren('cells-1') 129 | deleteChildren('cells-2') 130 | const elapsedTimeRow = document.getElementById('elapsed-time-row') 131 | elapsedTimeRow.style.display = 'none'; 132 | } 133 | 134 | const loadInputImage = async index => { 135 | console.log('[loadInputImage]') 136 | const inputImageSelector = document.getElementById('input-image-selector') 137 | const imageUrl = inputImageSelector.options[index].value 138 | const image = new Image() 139 | await new Promise(resolve => { 140 | image.onload = resolve 141 | image.src = imageUrl 142 | }) 143 | const canvas = document.getElementById('input-image') 144 | canvas.width = image.width 145 | canvas.height = image.height 146 | const ctx = canvas.getContext('2d') 147 | ctx.drawImage(image, 0, 0, image.width, image.height) 148 | const inputImageOverlay = document.getElementById('input-image-overlay') 149 | inputImageOverlay.width = image.width 150 | inputImageOverlay.height = image.height 151 | reset() 152 | } 153 | 154 | const onSelectImageSudoku = e => { 155 | console.log('[onSelectImageSudoku]') 156 | loadInputImage(e.target.selectedIndex) 157 | } 158 | 159 | const unpackImage = (module, [width, height, channels, addr]) => { 160 | const cb = width * height * channels 161 | const data = module.HEAPU8.slice(addr, addr + cb) 162 | return channels === 1 163 | ? imageDataFrom1Channel(data, width, height) 164 | : imageDataFrom4Channels(data, width, height) 165 | } 166 | 167 | const unpackCorners = data32 => { 168 | return R.splitEvery(2, data32).map(([x, y]) => ({ x, y })) 169 | } 170 | 171 | const unpackContour = (module, [numPoints, addr]) => { 172 | const addr32 = addr / module.HEAP32.BYTES_PER_ELEMENT 173 | const data32 = module.HEAP32.slice(addr32, addr32 + numPoints * 2) 174 | return R.splitEvery(2, data32).map(([x, y]) => ({ x, y })) 175 | } 176 | 177 | const unpackProcessImageResult = (module, addr) => { 178 | const NUM_INT_FIELDS = 22 179 | const addr32 = addr / module.HEAP32.BYTES_PER_ELEMENT 180 | const data32 = module.HEAP32.slice(addr32, addr32 + NUM_INT_FIELDS) 181 | const boundingBox = data32.slice(0, 4) 182 | const image1 = unpackImage(module, data32.slice(4, 8)) 183 | const image2 = unpackImage(module, data32.slice(8, 12)) 184 | const corners = unpackCorners(data32.slice(12, 20)) 185 | const contour = unpackContour(module, data32.slice(20, 22)) 186 | return { boundingBox, image1, image2, corners, contour } 187 | } 188 | 189 | const onProcessImage = (module, processImage) => () => { 190 | console.log('[onProcessImage]') 191 | reset() 192 | const { data, width, height } = getImageData() 193 | 194 | const startTime = performance.now() 195 | const addr = processImage(data, width, height) 196 | const endTime = performance.now() 197 | 198 | const elapsedTimeRow = document.getElementById('elapsed-time-row') 199 | elapsedTimeRow.style.display = 'block'; 200 | const elapsedTime = document.getElementById('elapsed-time') 201 | elapsedTime.innerText = (endTime - startTime).toFixed(2) 202 | 203 | const unpackedResult = unpackProcessImageResult(module, addr) 204 | module._free(addr) 205 | 206 | drawOutputImage(unpackedResult.image1, 'output-image-1') 207 | drawOutputImage(unpackedResult.image2, 'output-image-2') 208 | drawBoundingBox(unpackedResult.boundingBox, 'output-image-1-overlay') 209 | drawCorners(unpackedResult.corners, 'output-image-1-overlay') 210 | drawContour(unpackedResult.contour, 'output-image-1-overlay') 211 | 212 | const boundingBox1 = unpackedResult.boundingBox 213 | const boundingBox2 = [0, 0, unpackedResult.image2.width, unpackedResult.image2.height] 214 | 215 | cropCells('output-image-1', 'cells-1', boundingBox1) 216 | cropCells('output-image-2', 'cells-2', boundingBox2) 217 | } 218 | 219 | const wrapProcessImage = module => { 220 | console.log('[wrapProcessImage]') 221 | const ident = 'processImage' 222 | const returnType = 'number' 223 | const argTypes = ['array', 'number', 'number'] 224 | const processImage = module.cwrap(ident, returnType, argTypes) 225 | return processImage 226 | } 227 | 228 | const init = module => { 229 | console.log('[init]') 230 | const inputImageSelector = document.getElementById('input-image-selector') 231 | range(2).forEach(index => { 232 | const sudokuNumber = index + 1 233 | const value = `/images/sudoku-${sudokuNumber}.png` 234 | const label = `Sudoku ${sudokuNumber}` 235 | const optionElement = document.createElement('option') 236 | optionElement.setAttribute('value', value) 237 | optionElement.innerText = label 238 | inputImageSelector.appendChild(optionElement) 239 | }) 240 | inputImageSelector.addEventListener('change', onSelectImageSudoku) 241 | loadInputImage(0) 242 | const processImage = wrapProcessImage(module) 243 | const processImageBtn = document.getElementById('process-image-btn') 244 | processImageBtn.addEventListener('click', onProcessImage(module, processImage)) 245 | } 246 | 247 | const main = async () => { 248 | window.createHelloModule().then(init) 249 | } 250 | 251 | main() 252 | --------------------------------------------------------------------------------