29 |
39 |
40 |
41 |
--------------------------------------------------------------------------------
/src/index.js:
--------------------------------------------------------------------------------
1 | import React from "react";
2 | import ReactDOM from "react-dom";
3 | import * as tf from '@tensorflow/tfjs';
4 | import {loadGraphModel} from '@tensorflow/tfjs-converter';
5 | import "./styles.css";
6 | tf.setBackend('webgl');
7 |
8 | const threshold = 0.75;
9 |
10 | async function load_model() {
11 | // It's possible to load the model locally or from a repo
12 | // You can choose whatever IP and PORT you want in the "http://127.0.0.1:8080/model.json" just set it before in your https server
13 | //const model = await loadGraphModel("http://127.0.0.1:8080/model.json");
14 | const model = await loadGraphModel("https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/master/models/kangaroo-detector/model.json");
15 | return model;
16 | }
17 |
18 | let classesDir = {
19 | 1: {
20 | name: 'Kangaroo',
21 | id: 1,
22 | },
23 | 2: {
24 | name: 'Other',
25 | id: 2,
26 | }
27 | }
28 |
29 | class App extends React.Component {
30 | videoRef = React.createRef();
31 | canvasRef = React.createRef();
32 |
33 |
34 | componentDidMount() {
35 | if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
36 | const webCamPromise = navigator.mediaDevices
37 | .getUserMedia({
38 | audio: false,
39 | video: {
40 | facingMode: "user"
41 | }
42 | })
43 | .then(stream => {
44 | window.stream = stream;
45 | this.videoRef.current.srcObject = stream;
46 | return new Promise((resolve, reject) => {
47 | this.videoRef.current.onloadedmetadata = () => {
48 | resolve();
49 | };
50 | });
51 | });
52 |
53 | const modelPromise = load_model();
54 |
55 | Promise.all([modelPromise, webCamPromise])
56 | .then(values => {
57 | this.detectFrame(this.videoRef.current, values[0]);
58 | })
59 | .catch(error => {
60 | console.error(error);
61 | });
62 | }
63 | }
64 |
65 | detectFrame = (video, model) => {
66 | tf.engine().startScope();
67 | model.executeAsync(this.process_input(video)).then(predictions => {
68 | this.renderPredictions(predictions, video);
69 | requestAnimationFrame(() => {
70 | this.detectFrame(video, model);
71 | });
72 | tf.engine().endScope();
73 | });
74 | };
75 |
76 | process_input(video_frame){
77 | const tfimg = tf.browser.fromPixels(video_frame).toInt();
78 | const expandedimg = tfimg.transpose([0,1,2]).expandDims();
79 | return expandedimg;
80 | };
81 |
82 | buildDetectedObjects(scores, threshold, boxes, classes, classesDir) {
83 | const detectionObjects = []
84 | var video_frame = document.getElementById('frame');
85 |
86 | scores[0].forEach((score, i) => {
87 | if (score > threshold) {
88 | const bbox = [];
89 | const minY = boxes[0][i][0] * video_frame.offsetHeight;
90 | const minX = boxes[0][i][1] * video_frame.offsetWidth;
91 | const maxY = boxes[0][i][2] * video_frame.offsetHeight;
92 | const maxX = boxes[0][i][3] * video_frame.offsetWidth;
93 | bbox[0] = minX;
94 | bbox[1] = minY;
95 | bbox[2] = maxX - minX;
96 | bbox[3] = maxY - minY;
97 | detectionObjects.push({
98 | class: classes[i],
99 | label: classesDir[classes[i]].name,
100 | score: score.toFixed(4),
101 | bbox: bbox
102 | })
103 | }
104 | })
105 | return detectionObjects
106 | }
107 |
108 | renderPredictions = predictions => {
109 | const ctx = this.canvasRef.current.getContext("2d");
110 | ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
111 |
112 | // Font options.
113 | const font = "16px sans-serif";
114 | ctx.font = font;
115 | ctx.textBaseline = "top";
116 |
117 | //Getting predictions
118 | const boxes = predictions[4].arraySync();
119 | const scores = predictions[5].arraySync();
120 | const classes = predictions[6].dataSync();
121 | const detections = this.buildDetectedObjects(scores, threshold,
122 | boxes, classes, classesDir);
123 |
124 | detections.forEach(item => {
125 | const x = item['bbox'][0];
126 | const y = item['bbox'][1];
127 | const width = item['bbox'][2];
128 | const height = item['bbox'][3];
129 |
130 | // Draw the bounding box.
131 | ctx.strokeStyle = "#00FFFF";
132 | ctx.lineWidth = 4;
133 | ctx.strokeRect(x, y, width, height);
134 |
135 | // Draw the label background.
136 | ctx.fillStyle = "#00FFFF";
137 | const textWidth = ctx.measureText(item["label"] + " " + (100 * item["score"]).toFixed(2) + "%").width;
138 | const textHeight = parseInt(font, 10); // base 10
139 | ctx.fillRect(x, y, textWidth + 4, textHeight + 4);
140 | });
141 |
142 | detections.forEach(item => {
143 | const x = item['bbox'][0];
144 | const y = item['bbox'][1];
145 |
146 | // Draw the text last to ensure it's on top.
147 | ctx.fillStyle = "#000000";
148 | ctx.fillText(item["label"] + " " + (100*item["score"]).toFixed(2) + "%", x, y);
149 | });
150 | };
151 |
152 | render() {
153 | return (
154 |
155 |
Real-Time Object Detection: Kangaroo
156 | MobileNetV2
157 |
168 |
174 |
175 | );
176 | }
177 | }
178 |
179 | const rootElement = document.getElementById("root");
180 | ReactDOM.render(
, rootElement);
181 |
--------------------------------------------------------------------------------
/src/styles.css:
--------------------------------------------------------------------------------
1 | .size {
2 | position: fixed;
3 | top: 0;
4 | left: 0;
5 | }
--------------------------------------------------------------------------------
18 | 
30 |
31 |
33 |
34 |
35 |
36 | You can train your own model, upload it in the `model` folder and load as well.
37 |
38 | #### Serving the model
39 |
40 | To make the model available, it is necessary to define how the model is going to be loaded in the function `load_model` (lines 10–15 in the file `src>index.js`). There are two choices.
41 |
42 | The first option is to create an _HTTP server_ locally that will make the model available in a URL allowing requests and be treated as a REST API. When loading the model, _TensorFlow.js_ will do the following requests:
43 |
44 | ```
45 | GET /model.json
46 | GET /group1-shard1of5.bin
47 | GET /group1-shard2of5.bin
48 | GET /group1-shard3of5.bin
49 | GET /group1-shardo4f5.bin
50 | GET /group1-shardo5f5.bin
51 | ```
52 |
53 | If you choose this option, define the `load_model` function as follows:
54 |
55 |
56 | ```js
57 | async function load_model() {
58 | // It's possible to load the model locally or from a repo
59 | const model = await loadGraphModel("http://127.0.0.1:8080/model.json");
60 | //const model = await loadGraphModel("https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/master/models/kangaroo-detector/model.json");
61 | return model;
62 | }
63 | ```
64 | Then install the [http-server](https://www.npmjs.com/package/http-server):
65 |
66 | ```
67 | npm install http-server -g
68 | ```
69 |
70 | Go to `models > kangaroo-detector` and run the command below to make the model available at `http://127.0.0.1:8080` . This a good choice when you want to keep the model weights in a safe place and control who can request inferences to it.
71 |
72 | ```
73 | http-server -c1 --cors .
74 | ```
75 | The second option is to upload the model files somewhere, in my case, I chose my own Github repo and referenced to the `model.json` URL in the `load_model` function:
76 | homepage.png
77 | ```js
78 | async function load_model() {
79 | // It's possible to load the model locally or from a repo
80 | //const model = await loadGraphModel("http://127.0.0.1:8080/model.json");
81 | const model = await loadGraphModel("https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/master/models/kangaroo-detector/model.json");
82 | return model;
83 | }
84 | ```
85 |
86 | This is a good option because it gives more flexibility to the application and makes it easier to run on some platform as [Glitch](https://glitch.com/).
87 |
88 | #### Running locally
89 | To run the app locally, install the required packages:
90 |
91 | ```
92 | npm install
93 | ```
94 |
95 | And start:
96 |
97 | ```
98 | npm start
99 | ```
100 |
101 | The application is going to run at `[http://localhost:3000](http://localhost:3000)` and you should see something similar to this:
102 |
103 | 
104 |
105 | The model takes from 1 to 2 seconds to load and, after that, you can show the objects images to the camera and the application is going to draw bounding boxes around them.
106 |
107 |
--------------------------------------------------------------------------------
/git_media/dataset.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/4735463af4c596ce56434608cba7f30210dfed5b/git_media/dataset.png
--------------------------------------------------------------------------------
/git_media/header.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/4735463af4c596ce56434608cba7f30210dfed5b/git_media/header.png
--------------------------------------------------------------------------------
/git_media/homepage.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/4735463af4c596ce56434608cba7f30210dfed5b/git_media/homepage.png
--------------------------------------------------------------------------------
/git_media/kangaroo-demo.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/4735463af4c596ce56434608cba7f30210dfed5b/git_media/kangaroo-demo.gif
--------------------------------------------------------------------------------
/git_media/webcam.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/4735463af4c596ce56434608cba7f30210dfed5b/git_media/webcam.gif
--------------------------------------------------------------------------------
/models/kangaroo-detector/group1-shard1of5.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/4735463af4c596ce56434608cba7f30210dfed5b/models/kangaroo-detector/group1-shard1of5.bin
--------------------------------------------------------------------------------
/models/kangaroo-detector/group1-shard2of5.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/4735463af4c596ce56434608cba7f30210dfed5b/models/kangaroo-detector/group1-shard2of5.bin
--------------------------------------------------------------------------------
/models/kangaroo-detector/group1-shard3of5.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/4735463af4c596ce56434608cba7f30210dfed5b/models/kangaroo-detector/group1-shard3of5.bin
--------------------------------------------------------------------------------
/models/kangaroo-detector/group1-shard4of5.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/4735463af4c596ce56434608cba7f30210dfed5b/models/kangaroo-detector/group1-shard4of5.bin
--------------------------------------------------------------------------------
/models/kangaroo-detector/group1-shard5of5.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hugozanini/TFJS-object-detection/4735463af4c596ce56434608cba7f30210dfed5b/models/kangaroo-detector/group1-shard5of5.bin
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "tensorflowjs-real-time-object-detection",
3 | "version": "1.0.0",
4 | "description": "",
5 | "keywords": [],
6 | "main": "src/index.js",
7 | "dependencies": {
8 | "@tensorflow/tfjs": "^1.7.4",
9 | "@tensorflow/tfjs-converter": "^1.7.4",
10 | "@tensorflow/tfjs-core": "^1.7.4",
11 | "@tensorflow/tfjs-node": "^1.7.4",
12 | "react": "16.5.2",
13 | "react-dom": "16.5.2",
14 | "react-magic-dropzone": "1.0.1",
15 | "react-scripts": "2.0.3"
16 | },
17 | "devDependencies": {},
18 | "scripts": {
19 | "start": "react-scripts start",
20 | "build": "react-scripts build",
21 | "test": "react-scripts test --env=jsdom",
22 | "eject": "react-scripts eject"
23 | },
24 | "browserslist": [
25 | ">0.2%",
26 | "not dead",
27 | "not ie <= 11",
28 | "not op_mini all"
29 | ]
30 | }
31 |
--------------------------------------------------------------------------------
/public/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
7 |
8 |
9 |
10 |
11 |
20 |