├── .gitignore
├── LICENSE
├── README.md
├── electron
├── .babelrc
├── .eslintrc.js
├── package-lock.json
├── package.json
├── src
│ ├── components
│ │ ├── App.jsx
│ │ ├── GoButton.jsx
│ │ ├── Labels
│ │ │ ├── Label.jsx
│ │ │ └── index.jsx
│ │ ├── Selector.jsx
│ │ └── Status.jsx
│ ├── contexts
│ │ └── index.js
│ ├── index.html
│ ├── index.js
│ ├── main.js
│ └── utils
│ │ ├── i18n.js
│ │ ├── ml5.js
│ │ └── socket.js
└── webpack.config.js
├── index.js
├── main.maxpat
├── package-lock.json
└── package.json
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | dist/
3 | .DS_Store
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2019 Yuichi Yogo
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # n4m Feature Extractor
2 |
3 | Train your model in real-time!
4 |
5 | 
6 |
7 | ## What does it do?
8 |
9 | By feeding a stream of images via a web cam, you can create your own neural network model that can classify any kind of incoming images.
10 |
11 | It uses [ml5.js](https://ml5js.org/) under the hood.
12 |
13 | ## Get Started
14 |
15 | Open `main.maxpat` and follow the instructions.
16 |
17 | ## Requirements
18 |
19 | - Max 8.0.0 or higher
20 | - Internet connection (needed to install dependencies)
--------------------------------------------------------------------------------
/electron/.babelrc:
--------------------------------------------------------------------------------
1 | {
2 | "presets": [
3 | ["@babel/preset-env",{
4 | "modules": false
5 | }],
6 | "@babel/preset-react"
7 | ],
8 | }
--------------------------------------------------------------------------------
/electron/.eslintrc.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | "env": {
3 | "browser": true,
4 | "es6": true,
5 | "node": true
6 | },
7 | "extends": [
8 | "eslint:recommended",
9 | "plugin:react/recommended",
10 | "plugin:prettier/recommended",
11 | ],
12 | "globals": {
13 | "Atomics": "readonly",
14 | "SharedArrayBuffer": "readonly"
15 | },
16 | "parserOptions": {
17 | "ecmaFeatures": {
18 | "jsx": true
19 | },
20 | "ecmaVersion": 2018,
21 | "sourceType": "module"
22 | },
23 | "plugins": [
24 | "prettier",
25 | "react",
26 | "react-hooks"
27 | ],
28 | "rules": {
29 | "no-console": 0 ,
30 | "react-hooks/rules-of-hooks": "error",
31 | "react-hooks/exhaustive-deps": "warn",
32 | "react/prop-types": 0
33 |
34 | }
35 | };
--------------------------------------------------------------------------------
/electron/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "feature-extractor-electron",
3 | "version": "1.0.0",
4 | "description": "",
5 | "main": "src/main.js",
6 | "scripts": {
7 | "start": "npm run watch & npm run launch",
8 | "launch": "DEBUG=1 electron .",
9 | "watch": "webpack --mode development --watch",
10 | "build": "webpack --mode production",
11 | "postinstall": "npm run build"
12 | },
13 | "author": "Yuichi Yogo",
14 | "license": "MIT",
15 | "dependencies": {
16 | "@material-ui/core": "^4.0.0",
17 | "i18next": "^15.1.3",
18 | "ml5": "^0.2.3",
19 | "react": "^16.8.6",
20 | "react-dom": "^16.8.6",
21 | "react-i18next": "^10.10.0",
22 | "redux": "^4.0.1",
23 | "socket.io-client": "^2.1.1"
24 | },
25 | "devDependencies": {
26 | "@babel/core": "^7.4.5",
27 | "@babel/preset-env": "^7.4.5",
28 | "@babel/preset-react": "^7.0.0",
29 | "babel-core": "^6.26.3",
30 | "babel-loader": "^8.0.6",
31 | "babel-preset-env": "^1.7.0",
32 | "babel-preset-react": "^6.24.1",
33 | "electron": "^3.0.2",
34 | "eslint": "^5.16.0",
35 | "eslint-config-prettier": "^4.3.0",
36 | "eslint-plugin-prettier": "^3.1.0",
37 | "eslint-plugin-react": "^7.13.0",
38 | "eslint-plugin-react-hooks": "^1.6.0",
39 | "redux-actions": "^2.6.5",
40 | "webpack": "^4.34.0",
41 | "webpack-cli": "^3.3.4"
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/electron/src/components/App.jsx:
--------------------------------------------------------------------------------
1 | import React, { useState, useEffect, useRef } from "react";
2 | import Status from "./Status";
3 | import Selector from "./Selector";
4 | import Labels from "./Labels";
5 | import GoButton from "./GoButton";
6 | import { buildFeatureExtractor } from "../utils/ml5";
7 | import { StatusContext, LabelContext } from "../contexts";
8 |
9 | const containerStyle = {
10 | display: "flex",
11 | flexDirection: "column",
12 | justifyContent: "center",
13 | height: "800px"
14 | };
15 | const mainStyle = {
16 | display: "flex",
17 | flexDirection: "column",
18 | justifyContent: "space-between",
19 | height: "210px"
20 | };
21 |
22 | function App() {
23 | const videoEl = useRef(null);
24 | const [status, setStatus] = useState("status.ready");
25 | const [labelCount, setLabelCount] = useState(2);
26 | const [targetLabel, setTargetLabel] = useState(null);
27 | const [classifier, setClassifier] = useState(null);
28 |
29 | useEffect(() => {
30 | const setVideo = async () => {
31 | videoEl.current.srcObject = await navigator.mediaDevices.getUserMedia({
32 | video: true
33 | });
34 | setClassifier(buildFeatureExtractor(videoEl.current));
35 | };
36 | setVideo();
37 | }, []);
38 |
39 | return (
40 |
46 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 | );
66 | }
67 |
68 | export default App;
69 |
--------------------------------------------------------------------------------
/electron/src/components/GoButton.jsx:
--------------------------------------------------------------------------------
1 | import React, { useContext } from "react";
2 | import { withTranslation } from "react-i18next";
3 | import Button from "@material-ui/core/Button";
4 | import { StatusContext } from "../contexts";
5 | import socket from "../utils/socket";
6 |
7 | const GoButton = ({ classifier, t }) => {
8 | const { setStatus } = useContext(StatusContext);
9 | function classify() {
10 | classifier.classify((err, results) => {
11 | if (err) return;
12 | const topResult = results[0];
13 | const { confidence } = topResult;
14 | const { label } = topResult;
15 | const status =
16 | confidence > 0.8
17 | ? ["status.detected", { label }]
18 | : t("status.undetectable");
19 | setStatus(status);
20 | socket.emit("dispatch", {
21 | data: label
22 | });
23 | classify();
24 | });
25 | }
26 | return (
27 |
40 | );
41 | };
42 |
43 | export default withTranslation()(GoButton);
44 |
--------------------------------------------------------------------------------
/electron/src/components/Labels/Label.jsx:
--------------------------------------------------------------------------------
1 | import React, { useState, useContext } from "react";
2 | import { withTranslation } from "react-i18next";
3 | import Button from "@material-ui/core/Button";
4 | import TextField from "@material-ui/core/TextField";
5 | import { StatusContext, LabelContext } from "../../contexts";
6 |
7 | const half = { width: "50%" };
8 |
9 | const Label = ({ t, index, classifier }) => {
10 | const defaultLabel = t("label.default");
11 |
12 | const { setStatus } = useContext(StatusContext);
13 | const { targetLabel, setTargetLabel } = useContext(LabelContext);
14 |
15 | const [isFeeding, setIsFeeding] = useState(false);
16 | const [name, setName] = useState(`${defaultLabel}${index}`);
17 | const [intervalID, setIntervalID] = useState(null);
18 |
19 | const buttonStatus = isFeeding
20 | ? t("status.stopLearning")
21 | : t("status.startLearning");
22 |
23 | return (
24 |
25 | {
29 | setName(e.target.value);
30 | }}
31 | />
32 |
59 |
60 | );
61 | };
62 |
63 | export default withTranslation()(Label);
64 |
--------------------------------------------------------------------------------
/electron/src/components/Labels/index.jsx:
--------------------------------------------------------------------------------
1 | import React, { useContext } from "react";
2 | import Label from "./Label";
3 | import { LabelContext } from "../../contexts";
4 |
5 | const Labels = ({ classifier }) => {
6 | const labels = [];
7 | const { labelCount, targetLabel } = useContext(LabelContext);
8 |
9 | for (let i = 1; i <= labelCount; i++) {
10 | labels.push(
11 |