├── .gitignore ├── LICENSE ├── README.md ├── electron ├── .babelrc ├── .eslintrc.js ├── package-lock.json ├── package.json ├── src │ ├── components │ │ ├── App.jsx │ │ ├── GoButton.jsx │ │ ├── Labels │ │ │ ├── Label.jsx │ │ │ └── index.jsx │ │ ├── Selector.jsx │ │ └── Status.jsx │ ├── contexts │ │ └── index.js │ ├── index.html │ ├── index.js │ ├── main.js │ └── utils │ │ ├── i18n.js │ │ ├── ml5.js │ │ └── socket.js └── webpack.config.js ├── index.js ├── main.maxpat ├── package-lock.json └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | dist/ 3 | .DS_Store -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Yuichi Yogo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # n4m Feature Extractor 2 | 3 | Train your model in real-time! 4 | 5 | ![2](https://user-images.githubusercontent.com/14039540/56828415-43196480-682f-11e9-97df-701ad2c3528c.gif) 6 | 7 | ## What does it do? 8 | 9 | By feeding a stream of images via a web cam, you can create your own neural network model that can classify any kind of incoming images. 10 | 11 | It uses [ml5.js](https://ml5js.org/) under the hood. 12 | 13 | ## Get Started 14 | 15 | Open `main.maxpat` and follow the instructions. 16 | 17 | ## Requirements 18 | 19 | - Max 8.0.0 or higher 20 | - Internet connection (needed to install dependencies) -------------------------------------------------------------------------------- /electron/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | ["@babel/preset-env",{ 4 | "modules": false 5 | }], 6 | "@babel/preset-react" 7 | ], 8 | } -------------------------------------------------------------------------------- /electron/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "env": { 3 | "browser": true, 4 | "es6": true, 5 | "node": true 6 | }, 7 | "extends": [ 8 | "eslint:recommended", 9 | "plugin:react/recommended", 10 | "plugin:prettier/recommended", 11 | ], 12 | "globals": { 13 | "Atomics": "readonly", 14 | "SharedArrayBuffer": "readonly" 15 | }, 16 | "parserOptions": { 17 | "ecmaFeatures": { 18 | "jsx": true 19 | }, 20 | "ecmaVersion": 2018, 21 | "sourceType": "module" 22 | }, 23 | "plugins": [ 24 | "prettier", 25 | "react", 26 | "react-hooks" 27 | ], 28 | "rules": { 29 | "no-console": 0 , 30 | "react-hooks/rules-of-hooks": "error", 31 | "react-hooks/exhaustive-deps": "warn", 32 | "react/prop-types": 0 33 | 34 | } 35 | }; -------------------------------------------------------------------------------- /electron/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "feature-extractor-electron", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "src/main.js", 6 | "scripts": { 7 | "start": "npm run watch & npm run launch", 8 | "launch": "DEBUG=1 electron .", 9 | "watch": "webpack --mode development --watch", 10 | "build": "webpack --mode production", 11 | "postinstall": "npm run build" 12 | }, 13 | "author": "Yuichi Yogo", 14 | "license": "MIT", 15 | "dependencies": { 16 | "@material-ui/core": "^4.0.0", 17 | "i18next": "^15.1.3", 18 | "ml5": "^0.2.3", 19 | "react": "^16.8.6", 20 | "react-dom": "^16.8.6", 21 | "react-i18next": "^10.10.0", 22 | "redux": "^4.0.1", 23 | "socket.io-client": "^2.1.1" 24 | }, 25 | "devDependencies": { 26 | "@babel/core": "^7.4.5", 27 | "@babel/preset-env": "^7.4.5", 28 | "@babel/preset-react": "^7.0.0", 29 | "babel-core": "^6.26.3", 30 | "babel-loader": "^8.0.6", 31 | "babel-preset-env": "^1.7.0", 32 | "babel-preset-react": "^6.24.1", 33 | "electron": "^3.0.2", 34 | "eslint": "^5.16.0", 35 | "eslint-config-prettier": "^4.3.0", 36 | "eslint-plugin-prettier": "^3.1.0", 37 | "eslint-plugin-react": "^7.13.0", 38 | "eslint-plugin-react-hooks": "^1.6.0", 39 | "redux-actions": "^2.6.5", 40 | "webpack": "^4.34.0", 41 | "webpack-cli": "^3.3.4" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /electron/src/components/App.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect, useRef } from "react"; 2 | import Status from "./Status"; 3 | import Selector from "./Selector"; 4 | import Labels from "./Labels"; 5 | import GoButton from "./GoButton"; 6 | import { buildFeatureExtractor } from "../utils/ml5"; 7 | import { StatusContext, LabelContext } from "../contexts"; 8 | 9 | const containerStyle = { 10 | display: "flex", 11 | flexDirection: "column", 12 | justifyContent: "center", 13 | height: "800px" 14 | }; 15 | const mainStyle = { 16 | display: "flex", 17 | flexDirection: "column", 18 | justifyContent: "space-between", 19 | height: "210px" 20 | }; 21 | 22 | function App() { 23 | const videoEl = useRef(null); 24 | const [status, setStatus] = useState("status.ready"); 25 | const [labelCount, setLabelCount] = useState(2); 26 | const [targetLabel, setTargetLabel] = useState(null); 27 | const [classifier, setClassifier] = useState(null); 28 | 29 | useEffect(() => { 30 | const setVideo = async () => { 31 | videoEl.current.srcObject = await navigator.mediaDevices.getUserMedia({ 32 | video: true 33 | }); 34 | setClassifier(buildFeatureExtractor(videoEl.current)); 35 | }; 36 | setVideo(); 37 | }, []); 38 | 39 | return ( 40 | 46 | 54 |
55 |
63 |
64 |
65 | ); 66 | } 67 | 68 | export default App; 69 | -------------------------------------------------------------------------------- /electron/src/components/GoButton.jsx: -------------------------------------------------------------------------------- 1 | import React, { useContext } from "react"; 2 | import { withTranslation } from "react-i18next"; 3 | import Button from "@material-ui/core/Button"; 4 | import { StatusContext } from "../contexts"; 5 | import socket from "../utils/socket"; 6 | 7 | const GoButton = ({ classifier, t }) => { 8 | const { setStatus } = useContext(StatusContext); 9 | function classify() { 10 | classifier.classify((err, results) => { 11 | if (err) return; 12 | const topResult = results[0]; 13 | const { confidence } = topResult; 14 | const { label } = topResult; 15 | const status = 16 | confidence > 0.8 17 | ? ["status.detected", { label }] 18 | : t("status.undetectable"); 19 | setStatus(status); 20 | socket.emit("dispatch", { 21 | data: label 22 | }); 23 | classify(); 24 | }); 25 | } 26 | return ( 27 | 40 | ); 41 | }; 42 | 43 | export default withTranslation()(GoButton); 44 | -------------------------------------------------------------------------------- /electron/src/components/Labels/Label.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useContext } from "react"; 2 | import { withTranslation } from "react-i18next"; 3 | import Button from "@material-ui/core/Button"; 4 | import TextField from "@material-ui/core/TextField"; 5 | import { StatusContext, LabelContext } from "../../contexts"; 6 | 7 | const half = { width: "50%" }; 8 | 9 | const Label = ({ t, index, classifier }) => { 10 | const defaultLabel = t("label.default"); 11 | 12 | const { setStatus } = useContext(StatusContext); 13 | const { targetLabel, setTargetLabel } = useContext(LabelContext); 14 | 15 | const [isFeeding, setIsFeeding] = useState(false); 16 | const [name, setName] = useState(`${defaultLabel}${index}`); 17 | const [intervalID, setIntervalID] = useState(null); 18 | 19 | const buttonStatus = isFeeding 20 | ? t("status.stopLearning") 21 | : t("status.startLearning"); 22 | 23 | return ( 24 |
25 | { 29 | setName(e.target.value); 30 | }} 31 | /> 32 | 59 |
60 | ); 61 | }; 62 | 63 | export default withTranslation()(Label); 64 | -------------------------------------------------------------------------------- /electron/src/components/Labels/index.jsx: -------------------------------------------------------------------------------- 1 | import React, { useContext } from "react"; 2 | import Label from "./Label"; 3 | import { LabelContext } from "../../contexts"; 4 | 5 | const Labels = ({ classifier }) => { 6 | const labels = []; 7 | const { labelCount, targetLabel } = useContext(LabelContext); 8 | 9 | for (let i = 1; i <= labelCount; i++) { 10 | labels.push( 11 |