56 | )
57 | }
58 |
59 | export default SourceSelector;
60 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | 
2 |
3 | [Lobe](http://lobe.ai/) is a free, easy to use app that has everything you need to bring your machine learning ideas to life.
4 | Web Bootstrap takes the machine learning model created in Lobe, and adds it to a project in the browser that uses
5 | [React](https://reactjs.org), [Create React App](https://github.com/facebook/create-react-app), [TypeScript](https://www.typescriptlang.org/), and [TensorFlow.js](https://www.tensorflow.org/js).
6 |
7 | ## Get Started
8 |
9 | 1. Clone or download the project on your computer and install [Yarn](https://yarnpkg.com/). Yarn is the software package that will install all the dependencies and make sure the code automatically reloads when changes are made.
10 |
11 | 2. Run `yarn install` to install required dependencies and run `yarn start` to start the server in development mode. This will open a web browser to
12 | `localhost:3000`. By default, this project is using the TensorFlow.js exported model from Lobe found in the `public/model/` folder.
13 |
14 | 3. To use your own model file, open your Lobe project, go to the Use tab, select Export, and click on the TensorFlow.js model file.
15 | When exported, drag the `model.json`, `signature.json`, and all the `*.bin` files to the `public/model/` folder.
16 |
17 | ## Additional Information
18 |
19 | Check out the [Create React App documentation](https://create-react-app.dev/docs/getting-started)
20 | for more information on React and the project structure.
21 |
22 | There are three main components: Camera, Prediction, and StaticImage.
23 | The Camera, which runs in `components/camera/Camera.tsx` is responsible for displaying a live full screen view of the user's selected webcam.
24 | The Prediction component `components/prediction/Prediction.tsx` is the box in the lower left hand corner, and is responsible for displaying the top prediction results and their confidences.
25 | The StaticImage component `components/staticImage/StaticImage.tsx` displays an image selected from the file picker and runs it through the model from a canvas element.
26 |
27 | ### Known Issues
28 | TensorFlow.js on Safari may have problems initializing the WebGL backend for acceleration and will fall back to the CPU.
29 | You can use the WebAssembly (wasm) backend as an alternative to WebGL:
30 | https://www.tensorflow.org/js/guide/platform_environment#wasm_backend
31 |
32 | ## Contributing
33 |
34 | GitHub Issues are for reporting bugs, discussing features and general feedback on the Web Bootstrap project. Be sure to check our documentation, FAQ and past issues before opening any new ones.
35 |
36 | To share your project, get feedback on it, and learn more about Lobe, please visit our community on [Reddit](https://www.reddit.com/r/Lobe/).
37 | We look forward to seeing the amazing projects that can be built, when machine learning is made accessible to you.
38 |
--------------------------------------------------------------------------------
/src/components/camera/Camera.tsx:
--------------------------------------------------------------------------------
1 | import React, {useEffect, useState, useRef, useCallback} from "react";
2 | import Webcam from "react-webcam";
3 | import SourceSelector from "./SourceSelector";
4 | import "./Camera.css";
5 |
6 | type CameraProps = {
7 | predictCanvas: (canvas: HTMLCanvasElement) => void;
8 | predictions?: { [label: string]: number };
9 | }
10 |
11 | // Our webcam display and capture component
12 | function Camera({ predictCanvas, predictions }: CameraProps) {
13 | const [devices, setDevices] = useState([]);
14 | const [deviceId, setDeviceId] = useState(undefined);
15 | const [imageFlip, setImageFlip] = useState(true);
16 | const webcamRef = useRef(null);
17 | const [selectorVisible, setSelectorVisible] = useState(false);
18 |
19 | // handle any webcam plugged into the computer
20 | // https://github.com/mozmorris/react-webcam#show-all-cameras-by-deviceid
21 | const handleDevices = useCallback(
22 | (mediaDevices: MediaDeviceInfo[]) => {
23 | // find all the webcams
24 | const videoDevices = mediaDevices.filter(({kind}) => kind === "videoinput");
25 | setDevices(videoDevices);
26 | // set our initial webcam to be the first in the list
27 | if (videoDevices.length > 0) {
28 | setDeviceId(videoDevices[0].deviceId);
29 | }
30 | },[setDevices, setDeviceId]
31 | );
32 | useEffect(() => {
33 | navigator.mediaDevices.enumerateDevices().then(handleDevices);
34 | }, [handleDevices]);
35 |
36 | // function to grab the current frame drawn on canvas from the webcam
37 | const getCanvas: () => Promise = useCallback(async () => {
38 | let newImage;
39 | if (webcamRef.current) {
40 | newImage = webcamRef.current.getCanvas();
41 | if (newImage) {
42 | return newImage;
43 | }
44 | }
45 | }, [webcamRef]);
46 |
47 | // helper for waiting in our loop when the camera is loading (getting the image)
48 | const sleep = useCallback((ms: number) => {
49 | return new Promise(function (resolve, reject) {
50 | setTimeout(resolve, ms);
51 | });
52 | }, []);
53 |
54 | // while we have the webcam mounted, predict frames as fast as we get new predictions back from the model
55 | useEffect(() => {
56 | getCanvas().then(async (canvas: HTMLCanvasElement | undefined) => {
57 | let currentCanvas = canvas;
58 | while (!currentCanvas) {
59 | // if no canvas, wait 500ms and try again
60 | await sleep(500);
61 | currentCanvas = await getCanvas();
62 | }
63 | if (currentCanvas) {
64 | predictCanvas(currentCanvas);
65 | }
66 | })
67 | }, [sleep, predictions, deviceId, getCanvas, predictCanvas])
68 |
69 | return (
70 |