33 |
34 |
36 | );
37 | }
38 | };
39 |
40 | ReactDOM.render(
41 |
34 |
48 |
49 |
50 |
51 |
52 |
--------------------------------------------------------------------------------
/app/src/client/video.js:
--------------------------------------------------------------------------------
1 | import React, { Component } from 'react';
2 | import MediaCapturer from 'react-multimedia-capture';
3 | import imageBus from './socket';
4 | import RaisedButton from 'material-ui/RaisedButton';
5 | import FaPlay from 'react-icons/lib/fa/play';
6 | import FaPause from 'react-icons/lib/fa/pause';
7 |
8 |
9 | class VideoExample extends React.Component {
10 | constructor() {
11 | super();
12 | this.state = {
13 | granted: false,
14 | rejectedReason: '',
15 | recording: false,
16 | paused: false,
17 | emotionData: {}
18 | };
19 |
20 | this.handleGranted = this.handleGranted.bind(this);
21 | this.handleDenied = this.handleDenied.bind(this);
22 | this.handleStart = this.handleStart.bind(this);
23 | this.handleStop = this.handleStop.bind(this);
24 | this.handlePause = this.handlePause.bind(this);
25 | this.handleResume = this.handleResume.bind(this);
26 | this.setStreamToVideo = this.setStreamToVideo.bind(this);
27 | this.releaseStreamFromVideo = this.releaseStreamFromVideo.bind(this);
28 | this.downloadVideo = this.downloadVideo.bind(this);
29 | }
30 | handleGranted() {
31 | this.setState({ granted: true });
32 | console.log('Permission Granted!');
33 | }
34 | handleDenied(err) {
35 | this.setState({ rejectedReason: err.name });
36 | console.log('Permission Denied!', err);
37 | }
38 | handleStart(stream) {
39 | this.setState({
40 | recording: true
41 | });
42 |
43 | this.setStreamToVideo(stream);
44 | console.log('Recording Started.');
45 | }
46 | handleStop(blob) {
47 | this.setState({
48 | recording: false
49 | });
50 |
51 | this.releaseStreamFromVideo();
52 |
53 | console.log('Recording Stopped.');
54 | this.downloadVideo(blob);
55 | }
56 | handlePause() {
57 | this.releaseStreamFromVideo();
58 |
59 | this.setState({
60 | paused: true
61 | });
62 | }
63 | handleResume(stream) {
64 | this.setStreamToVideo(stream);
65 |
66 | this.setState({
67 | paused: false
68 | });
69 | }
70 | handleError(err) {
71 | console.error(err);
72 | }
73 | setStreamToVideo(stream) {
74 | let video = this.refs.app.querySelector('video');
75 |
76 | if(window.URL) {
77 | video.src = window.URL.createObjectURL(stream);
78 | const track = stream.getVideoTracks()[0]
79 | this.captureFrame(track)
80 | setInterval(function(){ this.captureFrame(track) }.bind(this), 2000);
81 | }
82 | else {
83 | video.src = stream;
84 | }
85 | }
86 | postImageUpdateEmotionData(img64) {
87 | let data = {
88 | uri: img64,
89 | timestamp: new Date().getTime(),
90 | }
91 | imageBus(data, (err, results) => {
92 | if (err) {
93 | console.error(err);
94 | return;
95 | }
96 | // this.setState({emotionData: results})
97 | this.props.onDataPush(results)
98 | // console.log(results)
99 | })
100 | }
101 | captureFrame(track) {
102 | let imageCapture = new ImageCapture(track)
103 | imageCapture.grabFrame()
104 | .then(imageBitmap => {
105 | const canvas = document.getElementById('myCanvas');
106 | this.drawCanvas(canvas, imageBitmap);
107 | canvas.toBlob((blob)=>{
108 | this.postImageUpdateEmotionData(blob)
109 | }, 'image/jpeg');
110 | })
111 | .catch(error => console.error(error));
112 | }
113 | drawCanvas(canvas, img) {
114 | canvas.width = getComputedStyle(canvas).width.split('px')[0];
115 | canvas.height = getComputedStyle(canvas).height.split('px')[0];
116 | let ratio = Math.min(canvas.width / img.width, canvas.height / img.height);
117 | let x = (canvas.width - img.width * ratio) / 2;
118 | let y = (canvas.height - img.height * ratio) / 2;
119 | canvas.getContext('2d').clearRect(0, 0, canvas.width, canvas.height);
120 | canvas.getContext('2d').drawImage(img, 0, 0, img.width, img.height,
121 | x, y, img.width * ratio, img.height * ratio);
122 | }
123 | releaseStreamFromVideo() {
124 | this.refs.app.querySelector('video').src = '';
125 | }
126 | downloadVideo(blob) {
127 | let url = URL.createObjectURL(blob);
128 | let a = document.createElement('a');
129 | a.style.display = 'none';
130 | a.href = url;
131 | a.target = '_blank';
132 | document.body.appendChild(a);
133 |
134 | a.click();
135 | }
136 | render() {
137 | const granted = this.state.granted;
138 | const rejectedReason = this.state.rejectedReason;
139 | const recording = this.state.recording;
140 | const paused = this.state.paused;
141 |
142 | return (
143 |
35 |
47 |
36 |
37 |
40 | Sentimental Analysis
38 | 39 |
41 |
46 | Overall Reaction
42 |
43 | 
44 |
45 |
44 |
144 |
156 |
175 | );
176 | }
177 | }
178 |
179 | export default VideoExample;
180 |
--------------------------------------------------------------------------------
/app.js:
--------------------------------------------------------------------------------
1 | const express = require('express'),
2 | app = express(),
3 | logger = require('morgan'),
4 | errorHandler = require('errorhandler'),
5 | debug = require('debug')('server'),
6 | socketio = require('socket.io'),
7 | http = require('http'),
8 | cookieParser = require('cookie-parser'),
9 | bodyParser = require('body-parser'),
10 | utils = require('./app/src/utils'),
11 | consts = require('./consts'),
12 | axios = require('axios'),
13 | BASE_PATH = `${__dirname}/app`,
14 | ENV = process.env.NODE_ENV || 'development',
15 | DEFAULT_PORT = 3001,
16 | SOCKET_PORT = 8000,
17 | EMOTION_FPS = 5; // return the analysed data after n frames
18 |
19 | /* Configuration */
20 | app.set('views', `${BASE_PATH}/views`)
21 | app.engine('html', require('ejs').renderFile);
22 | app.set('view engine', 'html');
23 | app.use(bodyParser.json())
24 | app.use(bodyParser.urlencoded({ extended: false }))
25 | app.use(cookieParser())
26 | app.use('/assets', express.static(`${BASE_PATH}/public`))
27 |
28 | if (ENV === 'development') {
29 | console.log('DEVELOPMENT env')
30 | app.use(errorHandler({dumpExceptions: true, showStack: true}))
31 | app.use(logger('dev'))
32 | } else {
33 | console.log('PRODUCTION env')
34 | app.use(errorHandler())
35 | app.use(logger())
36 | }
37 |
38 | // app.locals = {
39 | // accEmotions: {
40 | // anger: 0,
41 | // contempt: 0,
42 | // disgust: 0,
43 | // fear: 0,
44 | // happiness: 0,
45 | // neutral: 0,
46 | // sadness: 0,
47 | // surprise: 0
48 | // },
49 | // lecturer: {
50 | // overall: {
51 | // emotion: '',
52 | // emotionData: []
53 | // },
54 | // lectures: [
55 | // {
56 | // date: '',
57 | // emotion: '',
58 | // emotionData: [],
59 | // series: [
60 | // {
61 | // time: '',
62 | // emotion: '',
63 | // emotionData: []
64 | // }
65 | // ]
66 | // }
67 | // ]
68 | // }
69 | // }
70 |
71 | app.locals = {
72 | accEmotions: {
73 | anger: 0,
74 | contempt: 0,
75 | disgust: 0,
76 | fear: 0,
77 | happiness: 0,
78 | neutral: 0,
79 | sadness: 0,
80 | surprise: 0
81 | },
82 | lecturer: {
83 | overall: {
84 | emotion: ''
85 | },
86 | lecture: {
87 | accFrames: EMOTION_FPS,
88 | date: '',
89 | emotion: '',
90 | timeline: [],
91 | count: 0,
92 | avgcount: 0
93 | }
94 | }
95 | }
96 |
97 | /**
98 | * Get port from environment and use it for Express.
99 | */
100 | const PORT = utils.normalizePort(process.env.PORT || DEFAULT_PORT)
101 | app.set('port', PORT)
102 |
103 | /**
104 | * Create HTTP server.
105 | */
106 | const server = http.createServer(app)
107 |
108 | /**
109 | * Listen on provided port, on all network interfaces.
110 | */
111 |
112 | server.listen(PORT)
113 |
114 | /**
115 | * Server event handling
116 | */
117 | server.on('error', (err) => {
118 | throw err
119 | })
120 | server.on('listening', (err) => {
121 | let addr = server.address()
122 | let bind = typeof addr === 'string' ?
123 | 'pipe ' + addr :
124 | 'port ' + addr.port
125 | debug('DeepEval is alive on ' + bind)
126 | })
127 |
128 | /**
129 | * Init websockets
130 | */
131 | const io = socketio(server);
132 | io.listen(SOCKET_PORT);
133 | console.log('Listening on SOCKET_PORT ', SOCKET_PORT);
134 |
135 | io.on('connection', (client) => {
136 | client.on('imagePost', (imgData) => {
137 | // console.log('timestamp:', imgData.timestamp)
138 |
139 | axios({
140 | method: 'post',
141 | url: consts.endpoints.faceAPI,
142 | data: imgData.uri,
143 | headers: {
144 | 'Content-Type': 'application/octet-stream',
145 | 'Ocp-Apim-Subscription-Key': process.env.SUB_KEY
146 | },
147 | params: {
148 | 'returnFaceId': 'true',
149 | 'returnFaceLandmarks': 'false',
150 | 'returnFaceAttributes': 'headPose,emotion,blur,exposure,noise',
151 | },
152 | })
153 | .then(function (response) {
154 | let faceData = response.data
155 | if (faceData.length <= 0) {
156 | // console.log(faceData.length)
157 | return;
158 | }
159 | // Only include attentive faces
160 | // let attentiveFaces = faceData.filter((item) => {
161 | // return parseInt(item['headPose']['pitch']) == 0;
162 | // });
163 | let attentiveFaces = faceData
164 | app.locals.lecturer.lecture.count++;
165 |
166 | // console.log(attentiveFaces)
167 | // console.log(app.locals.lecturer.lecture.count)
168 | let maxEmotionMapping = {
169 | anger: 1,
170 | sadness: 2,
171 | disgust: 3,
172 | fear: 4,
173 | neutral: 5,
174 | contempt: 6,
175 | surprise: 7,
176 | happiness: 8,
177 | }
178 | let maximumEmotion = 0
179 | let sum = {
180 | anger: 0,
181 | contempt: 0,
182 | disgust: 0,
183 | fear: 0,
184 | happiness: 0,
185 | neutral: 0,
186 | sadness: 0,
187 | surprise: 0,
188 | }
189 | let max = 0
190 |
191 | function avg(count, x1, x2) {
192 | return ((count-1)*x1+x2)/2
193 | }
194 |
195 | attentiveFaces.forEach(elem => {
196 | for (let prop in elem['faceAttributes']['emotion']) {
197 | app.locals.accEmotions[prop] = avg(app.locals.lecturer.lecture.count, app.locals.accEmotions[prop], elem['faceAttributes']['emotion'][prop])
198 | // console.log(prop)
199 | sum[prop] = sum[prop] + elem['faceAttributes']['emotion'][prop]
200 | if(elem['faceAttributes']['emotion'][prop] > max) {
201 | max = elem['faceAttributes']['emotion'][prop]
202 | maximumEmotion = maxEmotionMapping[prop]
203 | }
204 | }
205 | });
206 | console.log(require('util').inspect(plotData, { depth: null }));
207 |
208 |
209 | if (--app.locals.lecturer.lecture.accFrames < 0) {
210 | let plotData = {
211 | emotion: maximumEmotion,
212 | timestamp: imgData.timestamp
213 | }
214 |
215 | app.locals.lecturer.lecture.timeline.append(plotData)
216 | app.locals.lecturer.lecture.avgcount++;
217 | app.locals.lecturer.lecture.emotion = app.locals.lecturer.overall.emotion = avg(app.locals.lecturer.lecture.avgcount, app.locals.lecturer.lecture.emotion, maximumEmotion)
218 |
219 |
220 |
221 | // Reset the accumlated frames
222 | app.locals.lecturer.lecture.accFrames = EMOTION_FPS
223 |
224 | // send back off to client
225 | client.emit('results', plotData)
226 | }
227 |
228 | })
229 | .catch(function (error) {
230 | // console.error(error);
231 | });
232 | });
233 | });
234 |
235 |
236 | // const websocket = socketio(server)
237 |
238 | app.get('/', function (req, res) {
239 | res.render('index')
240 | })
241 |
242 | module.exports = app
243 |
--------------------------------------------------------------------------------
Video Recorder
145 |
157 | {/*
173 | } />
174 | Granted: {granted.toString()}
158 |Rejected Reason: {rejectedReason}
159 |Recording: {recording.toString()}
160 |Paused: {paused.toString()}
*/} 161 | 162 | {/*Live Stream: {'this.state.emotionData'}
*/} 163 | 164 | 165 |
166 |
169 |
170 | {/*