├── README.md
└── index.html


/README.md:
--------------------------------------------------------------------------------
1 | An example of using MediaStreamTrack Insertable Streams (aka Breakout Box)
2 | for building a "smart gallery".
3 | 
4 | See https://webrtchacks.com/smart-gallery/
5 | 
6 | License: MIT
7 | 


--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
 1 | <style>
 2 | #localVideo {
 3 |     width: 640px;
 4 | }
 5 | #cropped video {
 6 |     width: 320px;
 7 | }
 8 | </style>
 9 | <script src="https://cdn.jsdelivr.net/npm/@mediapipe/face_detection/face_detection.js" crossorigin="anonymous"></script>
10 | <div>This is a demo using MediaStreamTrack Insertable Streams (a.k.a. Breakout Box) to detect faces (using the <a href="https://github.com/google/mediapipe">MediaPipe library</a>) and then crop them out.
11 | It allows splitting up a in-person meeting with multiple people in the same room into separate cropped streams similar to what is described <a href="https://blog.zoom.us/preparing-workspaces-for-hybrid-workforce-new-zoom-rooms-updates-2/">here</a>.
12 | </div>
13 | <div id="notsupported">This demo requires MediaStreamTrack Insertable Streams which requires experimental web platform features to be enabled in chrome://flags. It also needs OffscreenCanvas.</div>
14 | <video id="localVideo" autoplay playsinline></video>
15 | <br>
16 | <div id="cropped">
17 |     <video id="crop0" autoplay playsinline></video>
18 |     <video id="crop1" autoplay playsinline></video>
19 | </div>
20 | 
21 | <script>
22 | const canvas = new OffscreenCanvas(1, 1);
23 | const ctx = canvas.getContext('2d', {desynchronized: true});
24 | let faces;
25 | const width = 1280;
26 | const height = 720;
27 | const boxWidth = 640;
28 | const boxHeight = 360;
29 | 
30 | canvas.width = boxWidth;
31 | canvas.height = boxHeight;
32 | 
33 | function transform(frame, controller, index) {
34 |     // Cropping is a bit too complex. See https://github.com/w3c/webcodecs/issues/281
35 |     if (faces && faces.detections[index]) {
36 |         //console.log('DETECTED', faces.detections.length, faces.detections, frame);
37 |         const box = faces.detections[index].boundingBox;
38 |         ctx.drawImage(frame,
39 | 			Math.min(Math.max(Math.floor(box.xCenter * frame.codedWidth) - boxWidth / 2, 0), frame.codedWidth - boxWidth),
40 | 			Math.min(Math.max(Math.floor(box.yCenter * frame.codedHeight) - boxHeight / 2, 0), frame.codedHeight - boxHeight),
41 |             boxWidth, boxHeight,
42 |             0, 0, boxWidth, boxHeight);
43 |         const newFrame = new VideoFrame(canvas);
44 |         controller.enqueue(newFrame);
45 |     }
46 |     frame.close();
47 | }
48 | 
49 | const faceDetection = new FaceDetection({locateFile: (file) => {
50 |   return `https://cdn.jsdelivr.net/npm/@mediapipe/face_detection@0.4/${file}`;
51 | }});
52 | 
53 | faceDetection.setOptions({
54 |   model: 'short',
55 |   minDetectionConfidence: 0.5,
56 | });
57 | faceDetection.onResults(results => {
58 |     console.log('update results', results);
59 |     // For a real "product" one would have to track the different boxes and their
60 |     // movement to prevent jumps.
61 | 	faces = results;
62 | });
63 | 
64 | if (!(typeof MediaStreamTrackProcessor === 'undefined' || typeof MediaStreamTrackGenerator === 'undefined')) {
65 |     document.getElementById('notsupported').style.display = 'none';
66 |     navigator.mediaDevices.getUserMedia({video: {width, height}}).then(async (stream) => {
67 |         localVideo.srcObject = stream;
68 |         // Update face detection every two ѕeconds.
69 |         setInterval(() => {
70 |             faceDetection.send({image: document.getElementById('localVideo')});
71 |         }, 2000);
72 | 
73 |         const [track] = stream.getTracks();
74 |         const generator0 = new MediaStreamTrackGenerator({kind: 'video'});
75 |         generator0.addEventListener('mute', () => console.log('first generator muted'));
76 |         generator0.addEventListener('unmute', () => console.log('first generator unmuted'));
77 |         const generator1 = new MediaStreamTrackGenerator({kind: 'video'});
78 |         generator1.addEventListener('mute', () => console.log('second generator muted'));
79 |         generator1.addEventListener('unmute', () => console.log('second generator unmuted'));
80 |         crop0.srcObject = new MediaStream([generator0]);
81 |         crop1.srcObject = new MediaStream([generator1]);
82 | 
83 |         const processor0 = new MediaStreamTrackProcessor({track});
84 |         processor0.readable.pipeThrough(new TransformStream({
85 |             transform: (frame, controller) => transform(frame, controller, 0)
86 |         })).pipeTo(generator0.writable);
87 |         const processor1 = new MediaStreamTrackProcessor({track});
88 |         processor1.readable.pipeThrough(new TransformStream({
89 |             transform: (frame, controller) => transform(frame, controller, 1)
90 |         })).pipeTo(generator1.writable);
91 | 
92 |     });
93 | }
94 | </script>
95 | </body>
96 | </html>
97 | 


--------------------------------------------------------------------------------