├── .gitignore ├── README.md ├── main.py ├── requirements.txt ├── src ├── __init__.py └── schemas.py ├── static ├── client.js └── client_cv.js └── templates ├── index.html └── index_cv.html /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | idea 3 | .DS_Store 4 | 5 | *.egg-info/ 6 | .installed.cfg 7 | *.egg 8 | 9 | dist 10 | db.sqlite3 11 | sqlite.db 12 | venv 13 | 14 | **/migrations/** 15 | !**/migrations 16 | !**/migrations/__init__.py 17 | 18 | __pycache__/ 19 | */__pycache__/ 20 | *.py[cod] 21 | .pyc 22 | *.pyc 23 | 24 | !*media/.gitkeep 25 | media/* 26 | 27 | log/debug.log 28 | 29 | build 30 | doc 31 | home.rst 32 | make.bat 33 | Makefile 34 | 35 | doc-dev 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Webrtc Opencv Fastapi 2 | Трансляция видео с веб-камеры на сервер, обработка и вывод в браузер с использованием: 3 | 4 | - webrtc 5 | - opencv 6 | - fastapi 7 | - aiortc 8 | - javascript 9 | 10 | ## Start (Старт) 11 | 12 | pip install -r requirements.txt 13 | uvicorn main:app 14 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | import cv2 4 | 5 | from av import VideoFrame 6 | 7 | from imageai.Detection import VideoObjectDetection 8 | 9 | from aiortc import MediaStreamTrack, RTCPeerConnection, RTCSessionDescription 10 | from aiortc.contrib.media import MediaPlayer, MediaRelay, MediaBlackhole 11 | 12 | from fastapi import FastAPI 13 | from fastapi.staticfiles import StaticFiles 14 | 15 | from starlette.requests import Request 16 | from starlette.responses import HTMLResponse 17 | from starlette.templating import Jinja2Templates 18 | 19 | from src.schemas import Offer 20 | 21 | ROOT = os.path.dirname(__file__) 22 | 23 | app = FastAPI() 24 | app.mount("/static", StaticFiles(directory="static"), name="static") 25 | templates = Jinja2Templates(directory="templates") 26 | 27 | faces = cv2.CascadeClassifier(cv2.data.haarcascades+"haarcascade_frontalface_default.xml") 28 | eyes = cv2.CascadeClassifier(cv2.data.haarcascades+"haarcascade_eye.xml") 29 | smiles = cv2.CascadeClassifier(cv2.data.haarcascades+"haarcascade_smile.xml") 30 | 31 | 32 | class VideoTransformTrack(MediaStreamTrack): 33 | """ 34 | A video stream track that transforms frames from an another track. 35 | """ 36 | 37 | kind = "video" 38 | 39 | def __init__(self, track, transform): 40 | super().__init__() 41 | self.track = track 42 | self.transform = transform 43 | 44 | async def recv(self): 45 | frame = await self.track.recv() 46 | 47 | if self.transform == "cartoon": 48 | img = frame.to_ndarray(format="bgr24") 49 | 50 | # prepare color 51 | img_color = cv2.pyrDown(cv2.pyrDown(img)) 52 | for _ in range(6): 53 | img_color = cv2.bilateralFilter(img_color, 9, 9, 7) 54 | img_color = cv2.pyrUp(cv2.pyrUp(img_color)) 55 | 56 | # prepare edges 57 | img_edges = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) 58 | img_edges = cv2.adaptiveThreshold( 59 | cv2.medianBlur(img_edges, 7), 60 | 255, 61 | cv2.ADAPTIVE_THRESH_MEAN_C, 62 | cv2.THRESH_BINARY, 63 | 9, 64 | 2, 65 | ) 66 | img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB) 67 | 68 | # combine color and edges 69 | img = cv2.bitwise_and(img_color, img_edges) 70 | 71 | # rebuild a VideoFrame, preserving timing information 72 | new_frame = VideoFrame.from_ndarray(img, format="bgr24") 73 | new_frame.pts = frame.pts 74 | new_frame.time_base = frame.time_base 75 | return new_frame 76 | elif self.transform == "edges": 77 | # perform edge detection 78 | img = frame.to_ndarray(format="bgr24") 79 | img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR) 80 | 81 | # rebuild a VideoFrame, preserving timing information 82 | new_frame = VideoFrame.from_ndarray(img, format="bgr24") 83 | new_frame.pts = frame.pts 84 | new_frame.time_base = frame.time_base 85 | return new_frame 86 | elif self.transform == "rotate": 87 | # rotate image 88 | img = frame.to_ndarray(format="bgr24") 89 | rows, cols, _ = img.shape 90 | M = cv2.getRotationMatrix2D((cols / 2, rows / 2), frame.time * 45, 1) 91 | img = cv2.warpAffine(img, M, (cols, rows)) 92 | 93 | # rebuild a VideoFrame, preserving timing information 94 | new_frame = VideoFrame.from_ndarray(img, format="bgr24") 95 | new_frame.pts = frame.pts 96 | new_frame.time_base = frame.time_base 97 | return new_frame 98 | elif self.transform == "cv": 99 | img = frame.to_ndarray(format="bgr24") 100 | face = faces.detectMultiScale(img, 1.1, 19) 101 | for (x, y, w, h) in face: 102 | cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 2) 103 | 104 | eye = eyes.detectMultiScale(img, 1.1, 19) 105 | for (x, y, w, h) in eye: 106 | cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 2) 107 | 108 | # smile = smiles.detectMultiScale(img, 1.1, 19) 109 | # for (x, y, w, h) in smile: 110 | # cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 5), 2) 111 | 112 | new_frame = VideoFrame.from_ndarray(img, format="bgr24") 113 | new_frame.pts = frame.pts 114 | new_frame.time_base = frame.time_base 115 | return new_frame 116 | else: 117 | return frame 118 | 119 | 120 | def create_local_tracks(play_from=None): 121 | if play_from: 122 | player = MediaPlayer(play_from) 123 | return player.audio, player.video 124 | else: 125 | options = {"framerate": "30", "video_size": "1920x1080"} 126 | # if relay is None: 127 | # if platform.system() == "Darwin": 128 | # webcam = MediaPlayer( 129 | # "default:none", format="avfoundation", options=options 130 | # ) 131 | # elif platform.system() == "Windows": 132 | # webcam = MediaPlayer("video.mp4") 133 | webcam = MediaPlayer( 134 | "video=FULL HD 1080P Webcam", format="dshow", options=options 135 | ) 136 | 137 | 138 | # else: 139 | # webcam = MediaPlayer("/dev/video0", format="v4l2", options=options) 140 | # audio, video = VideoTransformTrack(webcam.video, transform="cv") 141 | relay = MediaRelay() 142 | return None, relay.subscribe(webcam.video) 143 | 144 | 145 | @app.get("/", response_class=HTMLResponse) 146 | async def index(request: Request): 147 | return templates.TemplateResponse("index.html", {"request": request}) 148 | 149 | 150 | @app.get("/cv", response_class=HTMLResponse) 151 | async def index(request: Request): 152 | return templates.TemplateResponse("index_cv.html", {"request": request}) 153 | 154 | 155 | @app.post("/offer") 156 | async def offer(params: Offer): 157 | offer = RTCSessionDescription(sdp=params.sdp, type=params.type) 158 | 159 | pc = RTCPeerConnection() 160 | pcs.add(pc) 161 | recorder = MediaBlackhole() 162 | 163 | @pc.on("connectionstatechange") 164 | async def on_connectionstatechange(): 165 | print("Connection state is %s" % pc.connectionState) 166 | if pc.connectionState == "failed": 167 | await pc.close() 168 | pcs.discard(pc) 169 | 170 | # open media source 171 | audio, video = create_local_tracks() 172 | 173 | # handle offer 174 | await pc.setRemoteDescription(offer) 175 | await recorder.start() 176 | 177 | # send answer 178 | answer = await pc.createAnswer() 179 | 180 | await pc.setRemoteDescription(offer) 181 | for t in pc.getTransceivers(): 182 | if t.kind == "audio" and audio: 183 | pc.addTrack(audio) 184 | elif t.kind == "video" and video: 185 | pc.addTrack(video) 186 | 187 | await pc.setLocalDescription(answer) 188 | 189 | return {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type} 190 | 191 | 192 | @app.post("/offer_cv") 193 | async def offer(params: Offer): 194 | offer = RTCSessionDescription(sdp=params.sdp, type=params.type) 195 | 196 | pc = RTCPeerConnection() 197 | pcs.add(pc) 198 | recorder = MediaBlackhole() 199 | 200 | relay = MediaRelay() 201 | 202 | @pc.on("connectionstatechange") 203 | async def on_connectionstatechange(): 204 | print("Connection state is %s" % pc.connectionState) 205 | if pc.connectionState == "failed": 206 | await pc.close() 207 | pcs.discard(pc) 208 | 209 | # open media source 210 | # audio, video = create_local_tracks() 211 | 212 | @pc.on("track") 213 | def on_track(track): 214 | 215 | # if track.kind == "audio": 216 | # pc.addTrack(player.audio) 217 | # recorder.addTrack(track) 218 | if track.kind == "video": 219 | pc.addTrack( 220 | VideoTransformTrack(relay.subscribe(track), transform=params.video_transform) 221 | ) 222 | # if args.record_to: 223 | # recorder.addTrack(relay.subscribe(track)) 224 | 225 | @track.on("ended") 226 | async def on_ended(): 227 | await recorder.stop() 228 | 229 | # handle offer 230 | await pc.setRemoteDescription(offer) 231 | await recorder.start() 232 | 233 | # send answer 234 | answer = await pc.createAnswer() 235 | await pc.setRemoteDescription(offer) 236 | await pc.setLocalDescription(answer) 237 | 238 | return {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type} 239 | 240 | 241 | pcs = set() 242 | args = '' 243 | 244 | 245 | @app.on_event("shutdown") 246 | async def on_shutdown(): 247 | # close peer connections 248 | coros = [pc.close() for pc in pcs] 249 | await asyncio.gather(*coros) 250 | pcs.clear() 251 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DJWOMS/webrtc_opencv_fastapi/fd2303115a2d27d19c1c766185f1caa531bf7927/requirements.txt -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DJWOMS/webrtc_opencv_fastapi/fd2303115a2d27d19c1c766185f1caa531bf7927/src/__init__.py -------------------------------------------------------------------------------- /src/schemas.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class Offer(BaseModel): 5 | sdp: str 6 | type: str 7 | video_transform: str = None 8 | -------------------------------------------------------------------------------- /static/client.js: -------------------------------------------------------------------------------- 1 | var pc = null; 2 | 3 | function negotiate() { 4 | pc.addTransceiver('video', {direction: 'recvonly'}); 5 | pc.addTransceiver('audio', {direction: 'recvonly'}); 6 | return pc.createOffer().then(function(offer) { 7 | return pc.setLocalDescription(offer); 8 | }).then(function() { 9 | // wait for ICE gathering to complete 10 | return new Promise(function(resolve) { 11 | if (pc.iceGatheringState === 'complete') { 12 | resolve(); 13 | } else { 14 | function checkState() { 15 | if (pc.iceGatheringState === 'complete') { 16 | pc.removeEventListener('icegatheringstatechange', checkState); 17 | resolve(); 18 | } 19 | } 20 | pc.addEventListener('icegatheringstatechange', checkState); 21 | } 22 | }); 23 | }).then(function() { 24 | var offer = pc.localDescription; 25 | return fetch('/offer', { 26 | body: JSON.stringify({ 27 | sdp: offer.sdp, 28 | type: offer.type, 29 | }), 30 | headers: { 31 | 'Content-Type': 'application/json' 32 | }, 33 | method: 'POST' 34 | }); 35 | }).then(function(response) { 36 | return response.json(); 37 | }).then(function(answer) { 38 | return pc.setRemoteDescription(answer); 39 | }).catch(function(e) { 40 | alert(e); 41 | }); 42 | } 43 | 44 | function start() { 45 | var config = { 46 | sdpSemantics: 'unified-plan' 47 | }; 48 | 49 | if (document.getElementById('use-stun').checked) { 50 | config.iceServers = [{urls: ['stun:stun.l.google.com:19302']}]; 51 | } 52 | 53 | pc = new RTCPeerConnection(config); 54 | 55 | // connect audio / video 56 | pc.addEventListener('track', function(evt) { 57 | if (evt.track.kind == 'video') { 58 | document.getElementById('video').srcObject = evt.streams[0]; 59 | } else { 60 | document.getElementById('audio').srcObject = evt.streams[0]; 61 | } 62 | }); 63 | 64 | document.getElementById('start').style.display = 'none'; 65 | negotiate(); 66 | document.getElementById('stop').style.display = 'inline-block'; 67 | } 68 | 69 | function stop() { 70 | document.getElementById('stop').style.display = 'none'; 71 | document.getElementById('start').style.display = 'inline-block'; 72 | // close peer connection 73 | setTimeout(function() { 74 | pc.close(); 75 | }, 500); 76 | } 77 | -------------------------------------------------------------------------------- /static/client_cv.js: -------------------------------------------------------------------------------- 1 | // get DOM elements 2 | var dataChannelLog = document.getElementById('data-channel'), 3 | iceConnectionLog = document.getElementById('ice-connection-state'), 4 | iceGatheringLog = document.getElementById('ice-gathering-state'), 5 | signalingLog = document.getElementById('signaling-state'); 6 | 7 | // peer connection 8 | var pc = null; 9 | 10 | // data channel 11 | var dc = null, dcInterval = null; 12 | 13 | function createPeerConnection() { 14 | var config = { 15 | sdpSemantics: 'unified-plan' 16 | }; 17 | 18 | if (document.getElementById('use-stun').checked) { 19 | config.iceServers = [{urls: ['stun:stun.l.google.com:19302']}]; 20 | } 21 | 22 | pc = new RTCPeerConnection(config); 23 | 24 | // register some listeners to help debugging 25 | pc.addEventListener('icegatheringstatechange', function() { 26 | iceGatheringLog.textContent += ' -> ' + pc.iceGatheringState; 27 | }, false); 28 | iceGatheringLog.textContent = pc.iceGatheringState; 29 | 30 | pc.addEventListener('iceconnectionstatechange', function() { 31 | iceConnectionLog.textContent += ' -> ' + pc.iceConnectionState; 32 | }, false); 33 | iceConnectionLog.textContent = pc.iceConnectionState; 34 | 35 | pc.addEventListener('signalingstatechange', function() { 36 | signalingLog.textContent += ' -> ' + pc.signalingState; 37 | }, false); 38 | signalingLog.textContent = pc.signalingState; 39 | 40 | // connect audio / video 41 | pc.addEventListener('track', function(evt) { 42 | if (evt.track.kind == 'video') 43 | document.getElementById('video').srcObject = evt.streams[0]; 44 | else 45 | document.getElementById('audio').srcObject = evt.streams[0]; 46 | }); 47 | 48 | return pc; 49 | } 50 | 51 | function negotiate() { 52 | return pc.createOffer().then(function(offer) { 53 | return pc.setLocalDescription(offer); 54 | }).then(function() { 55 | // wait for ICE gathering to complete 56 | return new Promise(function(resolve) { 57 | if (pc.iceGatheringState === 'complete') { 58 | resolve(); 59 | } else { 60 | function checkState() { 61 | if (pc.iceGatheringState === 'complete') { 62 | pc.removeEventListener('icegatheringstatechange', checkState); 63 | resolve(); 64 | } 65 | } 66 | pc.addEventListener('icegatheringstatechange', checkState); 67 | } 68 | }); 69 | }).then(function() { 70 | var offer = pc.localDescription; 71 | var codec; 72 | 73 | codec = document.getElementById('audio-codec').value; 74 | if (codec !== 'default') { 75 | offer.sdp = sdpFilterCodec('audio', codec, offer.sdp); 76 | } 77 | 78 | codec = document.getElementById('video-codec').value; 79 | if (codec !== 'default') { 80 | offer.sdp = sdpFilterCodec('video', codec, offer.sdp); 81 | } 82 | 83 | document.getElementById('offer-sdp').textContent = offer.sdp; 84 | return fetch('/offer_cv', { 85 | body: JSON.stringify({ 86 | sdp: offer.sdp, 87 | type: offer.type, 88 | video_transform: document.getElementById('video-transform').value 89 | }), 90 | headers: { 91 | 'Content-Type': 'application/json' 92 | }, 93 | method: 'POST' 94 | }); 95 | }).then(function(response) { 96 | return response.json(); 97 | }).then(function(answer) { 98 | document.getElementById('answer-sdp').textContent = answer.sdp; 99 | return pc.setRemoteDescription(answer); 100 | }).catch(function(e) { 101 | alert(e); 102 | }); 103 | } 104 | 105 | function start() { 106 | document.getElementById('start').style.display = 'none'; 107 | 108 | pc = createPeerConnection(); 109 | 110 | var time_start = null; 111 | 112 | function current_stamp() { 113 | if (time_start === null) { 114 | time_start = new Date().getTime(); 115 | return 0; 116 | } else { 117 | return new Date().getTime() - time_start; 118 | } 119 | } 120 | 121 | if (document.getElementById('use-datachannel').checked) { 122 | var parameters = JSON.parse(document.getElementById('datachannel-parameters').value); 123 | 124 | dc = pc.createDataChannel('chat', parameters); 125 | dc.onclose = function() { 126 | clearInterval(dcInterval); 127 | dataChannelLog.textContent += '- close\n'; 128 | }; 129 | dc.onopen = function() { 130 | dataChannelLog.textContent += '- open\n'; 131 | dcInterval = setInterval(function() { 132 | var message = 'ping ' + current_stamp(); 133 | dataChannelLog.textContent += '> ' + message + '\n'; 134 | dc.send(message); 135 | }, 1000); 136 | }; 137 | dc.onmessage = function(evt) { 138 | dataChannelLog.textContent += '< ' + evt.data + '\n'; 139 | 140 | if (evt.data.substring(0, 4) === 'pong') { 141 | var elapsed_ms = current_stamp() - parseInt(evt.data.substring(5), 10); 142 | dataChannelLog.textContent += ' RTT ' + elapsed_ms + ' ms\n'; 143 | } 144 | }; 145 | } 146 | 147 | var constraints = { 148 | audio: document.getElementById('use-audio').checked, 149 | video: false 150 | }; 151 | 152 | if (document.getElementById('use-video').checked) { 153 | var resolution = document.getElementById('video-resolution').value; 154 | if (resolution) { 155 | resolution = resolution.split('x'); 156 | constraints.video = { 157 | width: parseInt(resolution[0], 0), 158 | height: parseInt(resolution[1], 0) 159 | }; 160 | } else { 161 | constraints.video = true; 162 | } 163 | } 164 | 165 | if (constraints.audio || constraints.video) { 166 | if (constraints.video) { 167 | document.getElementById('media').style.display = 'block'; 168 | } 169 | navigator.mediaDevices.getUserMedia(constraints).then(function(stream) { 170 | stream.getTracks().forEach(function(track) { 171 | pc.addTrack(track, stream); 172 | }); 173 | return negotiate(); 174 | }, function(err) { 175 | alert('Could not acquire media: ' + err); 176 | }); 177 | } else { 178 | negotiate(); 179 | } 180 | 181 | document.getElementById('stop').style.display = 'inline-block'; 182 | } 183 | 184 | function stop() { 185 | document.getElementById('stop').style.display = 'none'; 186 | document.getElementById('start').style.display = 'inline-block'; 187 | // close data channel 188 | if (dc) { 189 | dc.close(); 190 | } 191 | 192 | // close transceivers 193 | if (pc.getTransceivers) { 194 | pc.getTransceivers().forEach(function(transceiver) { 195 | if (transceiver.stop) { 196 | transceiver.stop(); 197 | } 198 | }); 199 | } 200 | 201 | // close local audio / video 202 | pc.getSenders().forEach(function(sender) { 203 | sender.track.stop(); 204 | }); 205 | 206 | // close peer connection 207 | setTimeout(function() { 208 | pc.close(); 209 | }, 500); 210 | } 211 | 212 | function sdpFilterCodec(kind, codec, realSdp) { 213 | var allowed = [] 214 | var rtxRegex = new RegExp('a=fmtp:(\\d+) apt=(\\d+)\r$'); 215 | var codecRegex = new RegExp('a=rtpmap:([0-9]+) ' + escapeRegExp(codec)) 216 | var videoRegex = new RegExp('(m=' + kind + ' .*?)( ([0-9]+))*\\s*$') 217 | 218 | var lines = realSdp.split('\n'); 219 | 220 | var isKind = false; 221 | for (var i = 0; i < lines.length; i++) { 222 | if (lines[i].startsWith('m=' + kind + ' ')) { 223 | isKind = true; 224 | } else if (lines[i].startsWith('m=')) { 225 | isKind = false; 226 | } 227 | 228 | if (isKind) { 229 | var match = lines[i].match(codecRegex); 230 | if (match) { 231 | allowed.push(parseInt(match[1])); 232 | } 233 | 234 | match = lines[i].match(rtxRegex); 235 | if (match && allowed.includes(parseInt(match[2]))) { 236 | allowed.push(parseInt(match[1])); 237 | } 238 | } 239 | } 240 | 241 | var skipRegex = 'a=(fmtp|rtcp-fb|rtpmap):([0-9]+)'; 242 | var sdp = ''; 243 | 244 | isKind = false; 245 | for (var i = 0; i < lines.length; i++) { 246 | if (lines[i].startsWith('m=' + kind + ' ')) { 247 | isKind = true; 248 | } else if (lines[i].startsWith('m=')) { 249 | isKind = false; 250 | } 251 | 252 | if (isKind) { 253 | var skipMatch = lines[i].match(skipRegex); 254 | if (skipMatch && !allowed.includes(parseInt(skipMatch[2]))) { 255 | continue; 256 | } else if (lines[i].match(videoRegex)) { 257 | sdp += lines[i].replace(videoRegex, '$1 ' + allowed.join(' ')) + '\n'; 258 | } else { 259 | sdp += lines[i] + '\n'; 260 | } 261 | } else { 262 | sdp += lines[i] + '\n'; 263 | } 264 | } 265 | 266 | return sdp; 267 | } 268 | 269 | function escapeRegExp(string) { 270 | return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string 271 | } 272 | -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | WebRTC webcam 7 | 24 | 25 | 26 | 27 |
28 | 29 | 30 |
31 | 32 | 33 | 34 |
35 |

Media

36 | 37 | 38 | 39 |
40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /templates/index_cv.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | WebRTC demo 7 | 29 | 30 | 31 | 32 |

Options

33 |
34 | 35 | 36 | 41 |
42 |
43 | 44 | 45 | 51 |
52 |
53 | 54 | 55 | 62 | 69 | 74 |
75 |
76 | 77 | 78 |
79 | 80 | 81 | 82 | 83 |

State

84 |

85 | ICE gathering state: 86 |

87 |

88 | ICE connection state: 89 |

90 |

91 | Signaling state: 92 |

93 | 94 | 100 | 101 |

Data channel

102 | 

103 | 
104 | 
SDP

105 | 
106 | 
Offer

107 | 

108 | 
109 | 
Answer

110 | 

111 | 
112 | 
113 | 
114 | 
115 | 


--------------------------------------------------------------------------------