├── README.md ├── colabrtc ├── call.py ├── js │ ├── peer-ui.js │ ├── peer.js │ └── signaling.js ├── peer.py ├── server.py └── signaling.py ├── examples ├── avatarify_colab.py ├── colabrtc.ipynb └── install_avatarify.sh ├── install.sh └── requirements.txt /README.md: -------------------------------------------------------------------------------- 1 | ColabRTC 2 | ========== 3 | 4 | `ColabRTC` contains early experiments to stream media 5 | from your machine to a Google Colab instance using 6 | `Web Real-Time Communication (WebRTC)`. It is built on top of 7 | [aiortc](https://github.com/aiortc/aiortc). 8 | 9 | The motivation behind this project was to learn about WebRTC 10 | and because an efficient implementation of this kind of communication 11 | could be useful to prototype Deep Learning models without a local 12 | GPU. 13 | 14 | How to use ``ColabRTC``? 15 | ------------------------ 16 | 17 | The API allows one to create a `ColabCall`, which consists of a 18 | Python peer (running on Colab) and a Javascript peer 19 | (started in a Colab cell HTML context, which runs on the 20 | local machine). The signaling medium is the Colab filesystem 21 | (and Google Drive, optionally). 22 | 23 | An example: 24 | 25 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/thefonseca/colabrtc/blob/master/examples/colabrtc.ipynb) 26 | 27 | ``` 28 | from call import ColabCall 29 | import cv2 30 | 31 | # Define a frame processing routine 32 | def process_frame(frame, frame_idx): 33 | # To speed up, apply processing every two frames 34 | if frame_idx % 2 == 1: 35 | edges = cv2.Canny(frame, 100, 200) 36 | return cv2.cvtColor(edges, cv2.COLOR_GRAY2BGR) 37 | 38 | call = ColabCall() 39 | # Create the Python peer 40 | call.create(frame_transformer=process_frame) 41 | # Create the Javascript peer and display the UI 42 | call.join() 43 | ``` 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /colabrtc/call.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import nest_asyncio 4 | from IPython.display import display, Javascript 5 | from google.colab.output import eval_js 6 | 7 | from signaling import ColabSignaling, ColabApprtcSignaling 8 | from peer import start_peer 9 | 10 | nest_asyncio.apply() 11 | 12 | 13 | class ColabCall: 14 | def __init__(self, signaling_js='js/signaling.js', peer_js='js/peer.js', 15 | peer_ui_js='js/peer-ui.js'): 16 | 17 | file_path = os.path.dirname(os.path.abspath(__file__)) 18 | js_files = [signaling_js, peer_js, peer_ui_js] 19 | js_content = [] 20 | 21 | for js_file in js_files: 22 | with open(os.path.join(file_path, js_file), 'r') as js: 23 | js_content.append(js.read()) 24 | 25 | js_content.append(''' 26 | var start_js_peer = function(room) { 27 | new PeerUI(room); 28 | } 29 | ''') 30 | self._js = ' '.join(js_content) 31 | self._peer_process = None 32 | self.room = None 33 | self.signaling_folder = None 34 | 35 | def create(self, room=None, signaling_folder='/content/webrtc', 36 | frame_transformer=None, verbose=False, multiprocess=True): 37 | 38 | self.end() 39 | 40 | room, proc = start_peer(room, signaling_folder=signaling_folder, 41 | frame_transformer=frame_transformer, 42 | verbose=verbose, multiprocess=multiprocess) 43 | self._peer_process = proc 44 | self.room = room 45 | self.signaling_folder = signaling_folder 46 | self.js_signaling = None 47 | 48 | def join(self, room=None, signaling_folder='/content/webrtc', verbose=False): 49 | if self.room is None and room is None: 50 | raise ValueError('A room parameter must be specified') 51 | elif self.room: 52 | room = self.room 53 | 54 | if self.signaling_folder and self.signaling_folder != signaling_folder: 55 | signaling_folder = self.signaling_folder 56 | 57 | if signaling_folder: 58 | self.js_signaling = ColabSignaling(signaling_folder=signaling_folder, 59 | room=room, javacript_callable=True) 60 | else: 61 | self.js_signaling = ColabApprtcSignaling(room=room, javacript_callable=True) 62 | 63 | display(Javascript(self._js)) 64 | eval_js(f'start_js_peer("{room}")') 65 | 66 | def end(self): 67 | if self._peer_process: 68 | self._peer_process.terminate() 69 | self._peer_process.join() -------------------------------------------------------------------------------- /colabrtc/js/peer-ui.js: -------------------------------------------------------------------------------- 1 | var PeerUI = function(room, container_id) { 2 | // Define initial start time of the call (defined as connection between peers). 3 | startTime = null; 4 | constraints = {audio: false, video: true}; 5 | 6 | let peerDiv = null; 7 | 8 | if (container_id) { 9 | peerDiv = document.getElementById(container_id); 10 | } else { 11 | peerDiv = document.createElement('div'); 12 | document.body.appendChild(peerDiv); 13 | } 14 | 15 | var style = document.createElement('style'); 16 | style.type = 'text/css'; 17 | style.innerHTML = ` 18 | .loader { 19 | position: absolute; 20 | left: 38%; 21 | top: 60%; 22 | z-index: 1; 23 | width: 50px; 24 | height: 50px; 25 | margin: -75px 0 0 -75px; 26 | border: 16px solid #f3f3f3; 27 | border-radius: 50%; 28 | border-top: 16px solid #3498db; 29 | -webkit-animation: spin 2s linear infinite; 30 | animation: spin 2s linear infinite; 31 | } 32 | 33 | @keyframes spin { 34 | 0% { transform: rotate(0deg); } 35 | 100% { transform: rotate(360deg); } 36 | } 37 | `; 38 | document.getElementsByTagName('head')[0].appendChild(style); 39 | 40 | var adapter = document.createElement('script'); 41 | adapter.setAttribute('src','https://webrtc.github.io/adapter/adapter-latest.js'); 42 | document.getElementsByTagName('head')[0].appendChild(adapter); 43 | 44 | peerDiv.style.width = '70%'; 45 | 46 | // Define video elements. 47 | const videoDiv = document.createElement('div'); 48 | videoDiv.style.display = 'none'; 49 | videoDiv.style.textAlign = '-webkit-center'; 50 | const localView = document.createElement('video'); 51 | const remoteView = document.createElement('video'); 52 | remoteView.autoplay = true; 53 | localView.style.display = 'block'; 54 | remoteView.style.display = 'block'; 55 | localView.height = 240; 56 | localView.width = 320; 57 | remoteView.height = 240; 58 | remoteView.width = 320; 59 | videoDiv.appendChild(localView); 60 | videoDiv.appendChild(remoteView); 61 | const loader = document.createElement('div'); 62 | loader.style.display = 'none'; 63 | loader.className = 'loader'; 64 | videoDiv.appendChild(loader); 65 | 66 | // Logs a message with the id and size of a video element. 67 | function logVideoLoaded(event) { 68 | const video = event.target; 69 | trace(`${video.id} videoWidth: ${video.videoWidth}px, ` + 70 | `videoHeight: ${video.videoHeight}px.`); 71 | 72 | localView.style.width = '20%'; 73 | localView.style.position = 'absolute'; 74 | remoteView.style.display = 'block'; 75 | remoteView.style.width = '100%'; 76 | remoteView.style.height = 'auto'; 77 | loader.style.display = 'none'; 78 | fullscreenButton.style.display = 'inline'; 79 | } 80 | 81 | //localView.addEventListener('loadedmetadata', logVideoLoaded); 82 | remoteView.addEventListener('loadedmetadata', logVideoLoaded); 83 | //remoteView.addEventListener('onresize', logResizedVideo); 84 | 85 | // Define action buttons. 86 | const controlDiv = document.createElement('div'); 87 | controlDiv.style.textAlign = 'center'; 88 | const startButton = document.createElement('button'); 89 | const fullscreenButton = document.createElement('button'); 90 | const hangupButton = document.createElement('button'); 91 | startButton.textContent = 'Join room: ' + room; 92 | fullscreenButton.textContent = 'Fullscreen'; 93 | hangupButton.textContent = 'Hangup'; 94 | controlDiv.appendChild(startButton); 95 | controlDiv.appendChild(fullscreenButton); 96 | controlDiv.appendChild(hangupButton); 97 | 98 | // Set up initial action buttons status: disable call and hangup. 99 | //callButton.disabled = true; 100 | hangupButton.style.display = 'none'; 101 | fullscreenButton.style.display = 'none'; 102 | 103 | peerDiv.appendChild(videoDiv); 104 | peerDiv.appendChild(controlDiv); 105 | 106 | this.localView = localView; 107 | this.remoteView = remoteView; 108 | this.peerDiv = peerDiv; 109 | this.videoDiv = videoDiv; 110 | this.loader = loader; 111 | this.startButton = startButton; 112 | this.fullscreenButton = fullscreenButton; 113 | this.hangupButton = hangupButton; 114 | this.constraints = constraints; 115 | this.room = room; 116 | 117 | self = this; 118 | async function start() { 119 | await self.connect(this.room); 120 | } 121 | 122 | // Handles hangup action: ends up call, closes connections and resets peers. 123 | async function hangup() { 124 | await self.disconnect(); 125 | } 126 | 127 | function openFullscreen() { 128 | let elem = remoteView; 129 | if (elem.requestFullscreen) { 130 | elem.requestFullscreen(); 131 | } else if (elem.mozRequestFullScreen) { /* Firefox */ 132 | elem.mozRequestFullScreen(); 133 | } else if (elem.webkitRequestFullscreen) { /* Chrome, Safari & Opera */ 134 | elem.webkitRequestFullscreen(); 135 | } else if (elem.msRequestFullscreen) { /* IE/Edge */ 136 | elem.msRequestFullscreen(); 137 | } 138 | } 139 | 140 | // Add click event handlers for buttons. 141 | this.startButton.addEventListener('click', start); 142 | this.fullscreenButton.addEventListener('click', openFullscreen); 143 | this.hangupButton.addEventListener('click', hangup); 144 | }; 145 | 146 | 147 | PeerUI.prototype.connect = async function(room) { 148 | //startButton.disabled = true; 149 | const stream = await navigator.mediaDevices.getUserMedia(constraints); 150 | this.localView.srcObject = stream; 151 | this.localView.play(); 152 | trace('Received local stream.'); 153 | 154 | this.loader.style.display = 'block'; 155 | this.startButton.style.display = 'none'; 156 | this.localView.style.width = '100%'; 157 | this.localView.style.height = 'auto'; 158 | this.localView.style.position = 'relative'; 159 | this.remoteView.style.display = 'none'; 160 | this.videoDiv.style.display = 'block'; 161 | 162 | if (google) { 163 | // Resize the output to fit the video element. 164 | google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true); 165 | } 166 | 167 | try { 168 | //this.joinButton.style.display = 'none'; 169 | this.hangupButton.style.display = 'inline'; 170 | 171 | trace('Starting call.'); 172 | this.startTime = window.performance.now(); 173 | 174 | this.peer = new Peer(); 175 | await this.peer.connect(this.room); 176 | //const obj = JSON.stringify([this.peer.connect, this.room]); 177 | //this.worker.postMessage([this.peer, this.room]); 178 | 179 | this.peer.pc.ontrack = ({track, streams}) => { 180 | // once media for a remote track arrives, show it in the remote video element 181 | track.onunmute = () => { 182 | // don't set srcObject again if it is already set. 183 | if (this.remoteView.srcObject) return; 184 | console.log(streams); 185 | this.remoteView.srcObject = streams[0]; 186 | trace('Remote peer connection received remote stream.'); 187 | this.remoteView.play(); 188 | }; 189 | }; 190 | 191 | const localStream = this.localView.srcObject; 192 | console.log('adding local stream'); 193 | await this.peer.addLocalStream(localStream); 194 | 195 | await this.peer.waitMessage(); 196 | 197 | } catch (err) { 198 | console.error(err); 199 | } 200 | }; 201 | 202 | PeerUI.prototype.disconnect = async function() { 203 | await this.peer.disconnect(); 204 | this.startButton.style.display = 'inline'; 205 | //this.joinButton.style.display = 'inline'; 206 | this.hangupButton.style.display = 'none'; 207 | this.fullscreenButton.style.display = 'none'; 208 | this.videoDiv.style.display = 'none'; 209 | 210 | trace('Ending call.'); 211 | this.localView.srcObject.getVideoTracks()[0].stop(); 212 | this.peerDiv.remove(); 213 | }; 214 | 215 | // Logs an action (text) and the time when it happened on the console. 216 | function trace(text) { 217 | text = text.trim(); 218 | const now = (window.performance.now() / 1000).toFixed(3); 219 | console.log(now, text); 220 | } -------------------------------------------------------------------------------- /colabrtc/js/peer.js: -------------------------------------------------------------------------------- 1 | var Peer = function(room, configuration, polite=true) { 2 | this.makingOffer = false; 3 | this.ignoreOffer = false; 4 | this.polite = polite; 5 | this.timeout = null; 6 | }; 7 | 8 | Peer.prototype.connect = async function(room, configuration) { 9 | 10 | if (this.pc) { 11 | await this.disconnect(); 12 | } 13 | 14 | if (!configuration) { 15 | configuration = { 16 | iceServers: [{urls: 'stun:stun.l.google.com:19302'}] 17 | }; 18 | } 19 | 20 | pc = new RTCPeerConnection(configuration); 21 | signaling = new SignalingChannel(room); 22 | this.signaling = signaling; 23 | this.pc = pc; 24 | 25 | // send any ice candidates to the other peer 26 | pc.onicecandidate = async (event) => { 27 | if (event.candidate) { 28 | trace('Sending ICE candidate'); 29 | console.log(event); 30 | event.candidate.type = 'candidate' 31 | await this.signaling.send(event.candidate); 32 | } 33 | } 34 | 35 | pc.oniceconnectionstatechange = (event) => { 36 | const peerConnection = event.target; 37 | trace(`ICE state change: ${peerConnection.iceConnectionState}.`); 38 | } 39 | 40 | // let the "negotiationneeded" event trigger offer generation 41 | pc.onnegotiationneeded = async () => { 42 | try { 43 | trace('making offer'); 44 | this.makingOffer = true; 45 | try { 46 | await this.pc.setLocalDescription(); 47 | } catch (error) { 48 | // Some browsers do not support implicit descriptions yet 49 | // More info here: 50 | // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/setLocalDescription#Browser_compatibility 51 | // In this case, this peer will only wait for an offer 52 | } 53 | await this.signaling.send(pc.localDescription); 54 | } catch (err) { 55 | console.error(err); 56 | } finally { 57 | this.makingOffer = false; 58 | } 59 | }; 60 | 61 | // The perfect negotiation logic, separated from the rest of the application 62 | // from https://w3c.github.io/webrtc-pc/#perfect-negotiation-example 63 | 64 | this.signaling.onmessage = async (message) => { 65 | try { 66 | if (message == null) { 67 | return; 68 | } 69 | 70 | if (['offer', 'answer'].includes(message.type)) { 71 | const offerCollision = message.type == "offer" && 72 | (this.makingOffer || pc.signalingState != "stable"); 73 | 74 | this.ignoreOffer = !this.polite && offerCollision; 75 | if (this.ignoreOffer) { 76 | return; 77 | } 78 | await pc.setRemoteDescription(message); // SRD rolls back as needed 79 | if (message.type == "offer") { 80 | try { 81 | await pc.setLocalDescription(); 82 | } catch (error) { 83 | // Some browsers do not support implicit descriptions yet 84 | // More info here: 85 | // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/setLocalDescription#Browser_compatibility 86 | const answer = await pc.createAnswer(); 87 | await pc.setLocalDescription(answer); 88 | } 89 | await signaling.send(this.pc.localDescription); 90 | } 91 | } else if (message.type == 'candidate') { 92 | try { 93 | await pc.addIceCandidate(message); 94 | } catch (err) { 95 | if (!this.ignoreOffer) throw err; // Suppress ignored offer's candidates 96 | } 97 | } else if (message.type == 'bye') { 98 | await this.disconnect(); 99 | } 100 | 101 | //if (onmessage) { 102 | // onmessage(message); 103 | //} 104 | 105 | } catch (err) { 106 | console.error(err); 107 | } 108 | } 109 | 110 | const params = await this.signaling.connect(); 111 | this.signalingParams = params; 112 | return pc; 113 | }; 114 | 115 | Peer.prototype.addLocalStream = async function(localStream) { 116 | for (const track of localStream.getTracks()) { 117 | this.pc.addTrack(track, localStream); 118 | trace(`Adding device: ${track.label}.`); 119 | } 120 | }; 121 | 122 | Peer.prototype.disconnect = async function() { 123 | clearTimeout(this.timeout); 124 | await this.signaling.close(); 125 | 126 | if (this.pc) { 127 | await this.pc.close(); 128 | this.pc = null; 129 | } 130 | }; 131 | 132 | Peer.prototype.waitMessage = async function() { 133 | await this.signaling.receive(); 134 | if (this.pc != null) { 135 | this.timeout = setTimeout(this.waitMessage, 1000); 136 | } 137 | }; 138 | 139 | // Logs an action (text) and the time when it happened on the console. 140 | function trace(text) { 141 | text = text.trim(); 142 | const now = (window.performance.now() / 1000).toFixed(3); 143 | console.log(now, text); 144 | } 145 | -------------------------------------------------------------------------------- /colabrtc/js/signaling.js: -------------------------------------------------------------------------------- 1 | async function invoke_python(room, action, args) { 2 | // trace('Calling remote function: ' + room + '.colab.signaling.' + action); 3 | const result = await google.colab.kernel.invokeFunction( 4 | room + '.colab.signaling.' + action, // The callback name. 5 | args, // The arguments. 6 | {}); // kwargs 7 | 8 | let json = null; 9 | if ('application/json' in result.data) { 10 | json = result.data['application/json']; 11 | console.log('Result from python:') 12 | console.log(json); 13 | } 14 | return json; 15 | } 16 | 17 | var SignalingChannel = function(room) { 18 | this.room = room; 19 | 20 | // Public callbacks. Keep it sorted. 21 | this.onerror = null; 22 | this.onmessage = null; 23 | }; 24 | 25 | SignalingChannel.prototype.send = async function(message) { 26 | await invoke_python(this.room, 'send', [JSON.stringify(message)]); 27 | }; 28 | 29 | SignalingChannel.prototype.receive = async function() { 30 | const message = await invoke_python(this.room, 'receive', []); 31 | if (this.onmessage) { 32 | this.onmessage(message); 33 | } 34 | return message; 35 | }; 36 | 37 | SignalingChannel.prototype.connect = async function() { 38 | return await invoke_python(this.room, 'connect', []); 39 | }; 40 | 41 | SignalingChannel.prototype.close = async function() { 42 | return await invoke_python(this.room, 'close', []); 43 | }; -------------------------------------------------------------------------------- /colabrtc/peer.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import asyncio 3 | import logging 4 | import os 5 | import random 6 | from abc import ABC, abstractmethod 7 | from multiprocessing import Process 8 | 9 | # try: 10 | # from torch.multiprocessing import Process, set_start_method 11 | # set_start_method('forkserver') 12 | # except (ImportError, RuntimeError): 13 | # from multiprocessing import Process 14 | 15 | import fire 16 | import cv2 17 | from av import VideoFrame 18 | 19 | from aiortc import ( 20 | RTCIceCandidate, 21 | RTCPeerConnection, 22 | RTCSessionDescription, 23 | VideoStreamTrack, 24 | RTCConfiguration, RTCIceServer 25 | ) 26 | from aiortc.mediastreams import MediaStreamError 27 | from aiortc.contrib.media import MediaBlackhole, MediaPlayer, MediaRecorder, MediaRecorderContext 28 | from aiortc.contrib.signaling import BYE 29 | from server import FilesystemRTCServer 30 | from signaling import ColabSignaling, ColabApprtcSignaling 31 | 32 | import pathlib 33 | THIS_FOLDER = pathlib.Path(__file__).parent.absolute() 34 | PHOTO_PATH = os.path.join(THIS_FOLDER, 'photo.jpg') 35 | 36 | #logging.basicConfig(filename='peer.txt', filemode='w', format='%(name)s - %(levelname)s - %(message)s') 37 | logger = logging.getLogger("colabrtc.peer") 38 | 39 | 40 | class VideoImageTrack(VideoStreamTrack): 41 | """ 42 | A video stream track that returns a rotating image. 43 | """ 44 | 45 | def __init__(self): 46 | super().__init__() # don't forget this! 47 | self.img = cv2.imread(PHOTO_PATH, cv2.IMREAD_COLOR) 48 | 49 | async def recv(self): 50 | pts, time_base = await self.next_timestamp() 51 | 52 | # rotate image 53 | rows, cols, _ = self.img.shape 54 | M = cv2.getRotationMatrix2D((cols / 2, rows / 2), int(pts * time_base * 45), 1) 55 | img = cv2.warpAffine(self.img, M, (cols, rows)) 56 | 57 | # create video frame 58 | frame = VideoFrame.from_ndarray(img, format="bgr24") 59 | frame.pts = pts 60 | frame.time_base = time_base 61 | 62 | return frame 63 | 64 | 65 | class FrameTransformer(ABC): 66 | @abstractmethod 67 | def setup(self): 68 | ... 69 | @abstractmethod 70 | def transform(self, frame, frame_idx): 71 | ... 72 | 73 | 74 | class VideoTransformTrack(VideoStreamTrack): 75 | """ 76 | A video stream track that returns a rotating image. 77 | """ 78 | 79 | def __init__(self, track, frame_transformer): 80 | super().__init__() # don't forget this! 81 | 82 | if frame_transformer is None: 83 | frame_transformer = lambda x, y: x 84 | elif isinstance(frame_transformer, FrameTransformer): 85 | # frame_transformer = frame_transformer() 86 | frame_transformer.setup() 87 | self.__frame_transformer = frame_transformer 88 | 89 | self.track = track 90 | self.frame_idx = 0 91 | self.last_img = None 92 | 93 | async def recv(self): 94 | if self.track: 95 | frame = await self.track.recv() 96 | img = None 97 | 98 | try: 99 | # process video frame 100 | frame_img = frame.to_ndarray(format='bgr24') 101 | if isinstance(self.__frame_transformer, FrameTransformer): 102 | img = self.__frame_transformer.transform(frame_img, self.frame_idx) 103 | else: 104 | img = self.__frame_transformer(frame_img, self.frame_idx) 105 | except Exception as ex: 106 | logger.error(ex) 107 | 108 | if img is None and self.last_img is None: 109 | img = frame.to_ndarray(format='bgr24') 110 | elif img is None: 111 | img = self.last_img 112 | else: 113 | self.last_img = img 114 | 115 | self.frame_idx += 1 116 | else: 117 | img = np.zeros((640, 480, 3)) 118 | 119 | # rebuild a VideoFrame, preserving timing information 120 | new_frame = VideoFrame.from_ndarray(img, format='bgr24') 121 | new_frame.pts = frame.pts 122 | new_frame.time_base = frame.time_base 123 | return new_frame 124 | 125 | 126 | async def run(pc, player, recorder, signaling, frame_transformer=None): 127 | 128 | video_transform = VideoTransformTrack(None, frame_transformer) 129 | 130 | def add_tracks(): 131 | if player and player.audio: 132 | pc.addTrack(player.audio) 133 | 134 | if player and player.video: 135 | pc.addTrack(player.video) 136 | else: 137 | pc.addTrack(video_transform) 138 | # pc.addTrack(VideoImageTrack()) 139 | 140 | @pc.on("track") 141 | def on_track(track): 142 | logger.debug("Track %s received" % track.kind) 143 | #recorder.addTrack(track) 144 | 145 | if track.kind == 'video': 146 | #pc.addTrack(local_video) 147 | video_transform.track = track 148 | 149 | # connect to websocket and join 150 | params = await signaling.connect() 151 | if params["is_initiator"] == True: 152 | # send offer 153 | logger.debug('Sending OFFER...') 154 | add_tracks() 155 | await pc.setLocalDescription(await pc.createOffer()) 156 | await signaling.send(pc.localDescription) 157 | 158 | # consume signaling 159 | while True: 160 | # print('>> Python: Waiting for SDP message...') 161 | obj = await signaling.receive() 162 | if obj is None: 163 | await asyncio.sleep(1) 164 | continue 165 | 166 | if isinstance(obj, RTCSessionDescription): 167 | logger.debug(obj.type, pc.signalingState) 168 | if obj.type == 'answer' and pc.signalingState == 'stable': 169 | continue 170 | if obj.type == "offer" and pc.signalingState == 'have-local-offer': 171 | continue 172 | 173 | logger.debug(f'Received {obj.type.upper()}:', str(obj)[:100]) 174 | await pc.setRemoteDescription(obj) 175 | await recorder.start() 176 | 177 | # if obj.type == "offer": 178 | # # send answer 179 | # # add_tracks() 180 | # logger.info('Sending ANSWER...') 181 | # await pc.setLocalDescription(await pc.createAnswer()) 182 | # await signaling.send(pc.localDescription) 183 | 184 | elif isinstance(obj, RTCIceCandidate): 185 | logger.debug('Received ICE candidate:', obj) 186 | pc.addIceCandidate(obj) 187 | elif obj is BYE: 188 | logger.debug('Received BYE') 189 | logger.debug("Exiting") 190 | break 191 | 192 | 193 | def run_process(pc, player, recorder, signaling, frame_transformer): 194 | try: 195 | # run event loop 196 | loop = asyncio.get_event_loop() 197 | loop.run_until_complete( 198 | run(pc=pc, player=player, recorder=recorder, signaling=signaling, frame_transformer=frame_transformer) 199 | ) 200 | except KeyboardInterrupt: 201 | pass 202 | finally: 203 | # cleanup 204 | loop.run_until_complete(recorder.stop()) 205 | loop.run_until_complete(signaling.close()) 206 | loop.run_until_complete(pc.close()) 207 | 208 | 209 | def start_peer(room=None, signaling_folder=None, play_from=None, record_to=None, 210 | frame_transformer=None, verbose=False, ice_servers=None, multiprocess=False): 211 | 212 | if verbose: 213 | logging.basicConfig(level=logging.DEBUG) 214 | else: 215 | logging.basicConfig(level=logging.INFO) 216 | 217 | if ice_servers: 218 | logger.debug('Using ICE servers:', ice_servers) 219 | servers = [RTCIceServer(*server) if type(server) == tuple else RTCIceServer(server) for server in ice_servers] 220 | pc = RTCPeerConnection( 221 | configuration=RTCConfiguration(servers)) 222 | else: 223 | pc = RTCPeerConnection() 224 | 225 | # room = str(room) 226 | if signaling_folder: 227 | signaling = ColabSignaling(signaling_folder=signaling_folder, room=room) 228 | else: 229 | signaling = ColabApprtcSignaling(room=room) 230 | 231 | # create media source 232 | if play_from: 233 | player = MediaPlayer(play_from) 234 | else: 235 | player = None 236 | 237 | # create media sink 238 | if record_to: 239 | recorder = MediaRecorder(record_to) 240 | else: 241 | recorder = MediaBlackhole() 242 | 243 | if multiprocess: 244 | p = Process(target=run_process, args=(pc, player, recorder, signaling, frame_transformer)) 245 | p.start() 246 | return signaling.room, p 247 | else: 248 | run_process(pc, player, recorder, signaling, frame_transformer) 249 | return signaling.room, None 250 | 251 | 252 | if __name__ == '__main__': 253 | fire.Fire(start_peer) 254 | -------------------------------------------------------------------------------- /colabrtc/server.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import random 4 | import uuid 5 | import os 6 | import glob 7 | import re 8 | from datetime import datetime 9 | 10 | 11 | 12 | logger = logging.getLogger("colabrtc.server") 13 | 14 | 15 | class Room(): 16 | _folder_prefix = 'room' 17 | 18 | def __init__(self, room_id, parent_folder='webrtc'): 19 | if not Room.is_valid_id(room_id): 20 | raise ValueError(f'Room ID must have only numbers, letters, "_", "@", and ".": {room_id}') 21 | 22 | self._room_id = room_id 23 | self._parent_folder = parent_folder 24 | self._folder = os.path.join(self._parent_folder, 25 | f'{Room._folder_prefix}_{self._room_id}') 26 | self._messages = [] 27 | self._peers = {} 28 | 29 | @staticmethod 30 | def is_valid_id(room_id): 31 | return re.match(r'^[a-zA-Z0-9_@\.]+$', room_id) 32 | 33 | @staticmethod 34 | def is_valid_folder(folder): 35 | return re.match(f'{Room._folder_prefix}_\d{10}', folder) 36 | 37 | @staticmethod 38 | def get_id_from_folder(folder): 39 | folder = os.path.basename(os.path.normpath(folder)) 40 | if Room.is_valid_folder(folder): 41 | room_id = folder.replace(f'{Room._folder_prefix}_', '') 42 | return room_id 43 | 44 | @property 45 | def room_id(self): 46 | return self._room_id 47 | 48 | @property 49 | def folder(self): 50 | return self._folder 51 | 52 | @property 53 | def messages(self): 54 | return self._messages 55 | 56 | @property 57 | def peers(self): 58 | return self._peers 59 | 60 | def add_peer(self, peer): 61 | if peer: 62 | self._peers[peer.peer_id] = peer 63 | 64 | def add_message(self, message): 65 | self._messages.append(message) 66 | 67 | def get_peer(self, peer_id): 68 | return self._peers.get(peer_id) 69 | 70 | def save(self): 71 | for message in self._messages: 72 | message.save() 73 | 74 | for p_id, peer in self._peers.items(): 75 | peer.room = self 76 | peer.save() 77 | 78 | def load(self, create=False): 79 | if create: 80 | os.makedirs(self._folder, exist_ok=True) 81 | elif not os.path.exists: 82 | raise ValueError(f'Room with id {self._room_id} does not exist') 83 | 84 | self._peers = Peer.load_peers(self) 85 | self._messages = Message.load_messages(self._folder) 86 | for msg in self._messages: 87 | msg.room = self 88 | return self 89 | 90 | 91 | class Peer(): 92 | _folder_prefix = 'peer' 93 | 94 | def __init__(self, room, peer_id=None): 95 | if not peer_id: 96 | peer_id = "".join([random.choice("0123456789") for x in range(10)]) 97 | 98 | self._peer_id = peer_id 99 | self.room = room 100 | self._folder = f'{Peer._folder_prefix}_{peer_id}' 101 | self._folder = os.path.join(room.folder, self._folder) 102 | os.makedirs(self._folder, exist_ok=True) 103 | self._messages = [] 104 | self._is_initiator = False 105 | # self._registered = False 106 | room.add_peer(self) 107 | 108 | @staticmethod 109 | def is_valid_folder(folder): 110 | return re.match(f'{Peer._folder_prefix}_.+', folder) 111 | 112 | @staticmethod 113 | def get_id_from_folder(folder): 114 | folder = os.path.basename(os.path.normpath(folder)) 115 | if Peer.is_valid_folder(folder): 116 | peer_id = folder.replace(f'{Peer._folder_prefix}_', '') 117 | return peer_id 118 | 119 | @staticmethod 120 | def load_peers(room): 121 | peers = {} 122 | file_pattern = os.path.join(room.folder, f'{Peer._folder_prefix}_*') 123 | for peer_folder in glob.glob(file_pattern): 124 | peer_id = Peer.get_id_from_folder(peer_folder) 125 | if peer_id: 126 | peer = Peer(room, peer_id).load() 127 | if peer: 128 | peers[peer_id] = peer 129 | return peers 130 | 131 | @property 132 | def peer_id(self): 133 | return self._peer_id 134 | 135 | @property 136 | def is_initiator(self): 137 | return self._is_initiator 138 | 139 | @property 140 | def folder(self): 141 | return self._folder 142 | 143 | @property 144 | def messages(self): 145 | return self._messages 146 | 147 | @is_initiator.setter 148 | def is_initiator(self, value): 149 | self._is_initiator = value 150 | 151 | def to_json(self): 152 | return { 153 | 'id': self._peer_id, 154 | 'room_id': self.room.room_id, 155 | 'is_initiator': self._is_initiator 156 | #'registered': self._registered 157 | } 158 | 159 | def add_message(self, message): 160 | self._messages.append(message) 161 | 162 | def save(self): 163 | peer_data_file = os.path.join(self._folder, 'peer.json') 164 | if not os.path.exists(peer_data_file): 165 | with open(peer_data_file, 'w') as json_file: 166 | json.dump(self.to_json(), json_file) 167 | 168 | for message in self._messages: 169 | message.save() 170 | 171 | def load(self): 172 | peer_data_file = os.path.join(self._folder, 'peer.json') 173 | if not os.path.exists(peer_data_file): 174 | return 175 | 176 | with open(peer_data_file, 'r') as json_file: 177 | peer_data = json.load(json_file) 178 | self._is_initiator = peer_data['is_initiator'] 179 | # self._registered = peer_data['registered'] 180 | 181 | self._messages = Message.load_messages(self._folder) 182 | for msg in self._messages: 183 | msg.peer = self 184 | msg.room = self.room 185 | return self 186 | 187 | 188 | class Message(): 189 | _prefix = 'msg' 190 | _read_prefix = 'read' 191 | _extension = 'txt' 192 | 193 | def __init__(self, sender_id, message_id=None, msg_type=None, 194 | content=None, room=None, peer=None): 195 | if not message_id: 196 | now = datetime.now() 197 | message_id = str(datetime.timestamp(now)) 198 | self._message_id = message_id 199 | self._sender_id = sender_id 200 | self.room = room 201 | self.peer = peer 202 | self._msg_type = msg_type 203 | self.content = content 204 | self._is_read = False 205 | 206 | if room: 207 | room.add_message(self) 208 | if peer: 209 | peer.add_message(self) 210 | 211 | @staticmethod 212 | def is_valid_filename(filename): 213 | pattern = f'({Message._read_prefix}_)?{Message._prefix}_.+\.{Message._extension}' 214 | return re.match(pattern, filename) 215 | 216 | @staticmethod 217 | def get_id_from_folder(folder): 218 | filename = os.path.basename(os.path.normpath(folder)) 219 | if Message.is_valid_filename(filename): 220 | message_id = filename.replace(f'{Message._prefix}_', '') 221 | message_id = message_id.replace(f'{Message._read_prefix}_', '') 222 | message_id = message_id.replace(f'.{Message._extension}', '') 223 | message_split = message_id.split('_') 224 | message_type = None 225 | sender_id = None 226 | 227 | if len(message_split) > 1: 228 | sender_id = message_split[2] 229 | message_type = message_split[1] 230 | message_id = message_split[0] 231 | 232 | return message_id, message_type, sender_id 233 | 234 | @staticmethod 235 | def load_messages(folder): 236 | messages = [] 237 | file_pattern = os.path.join(folder, f'{Message._prefix}_*') 238 | for message_folder in glob.glob(file_pattern): 239 | msg_id, msg_type, sender_id = Message.get_id_from_folder(message_folder) 240 | if msg_id: 241 | message = Message(sender_id, message_id=msg_id, 242 | msg_type=msg_type).load(message_folder) 243 | messages.append(message) 244 | return messages 245 | 246 | @property 247 | def message_id(self): 248 | return self._message_id 249 | 250 | @property 251 | def msg_type(self): 252 | return self._msg_type 253 | 254 | @msg_type.setter 255 | def msg_type(self, value): 256 | self._msg_type = value 257 | 258 | @property 259 | def is_read(self): 260 | return self._is_read 261 | 262 | @is_read.setter 263 | def is_read(self, value): 264 | self._is_read = value 265 | 266 | def _get_filename(self): 267 | # message["id"] is the message timestamp 268 | msg_filename = f'{Message._prefix}_{self._message_id}' 269 | 270 | if self._msg_type: 271 | msg_filename = f'{msg_filename}_{self._msg_type}' 272 | 273 | if self._is_read: 274 | """ 275 | Add read prefix, so messages are not received multiple times. 276 | TODO: use timestamps for more robust message receipt verification. 277 | E.g.: the peer could call receive sending as parameter the timestamp 278 | of the last message succesfully read, and the server would return 279 | the next message given the informed timestamp. Prefixes would be 280 | still useful for inspecting message exchange using the file browser. 281 | """ 282 | msg_filename = f'{Message._read_prefix}_{msg_filename}' 283 | 284 | msg_filename = f'{msg_filename}_{self._sender_id}' 285 | msg_filename = f'{msg_filename}.{Message._extension}' 286 | return msg_filename 287 | 288 | def _save_to_folder(self, folder): 289 | message_file = self._get_filename() 290 | message_file = os.path.join(folder, message_file) 291 | 292 | if self._is_read: 293 | unread_file = message_file.replace(f'{Message._read_prefix}_', '') 294 | unread_file = os.path.join(folder, unread_file) 295 | 296 | if os.path.exists(unread_file): 297 | os.rename(unread_file, message_file) 298 | 299 | if not os.path.exists(message_file): 300 | with open(message_file, 'w') as txt_file: 301 | txt_file.write(self.content) 302 | 303 | def save(self): 304 | if self.room: 305 | self._save_to_folder(self.room.folder) 306 | if self.peer: 307 | self._save_to_folder(self.peer.folder) 308 | 309 | def load(self, folder=None): 310 | message_filename = self._get_filename() 311 | 312 | if folder: 313 | message_file = folder 314 | elif self.room: 315 | message_file = os.path.join(self.room.folder, message_filename) 316 | elif self.peer: 317 | message_file = os.path.join(self.peer.folder, message_filename) 318 | 319 | with open(message_file, 'r') as txt_file: 320 | self.content = txt_file.read() 321 | return self 322 | 323 | 324 | class FilesystemRTCServer: 325 | def __init__(self, folder='webrtc'): 326 | self._folder = folder 327 | os.makedirs(folder, exist_ok=True) 328 | 329 | def _get_room(self, room_id, create=False): 330 | return Room(room_id, parent_folder=self._folder).load(create=create) 331 | 332 | def _get_peer(self, room_id, peer_id): 333 | room = self._get_room(room_id) 334 | peer = room.get_peer(peer_id) 335 | if not peer: 336 | raise ValueError(f'invalid peer id: {peer_id}') 337 | return peer 338 | 339 | def join(self, room_id): 340 | room = self._get_room(room_id, create=True) 341 | new_peer = Peer(room) 342 | 343 | relevant_messages = [msg for msg in room.messages if msg.msg_type in ['offer', 'candidate']] 344 | if len(relevant_messages) == 0 or len(room.peers) == 0: 345 | new_peer.is_initiator = True 346 | 347 | room.save() 348 | 349 | if relevant_messages: 350 | logger.debug(f'> {len(room.messages)} messages in room {room_id}') 351 | for message in relevant_messages: 352 | message.peer = new_peer 353 | message.save() 354 | 355 | params = { 356 | 'messages': None, 357 | 'room_id': room_id, 358 | 'peer_id': new_peer.peer_id, 359 | 'is_initiator': new_peer.is_initiator 360 | } 361 | 362 | response = {'result': 'SUCCESS'} 363 | response['params'] = params 364 | return response 365 | 366 | def receive_message(self, room_id, peer_id): 367 | try: 368 | peer = self._get_peer(room_id, peer_id) 369 | messages = peer.messages 370 | if messages: 371 | message = messages[0] 372 | message.is_read = True 373 | message.save() 374 | return message.content 375 | except ValueError as err: 376 | return {'result': 'error', 'reason': str(err)} 377 | 378 | def send_message(self, room_id, peer_id, message_str): 379 | try: 380 | room = self._get_room(room_id) 381 | peer = room.get_peer(peer_id) 382 | message = Message(sender_id=peer.peer_id, room=room, 383 | content=message_str) 384 | 385 | message_json = json.loads(message_str) 386 | if 'type' in message_json: 387 | message.msg_type = message_json['type'] 388 | elif 'candidate' in message_json: 389 | message.msg_type = 'candidate' 390 | message_json['type'] = 'candidate' 391 | message_json["id"] = message_json["sdpMid"] 392 | message_json["label"] = message_json["sdpMLineIndex"] 393 | message.content = json.dumps(message_json) 394 | message.candidate = message_json['candidate'] 395 | else: 396 | message.msg_type = 'other' 397 | message_json['type'] = 'other' 398 | message.content = json.dumps(message_json) 399 | 400 | message.save() 401 | for p_id, peer in room.peers.items(): 402 | if peer.peer_id == peer_id: 403 | continue 404 | message.peer = peer 405 | message.save() 406 | 407 | except ValueError as err: 408 | return {'result': 'error', 'reason': str(err)} 409 | -------------------------------------------------------------------------------- /colabrtc/signaling.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import random 4 | import IPython 5 | import asyncio 6 | 7 | from aiortc import RTCIceCandidate, RTCSessionDescription 8 | from aiortc.contrib.signaling import object_from_string, object_to_string, BYE 9 | from aiortc.contrib.signaling import ApprtcSignaling 10 | 11 | from server import FilesystemRTCServer 12 | 13 | try: 14 | import aiohttp 15 | import websockets 16 | except ImportError: # pragma: no cover 17 | aiohttp = None 18 | websockets = None 19 | 20 | logger = logging.getLogger("colabrtc.signaling") 21 | 22 | try: 23 | from google.colab import output 24 | except ImportError: 25 | output = None 26 | logger.info('google.colab not available') 27 | 28 | 29 | class ColabApprtcSignaling(ApprtcSignaling): 30 | def __init__(self, room=None, javacript_callable=False): 31 | super().__init__(room) 32 | 33 | self._javascript_callable = javacript_callable 34 | 35 | if output and javacript_callable: 36 | output.register_callback(f'{room}.colab.signaling.connect', self.connect_sync) 37 | output.register_callback(f'{room}.colab.signaling.send', self.send_sync) 38 | output.register_callback(f'{room}.colab.signaling.receive', self.receive_sync) 39 | output.register_callback(f'{room}.colab.signaling.close', self.close_sync) 40 | 41 | @property 42 | def room(self): 43 | return self._room 44 | 45 | async def connect(self): 46 | join_url = self._origin + "/join/" + self._room 47 | 48 | # fetch room parameters 49 | self._http = aiohttp.ClientSession() 50 | async with self._http.post(join_url) as response: 51 | # we cannot use response.json() due to: 52 | # https://github.com/webrtc/apprtc/issues/562 53 | data = json.loads(await response.text()) 54 | assert data["result"] == "SUCCESS" 55 | params = data["params"] 56 | 57 | self.__is_initiator = params["is_initiator"] == "true" 58 | self.__messages = params["messages"] 59 | self.__post_url = ( 60 | self._origin + "/message/" + self._room + "/" + params["client_id"] 61 | ) 62 | 63 | # connect to websocket 64 | self._websocket = await websockets.connect( 65 | params["wss_url"], extra_headers={"Origin": self._origin} 66 | ) 67 | await self._websocket.send( 68 | json.dumps( 69 | { 70 | "clientid": params["client_id"], 71 | "cmd": "register", 72 | "roomid": params["room_id"], 73 | } 74 | ) 75 | ) 76 | 77 | print(f"AppRTC room is {params['room_id']} {params['room_link']}") 78 | 79 | return params 80 | 81 | def connect_sync(self): 82 | loop = asyncio.get_event_loop() 83 | result = loop.run_until_complete(self.connect()) 84 | if self._javascript_callable: 85 | return IPython.display.JSON(result) 86 | return result 87 | 88 | def close_sync(self): 89 | loop = asyncio.get_event_loop() 90 | return loop.run_until_complete(self.close()) 91 | 92 | def recv_nowait(self): 93 | try: 94 | return self._websocket.messages.popleft() # .get_nowait() 95 | #except (asyncio.queues.QueueEmpty, IndexError): 96 | except IndexError: 97 | pass 98 | 99 | async def receive(self): 100 | if self.__messages: 101 | message = self.__messages.pop(0) 102 | else: 103 | message = self.recv_nowait() 104 | if message: 105 | message = json.loads(message)["msg"] 106 | 107 | if message: 108 | logger.debug("< " + message) 109 | return object_from_string(message) 110 | 111 | def receive_sync(self): 112 | loop = asyncio.get_event_loop() 113 | message = loop.run_until_complete(self.receive()) 114 | if message and self._javascript_callable: 115 | message = object_to_string(message) 116 | print('receive:', message) 117 | message = json.loads(message) 118 | message = IPython.display.JSON(message) 119 | return message 120 | 121 | async def send(self, obj): 122 | message = object_to_string(obj) 123 | logger.debug("> " + message) 124 | if self.__is_initiator: 125 | await self._http.post(self.__post_url, data=message) 126 | else: 127 | await self._websocket.send(json.dumps({"cmd": "send", "msg": message})) 128 | 129 | def send_sync(self, message): 130 | print('send:', message) 131 | if type(message) == str: 132 | message_json = json.loads(message) 133 | if 'candidate' in message_json: 134 | message_json['type'] = 'candidate' 135 | message_json["id"] = message_json["sdpMid"] 136 | message_json["label"] = message_json["sdpMLineIndex"] 137 | message = json.dumps(message_json) 138 | message = object_from_string(message) 139 | loop = asyncio.get_event_loop() 140 | return loop.run_until_complete(self.send(message)) 141 | 142 | 143 | class ColabSignaling: 144 | def __init__(self, signaling_folder=None, webrtc_server=None, room=None, javacript_callable=False): 145 | if room is None: 146 | room = "".join([random.choice("0123456789") for x in range(10)]) 147 | 148 | if webrtc_server is None and signaling_folder is None: 149 | raise ValueError('Either a WebRTC server or a signaling folder must be provided.') 150 | if webrtc_server is None: 151 | self._webrtc_server = FilesystemRTCServer(folder=signaling_folder) 152 | else: 153 | self._webrtc_server = webrtc_server 154 | 155 | self._room = room 156 | self._javascript_callable = javacript_callable 157 | 158 | if output and javacript_callable: 159 | output.register_callback(f'{room}.colab.signaling.connect', self.connect_sync) 160 | output.register_callback(f'{room}.colab.signaling.send', self.send_sync) 161 | output.register_callback(f'{room}.colab.signaling.receive', self.receive_sync) 162 | output.register_callback(f'{room}.colab.signaling.close', self.close_sync) 163 | 164 | @property 165 | def room(self): 166 | return self._room 167 | 168 | async def connect(self): 169 | data = self._webrtc_server.join(self._room) 170 | assert data["result"] == "SUCCESS" 171 | params = data["params"] 172 | 173 | self.__is_initiator = params["is_initiator"] == "true" 174 | self.__messages = params["messages"] 175 | self.__peer_id = params["peer_id"] 176 | 177 | logger.info(f"Room ID: {params['room_id']}") 178 | logger.info(f"Peer ID: {self.__peer_id}") 179 | return params 180 | 181 | def connect_sync(self): 182 | loop = asyncio.get_event_loop() 183 | result = loop.run_until_complete(self.connect()) 184 | if self._javascript_callable: 185 | return IPython.display.JSON(result) 186 | return result 187 | 188 | async def close(self): 189 | if self._javascript_callable: 190 | return self.send_sync(BYE) 191 | else: 192 | await self.send(BYE) 193 | 194 | def close_sync(self): 195 | loop = asyncio.get_event_loop() 196 | return loop.run_until_complete(self.close()) 197 | 198 | async def receive(self): 199 | message = self._webrtc_server.receive_message(self._room, self.__peer_id) 200 | # if self._javascript_callable: 201 | # print('ColabSignaling: sending message to Javascript peer:', message) 202 | # else: 203 | # print('ColabSignaling: sending message to Python peer:', message) 204 | if message and type(message) == str and not self._javascript_callable: 205 | message = object_from_string(message) 206 | return message 207 | 208 | def receive_sync(self): 209 | loop = asyncio.get_event_loop() 210 | message = loop.run_until_complete(self.receive()) 211 | if message and self._javascript_callable: 212 | message = json.loads(message) 213 | message = IPython.display.JSON(message) 214 | return message 215 | 216 | async def send(self, message): 217 | if not self._javascript_callable or type(message) != str: 218 | message = object_to_string(message) 219 | self._webrtc_server.send_message(self._room, self.__peer_id, message) 220 | 221 | def send_sync(self, message): 222 | loop = asyncio.get_event_loop() 223 | return loop.run_until_complete(self.send(message)) 224 | -------------------------------------------------------------------------------- /examples/avatarify_colab.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import time 4 | 5 | import imageio 6 | import numpy as np 7 | from skimage.transform import resize 8 | import cv2 9 | import fire 10 | 11 | import torch 12 | import sys 13 | 14 | sys.path.insert(0, "/content/colabrtc/colabrtc") 15 | sys.path.insert(0, "/content/avatarify") 16 | sys.path.insert(0, "/content/avatarify/fomm") 17 | 18 | from animate import normalize_kp 19 | from cam_fomm import (load_checkpoints, normalize_alignment_kp, 20 | get_frame_kp, is_new_frame_better, crop, 21 | pad_img, load_stylegan_avatar, log) 22 | 23 | import face_alignment 24 | 25 | from peer import FrameTransformer 26 | from call import ColabCall 27 | 28 | 29 | def predict(driving_frame, source_image, relative, adapt_movement_scale, fa, 30 | generator, kp_detector, kp_driving_initial, device='cuda'): 31 | global start_frame 32 | global start_frame_kp 33 | # global kp_driving_initial 34 | 35 | with torch.no_grad(): 36 | source = torch.tensor(source_image[np.newaxis].astype(np.float32)).permute(0, 3, 1, 2).to(device) 37 | driving = torch.tensor(driving_frame[np.newaxis].astype(np.float32)).permute(0, 3, 1, 2).to(device) 38 | kp_source = kp_detector(source) 39 | 40 | if kp_driving_initial is None: 41 | kp_driving_initial = kp_detector(driving) 42 | start_frame = driving_frame.copy() 43 | start_frame_kp = get_frame_kp(fa, driving_frame) 44 | 45 | kp_driving = kp_detector(driving) 46 | kp_norm = normalize_kp(kp_source=kp_source, kp_driving=kp_driving, 47 | kp_driving_initial=kp_driving_initial, use_relative_movement=relative, 48 | use_relative_jacobian=relative, adapt_movement_scale=adapt_movement_scale) 49 | out = generator(source, kp_source=kp_source, kp_driving=kp_norm) 50 | 51 | out = np.transpose(out['prediction'].data.cpu().numpy(), [0, 2, 3, 1])[0] 52 | out = (np.clip(out, 0, 1) * 255).astype(np.uint8) 53 | 54 | return out 55 | 56 | 57 | def change_avatar(fa, new_avatar): 58 | # global avatar, avatar_kp 59 | avatar_kp = get_frame_kp(fa, new_avatar) 60 | avatar = new_avatar 61 | return avatar, avatar_kp 62 | 63 | 64 | def load_avatars(avatars_dir='./avatarify/avatars'): 65 | avatars = [] 66 | images_list = sorted(glob.glob(f'{avatars_dir}/*')) 67 | for i, f in enumerate(images_list): 68 | if f.endswith('.jpg') or f.endswith('.jpeg') or f.endswith('.png'): 69 | log(f'{i}: {f}') 70 | img = imageio.imread(f) 71 | if img.ndim == 2: 72 | img = np.tile(img[..., None], [1, 1, 3]) 73 | img = resize(img, (256, 256))[..., :3] 74 | avatars.append(img) 75 | return avatars 76 | 77 | 78 | def generate_fake_frame(frame, avatar, generator, kp_detector, relative=False, adapt_scale=True, 79 | no_pad=False, verbose=False, device='cuda', 80 | passthrough=False, kp_driving_initial=None, 81 | show_fps=False): 82 | fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, flip_input=True, device=device) 83 | avatar, avatar_kp = change_avatar(fa, avatar) 84 | 85 | frame_proportion = 0.9 86 | overlay_alpha = 0.0 87 | preview_flip = False 88 | output_flip = False 89 | find_keyframe = False 90 | 91 | fps_hist = [] 92 | fps = 0 93 | 94 | t_start = time.time() 95 | 96 | green_overlay = False 97 | frame_orig = frame.copy() 98 | 99 | frame, lrud = crop(frame, p=frame_proportion) 100 | frame = resize(frame, (256, 256))[..., :3] 101 | 102 | if find_keyframe: 103 | if is_new_frame_better(fa, avatar, frame, device): 104 | log("Taking new frame!") 105 | green_overlay = True 106 | kp_driving_initial = None 107 | 108 | if verbose: 109 | preproc_time = (time.time() - t_start) * 1000 110 | log(f'PREPROC: {preproc_time:.3f}ms') 111 | 112 | if passthrough: 113 | out = frame_orig[..., ::-1] 114 | else: 115 | pred_start = time.time() 116 | pred = predict(frame, avatar, relative, adapt_scale, fa, generator, kp_detector, kp_driving_initial, 117 | device=device) 118 | out = pred 119 | pred_time = (time.time() - pred_start) * 1000 120 | if verbose: 121 | log(f'PRED: {pred_time:.3f}ms') 122 | 123 | postproc_start = time.time() 124 | 125 | if not no_pad: 126 | out = pad_img(out, frame_orig) 127 | 128 | if out.dtype != np.uint8: 129 | out = (out * 255).astype(np.uint8) 130 | 131 | # elif key == ord('w'): 132 | # frame_proportion -= 0.05 133 | # frame_proportion = max(frame_proportion, 0.1) 134 | # elif key == ord('s'): 135 | # frame_proportion += 0.05 136 | # frame_proportion = min(frame_proportion, 1.0) 137 | # elif key == ord('x'): 138 | # kp_driving_initial = None 139 | # elif key == ord('z'): 140 | # overlay_alpha = max(overlay_alpha - 0.1, 0.0) 141 | # elif key == ord('c'): 142 | # overlay_alpha = min(overlay_alpha + 0.1, 1.0) 143 | # elif key == ord('r'): 144 | # preview_flip = not preview_flip 145 | # elif key == ord('t'): 146 | # output_flip = not output_flip 147 | # elif key == ord('f'): 148 | # find_keyframe = not find_keyframe 149 | # elif key == ord('q'): 150 | # try: 151 | # log('Loading StyleGAN avatar...') 152 | # avatar = load_stylegan_avatar() 153 | # passthrough = False 154 | # change_avatar(fa, avatar) 155 | # except: 156 | # log('Failed to load StyleGAN avatar') 157 | # elif key == ord('i'): 158 | # show_fps = not show_fps 159 | # elif key == 48: 160 | # passthrough = not passthrough 161 | # elif key != -1: 162 | # log(key) 163 | 164 | preview_frame = cv2.addWeighted(avatar[:, :, ::-1], overlay_alpha, frame, 1.0 - overlay_alpha, 0.0) 165 | 166 | if preview_flip: 167 | preview_frame = cv2.flip(preview_frame, 1) 168 | 169 | if output_flip: 170 | out = cv2.flip(out, 1) 171 | 172 | if green_overlay: 173 | green_alpha = 0.8 174 | overlay = preview_frame.copy() 175 | overlay[:] = (0, 255, 0) 176 | preview_frame = cv2.addWeighted(preview_frame, green_alpha, overlay, 1.0 - green_alpha, 0.0) 177 | 178 | if find_keyframe: 179 | preview_frame = cv2.putText(preview_frame, display_string, (10, 220), 0, 0.5, (255, 255, 255), 1) 180 | 181 | if show_fps: 182 | fps_string = f'FPS: {fps:.2f}' 183 | preview_frame = cv2.putText(preview_frame, fps_string, (10, 240), 0, 0.5, (255, 255, 255), 1) 184 | frame = cv2.putText(frame, fps_string, (10, 240), 0, 0.5, (255, 255, 255), 1) 185 | 186 | if verbose: 187 | postproc_time = (time.time() - postproc_start) * 1000 188 | log(f'POSTPROC: {postproc_time:.3f}ms') 189 | log(f'FPS: {fps:.2f}') 190 | 191 | fps_hist.append(time.time() - t_start) 192 | if len(fps_hist) == 10: 193 | fps = 10 / sum(fps_hist) 194 | fps_hist = [] 195 | 196 | return preview_frame, out[..., ::-1], kp_driving_initial 197 | 198 | 199 | class Avatarify(FrameTransformer): 200 | global kp_driving_initial 201 | kp_driving_initial = None 202 | 203 | def __init__(self, freq=1. / 30, avatar=0): 204 | import torch 205 | self.config = '/content/avatarify/fomm/config/vox-adv-256.yaml' 206 | self.checkpoint = '/content/avatarify/vox-adv-cpk.pth.tar' 207 | self.device = 'cuda' if torch.cuda.is_available() else 'cpu' 208 | self.avatar = avatar 209 | self.freq = freq 210 | 211 | def setup(self): 212 | from avatarify_colab import load_checkpoints, generate_fake_frame, load_avatars 213 | import traceback 214 | import cv2 215 | 216 | self.load_checkpoints = load_checkpoints 217 | self.generate_fake_frame = generate_fake_frame 218 | self.load_avatars = load_avatars 219 | self.traceback = traceback 220 | self.cv2 = cv2 221 | generator, kp_detector = load_checkpoints(config_path=self.config, 222 | checkpoint_path=self.checkpoint, 223 | device=self.device) 224 | self.generator = generator 225 | self.kp_detector = kp_detector 226 | self.avatars = load_avatars() 227 | 228 | import numpy as np 229 | test_frame = np.zeros((200, 200, 3)) 230 | self.generate_fake_frame(test_frame, self.avatars[self.avatar], 231 | self.generator, self.kp_detector, 232 | kp_driving_initial=kp_driving_initial, 233 | verbose=True) 234 | 235 | def transform(self, frame, frame_idx=None, avatar=0): 236 | if frame_idx % int(1. / self.freq) != 0: 237 | return 238 | 239 | try: 240 | global display_string 241 | display_string = "" 242 | global kp_driving_initial 243 | 244 | frame = self.cv2.resize(frame, (0, 0), fx=0.5, fy=0.5) 245 | # Call avatarify models here 246 | (preview_frame, fake_frame, 247 | kp_driving_initial) = self.generate_fake_frame(frame, 248 | self.avatars[self.avatar], 249 | self.generator, 250 | self.kp_detector, 251 | kp_driving_initial=kp_driving_initial, 252 | verbose=True) 253 | # fake_frame = self.cv2.resize(fake_frame, (0,0), fx=2., fy=2.) 254 | return fake_frame 255 | except Exception as err: 256 | self.traceback.print_exc() 257 | return frame 258 | 259 | 260 | def run(room=None, signaling_folder='/content/webrtc', avatar=0, frame_freq=1. / 30, verbose=False): 261 | if room: 262 | room = str(room) 263 | 264 | afy = Avatarify(freq=frame_freq, avatar=avatar) 265 | call = ColabCall() 266 | call.create(room, signaling_folder=signaling_folder, verbose=verbose, 267 | frame_transformer=afy, multiprocess=False) 268 | 269 | 270 | if __name__ == '__main__': 271 | fire.Fire(run) -------------------------------------------------------------------------------- /examples/colabrtc.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "colabrtc.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "toc_visible": true 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "accelerator": "GPU" 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "id": "QFfqXX4A8jpZ", 22 | "colab_type": "text" 23 | }, 24 | "source": [ 25 | "# ColabRTC examples\n", 26 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/thefonseca/colabrtc/blob/master/examples/colabrtc.ipynb)\n", 27 | "\n", 28 | "This notebook show how stream images from a camera on localhost to a remote Colaboratory session. The API allows one to specify an arbitrary frame processing routine that can leverage Colab resources." 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": { 34 | "id": "OFabAWq3wd2W", 35 | "colab_type": "text" 36 | }, 37 | "source": [ 38 | "## Setting up the environment" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "metadata": { 44 | "id": "xC0etC6920_6", 45 | "colab_type": "code", 46 | "colab": {} 47 | }, 48 | "source": [ 49 | "!git clone https://github.com/thefonseca/colabrtc.git\n", 50 | "!chmod +x ./colabrtc/install.sh && ./colabrtc/install.sh\n", 51 | "\n", 52 | "import sys\n", 53 | "sys.path.insert(0, \"/content/colabrtc/colabrtc\")" 54 | ], 55 | "execution_count": 0, 56 | "outputs": [] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": { 61 | "id": "cVAKOh-4952d", 62 | "colab_type": "text" 63 | }, 64 | "source": [ 65 | "## A simple echo call\n", 66 | "In this simple example the Colab peer will just return the received frames." 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "metadata": { 72 | "id": "_EE8rE5QpWgQ", 73 | "colab_type": "code", 74 | "outputId": "26884327-b515-4c79-ec10-eb953787b788", 75 | "colab": { 76 | "base_uri": "https://localhost:8080/", 77 | "height": 103 78 | } 79 | }, 80 | "source": [ 81 | "from call import ColabCall\n", 82 | "\n", 83 | "call = ColabCall()\n", 84 | "call.create()\n", 85 | "\n", 86 | "# One could also specify a signaling folder on Google Drive and establish \n", 87 | "# signaling between different Colab instances (or other apps with access to \n", 88 | "# your Google Drive)\n", 89 | "# call.create(signaling_folder='/content/drive/My Drive/webrtc/rooms')\n", 90 | "call.join()" 91 | ], 92 | "execution_count": 24, 93 | "outputs": [ 94 | { 95 | "output_type": "stream", 96 | "text": [ 97 | "INFO:colabrtc.signaling:Room ID: 0792707372\n" 98 | ], 99 | "name": "stderr" 100 | }, 101 | { 102 | "output_type": "display_data", 103 | "data": { 104 | "application/javascript": [ 105 | "async function invoke_python(room, action, args) {\n", 106 | " // trace('Calling remote function: ' + room + '.colab.signaling.' + action);\n", 107 | " const result = await google.colab.kernel.invokeFunction(\n", 108 | " room + '.colab.signaling.' + action, // The callback name.\n", 109 | " args, // The arguments.\n", 110 | " {}); // kwargs\n", 111 | "\n", 112 | " let json = null;\n", 113 | " if ('application/json' in result.data) {\n", 114 | " json = result.data['application/json'];\n", 115 | " console.log('Result from python:') \n", 116 | " console.log(json);\n", 117 | " }\n", 118 | " return json;\n", 119 | "}\n", 120 | "\n", 121 | "var SignalingChannel = function(room) {\n", 122 | " this.room = room;\n", 123 | "\n", 124 | " // Public callbacks. Keep it sorted.\n", 125 | " this.onerror = null;\n", 126 | " this.onmessage = null;\n", 127 | "};\n", 128 | "\n", 129 | "SignalingChannel.prototype.send = async function(message) {\n", 130 | " await invoke_python(this.room, 'send', [JSON.stringify(message)]);\n", 131 | "};\n", 132 | "\n", 133 | "SignalingChannel.prototype.receive = async function() {\n", 134 | " const message = await invoke_python(this.room, 'receive', []);\n", 135 | " if (this.onmessage) {\n", 136 | " this.onmessage(message);\n", 137 | " }\n", 138 | " return message;\n", 139 | "};\n", 140 | "\n", 141 | "SignalingChannel.prototype.connect = async function() {\n", 142 | " return await invoke_python(this.room, 'connect', []);\n", 143 | "};\n", 144 | "\n", 145 | "SignalingChannel.prototype.close = async function() {\n", 146 | " return await invoke_python(this.room, 'close', []);\n", 147 | "}; var Peer = function(room, configuration, polite=true) {\n", 148 | " this.makingOffer = false;\n", 149 | " this.ignoreOffer = false;\n", 150 | " this.polite = polite;\n", 151 | " this.timeout = null;\n", 152 | "};\n", 153 | "\n", 154 | "Peer.prototype.connect = async function(room, configuration) {\n", 155 | " \n", 156 | " if (this.pc) {\n", 157 | " await this.disconnect();\n", 158 | " }\n", 159 | " \n", 160 | " if (!configuration) {\n", 161 | " configuration = {\n", 162 | " iceServers: [{urls: 'stun:stun.l.google.com:19302'}]\n", 163 | " };\n", 164 | " }\n", 165 | " \n", 166 | " pc = new RTCPeerConnection(configuration);\n", 167 | " signaling = new SignalingChannel(room);\n", 168 | " this.signaling = signaling;\n", 169 | " this.pc = pc;\n", 170 | " \n", 171 | " // send any ice candidates to the other peer\n", 172 | " pc.onicecandidate = async (event) => {\n", 173 | " if (event.candidate) {\n", 174 | " trace('Sending ICE candidate');\n", 175 | " console.log(event);\n", 176 | " event.candidate.type = 'candidate'\n", 177 | " await this.signaling.send(event.candidate);\n", 178 | " }\n", 179 | " }\n", 180 | " \n", 181 | " pc.oniceconnectionstatechange = (event) => {\n", 182 | " const peerConnection = event.target;\n", 183 | " trace(`ICE state change: ${peerConnection.iceConnectionState}.`);\n", 184 | " }\n", 185 | " \n", 186 | " // let the \"negotiationneeded\" event trigger offer generation\n", 187 | " pc.onnegotiationneeded = async () => {\n", 188 | " try {\n", 189 | " trace('making offer');\n", 190 | " this.makingOffer = true;\n", 191 | " await this.pc.setLocalDescription();\n", 192 | " await this.signaling.send(pc.localDescription);\n", 193 | " } catch (err) {\n", 194 | " console.error(err);\n", 195 | " } finally {\n", 196 | " this.makingOffer = false;\n", 197 | " }\n", 198 | " };\n", 199 | " \n", 200 | " // The perfect negotiation logic, separated from the rest of the application\n", 201 | " // from https://w3c.github.io/webrtc-pc/#perfect-negotiation-example\n", 202 | " \n", 203 | " this.signaling.onmessage = async (message) => {\n", 204 | " try {\n", 205 | " if (message == null) {\n", 206 | " return;\n", 207 | " }\n", 208 | "\n", 209 | " if (['offer', 'answer'].includes(message.type)) {\n", 210 | " const offerCollision = message.type == \"offer\" &&\n", 211 | " (this.makingOffer || pc.signalingState != \"stable\");\n", 212 | "\n", 213 | " this.ignoreOffer = !this.polite && offerCollision;\n", 214 | " if (this.ignoreOffer) {\n", 215 | " return;\n", 216 | " }\n", 217 | " await pc.setRemoteDescription(message); // SRD rolls back as needed\n", 218 | " if (message.type == \"offer\") {\n", 219 | " await pc.setLocalDescription();\n", 220 | " await signaling.send(this.pc.localDescription);\n", 221 | " // The Python peer does not send candidates, so we do not expect more messages\n", 222 | " //clearTimeout(this.timeout);\n", 223 | " }\n", 224 | " } else if (message.type == 'candidate') {\n", 225 | " try {\n", 226 | " await pc.addIceCandidate(message);\n", 227 | " } catch (err) {\n", 228 | " if (!this.ignoreOffer) throw err; // Suppress ignored offer's candidates\n", 229 | " }\n", 230 | " } else if (message.type == 'bye') {\n", 231 | " await this.disconnect();\n", 232 | " }\n", 233 | " \n", 234 | " //if (onmessage) {\n", 235 | " // onmessage(message);\n", 236 | " //}\n", 237 | " \n", 238 | " } catch (err) {\n", 239 | " console.error(err);\n", 240 | " }\n", 241 | " }\n", 242 | " \n", 243 | " const params = await this.signaling.connect();\n", 244 | " this.signalingParams = params;\n", 245 | " return pc;\n", 246 | "};\n", 247 | "\n", 248 | "Peer.prototype.addLocalStream = async function(localStream) {\n", 249 | " for (const track of localStream.getTracks()) {\n", 250 | " this.pc.addTrack(track, localStream);\n", 251 | " trace(`Adding device: ${track.label}.`);\n", 252 | " }\n", 253 | "};\n", 254 | "\n", 255 | "Peer.prototype.disconnect = async function() {\n", 256 | " clearTimeout(this.timeout);\n", 257 | " await this.signaling.close();\n", 258 | "\n", 259 | " if (this.pc) {\n", 260 | " await this.pc.close();\n", 261 | " this.pc = null;\n", 262 | " }\n", 263 | "};\n", 264 | "\n", 265 | "Peer.prototype.waitMessage = async function() {\n", 266 | " await this.signaling.receive();\n", 267 | " if (this.pc != null) {\n", 268 | " this.timeout = setTimeout(this.waitMessage, 1000);\n", 269 | " }\n", 270 | "};\n", 271 | "\n", 272 | "// Logs an action (text) and the time when it happened on the console.\n", 273 | "function trace(text) {\n", 274 | " text = text.trim();\n", 275 | " const now = (window.performance.now() / 1000).toFixed(3);\n", 276 | " console.log(now, text);\n", 277 | "}\n", 278 | " var PeerUI = function(room, container_id) {\n", 279 | " // Define initial start time of the call (defined as connection between peers).\n", 280 | " startTime = null;\n", 281 | " constraints = {audio: false, video: true};\n", 282 | " \n", 283 | " let peerDiv = null;\n", 284 | " \n", 285 | " if (container_id) {\n", 286 | " peerDiv = document.getElementById(container_id);\n", 287 | " } else {\n", 288 | " peerDiv = document.createElement('div');\n", 289 | " document.body.appendChild(peerDiv);\n", 290 | " }\n", 291 | "\n", 292 | " var style = document.createElement('style');\n", 293 | " style.type = 'text/css';\n", 294 | " style.innerHTML = `\n", 295 | " .loader {\n", 296 | " position: absolute;\n", 297 | " left: 38%;\n", 298 | " top: 60%;\n", 299 | " z-index: 1;\n", 300 | " width: 50px;\n", 301 | " height: 50px;\n", 302 | " margin: -75px 0 0 -75px;\n", 303 | " border: 16px solid #f3f3f3;\n", 304 | " border-radius: 50%;\n", 305 | " border-top: 16px solid #3498db;\n", 306 | " -webkit-animation: spin 2s linear infinite;\n", 307 | " animation: spin 2s linear infinite;\n", 308 | " }\n", 309 | "\n", 310 | " @keyframes spin {\n", 311 | " 0% { transform: rotate(0deg); }\n", 312 | " 100% { transform: rotate(360deg); }\n", 313 | " }\n", 314 | " `;\n", 315 | " document.getElementsByTagName('head')[0].appendChild(style);\n", 316 | " \n", 317 | " peerDiv.style.width = '70%';\n", 318 | " \n", 319 | " // Define video elements.\n", 320 | " const videoDiv = document.createElement('div');\n", 321 | " videoDiv.style.display = 'none';\n", 322 | " videoDiv.style.textAlign = '-webkit-center';\n", 323 | " const localView = document.createElement('video');\n", 324 | " const remoteView = document.createElement('video');\n", 325 | " remoteView.autoplay = true;\n", 326 | " localView.style.display = 'block';\n", 327 | " remoteView.style.display = 'block';\n", 328 | " localView.height = 240;\n", 329 | " localView.width = 320;\n", 330 | " remoteView.height = 240;\n", 331 | " remoteView.width = 320;\n", 332 | " videoDiv.appendChild(localView);\n", 333 | " videoDiv.appendChild(remoteView);\n", 334 | " const loader = document.createElement('div');\n", 335 | " loader.style.display = 'none';\n", 336 | " loader.className = 'loader';\n", 337 | " videoDiv.appendChild(loader);\n", 338 | " \n", 339 | " // Logs a message with the id and size of a video element.\n", 340 | " function logVideoLoaded(event) {\n", 341 | " const video = event.target;\n", 342 | " trace(`${video.id} videoWidth: ${video.videoWidth}px, ` +\n", 343 | " `videoHeight: ${video.videoHeight}px.`);\n", 344 | "\n", 345 | " localView.style.width = '20%';\n", 346 | " localView.style.position = 'absolute';\n", 347 | " remoteView.style.display = 'block';\n", 348 | " remoteView.style.width = '100%';\n", 349 | " remoteView.style.height = 'auto';\n", 350 | " loader.style.display = 'none';\n", 351 | " }\n", 352 | "\n", 353 | " //localView.addEventListener('loadedmetadata', logVideoLoaded);\n", 354 | " remoteView.addEventListener('loadedmetadata', logVideoLoaded);\n", 355 | " //remoteView.addEventListener('onresize', logResizedVideo);\n", 356 | "\n", 357 | " // Define action buttons.\n", 358 | " const controlDiv = document.createElement('div');\n", 359 | " controlDiv.style.textAlign = 'center';\n", 360 | " const startButton = document.createElement('button');\n", 361 | " //const joinButton = document.createElement('button');\n", 362 | " const hangupButton = document.createElement('button');\n", 363 | " startButton.textContent = 'Join room: ' + room;\n", 364 | " //joinButton.textContent = 'Join';\n", 365 | " hangupButton.textContent = 'Hangup';\n", 366 | " controlDiv.appendChild(startButton);\n", 367 | " //controlDiv.appendChild(joinButton);\n", 368 | " controlDiv.appendChild(hangupButton);\n", 369 | " \n", 370 | " // Set up initial action buttons status: disable call and hangup.\n", 371 | " //callButton.disabled = true;\n", 372 | " hangupButton.style.display = 'none';\n", 373 | " \n", 374 | " peerDiv.appendChild(videoDiv);\n", 375 | " peerDiv.appendChild(controlDiv);\n", 376 | " \n", 377 | " this.localView = localView;\n", 378 | " this.remoteView = remoteView;\n", 379 | " this.peerDiv = peerDiv;\n", 380 | " this.videoDiv = videoDiv;\n", 381 | " this.loader = loader;\n", 382 | " this.startButton = startButton;\n", 383 | " //this.joinButton = joinButton;\n", 384 | " this.hangupButton = hangupButton;\n", 385 | " this.constraints = constraints;\n", 386 | " this.room = room;\n", 387 | " \n", 388 | " self = this;\n", 389 | " async function start() {\n", 390 | " await self.connect(this.room);\n", 391 | " }\n", 392 | " \n", 393 | " // Handles hangup action: ends up call, closes connections and resets peers.\n", 394 | " async function hangup() {\n", 395 | " await self.disconnect();\n", 396 | " }\n", 397 | " \n", 398 | " // Add click event handlers for buttons.\n", 399 | " this.startButton.addEventListener('click', start);\n", 400 | " //this.joinButton.addEventListener('click', join);\n", 401 | " this.hangupButton.addEventListener('click', hangup);\n", 402 | " \n", 403 | "// const workerCode = () => {\n", 404 | "// onmessage = async function(e) {\n", 405 | "// //const data = JSON.parse(e.data);\n", 406 | "// console.log(e.data);\n", 407 | "// console.log(e.data[0].connect);\n", 408 | "// const [async_fn, ...args] = e.data;\n", 409 | "// await async_fn(...args);\n", 410 | "// //self.postMessage('msg from worker');\n", 411 | "// };\n", 412 | "// }\n", 413 | "// const workerCodeStr = workerCode.toString().replace(/^[^{]*{\\s*/,'').replace(/\\s*}[^}]*$/,'');\n", 414 | "// console.log(workerCodeStr);\n", 415 | "// const workerBlob = new Blob([workerCodeStr], { type: \"text/javascript\" })\n", 416 | "// this.worker = new Worker(window.URL.createObjectURL(workerBlob));\n", 417 | "};\n", 418 | "\n", 419 | "\n", 420 | "PeerUI.prototype.connect = async function(room) {\n", 421 | " //startButton.disabled = true;\n", 422 | " const stream = await navigator.mediaDevices.getUserMedia(constraints);\n", 423 | " this.localView.srcObject = stream;\n", 424 | " this.localView.play();\n", 425 | " trace('Received local stream.');\n", 426 | "\n", 427 | " this.loader.style.display = 'block';\n", 428 | " this.startButton.style.display = 'none';\n", 429 | " this.localView.style.width = '100%';\n", 430 | " this.localView.style.height = 'auto';\n", 431 | " this.localView.style.position = 'relative';\n", 432 | " this.remoteView.style.display = 'none';\n", 433 | " this.videoDiv.style.display = 'block';\n", 434 | "\n", 435 | " if (google) {\n", 436 | " // Resize the output to fit the video element.\n", 437 | " google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true);\n", 438 | " }\n", 439 | " \n", 440 | " try {\n", 441 | " //this.joinButton.style.display = 'none';\n", 442 | " this.hangupButton.style.display = 'inline';\n", 443 | " \n", 444 | " trace('Starting call.');\n", 445 | " this.startTime = window.performance.now();\n", 446 | " \n", 447 | " this.peer = new Peer();\n", 448 | " await this.peer.connect(this.room);\n", 449 | " //const obj = JSON.stringify([this.peer.connect, this.room]);\n", 450 | " //this.worker.postMessage([this.peer, this.room]);\n", 451 | " \n", 452 | " this.peer.pc.ontrack = ({track, streams}) => {\n", 453 | " // once media for a remote track arrives, show it in the remote video element\n", 454 | " track.onunmute = () => {\n", 455 | " // don't set srcObject again if it is already set.\n", 456 | " if (this.remoteView.srcObject) return;\n", 457 | " console.log(streams);\n", 458 | " this.remoteView.srcObject = streams[0];\n", 459 | " trace('Remote peer connection received remote stream.');\n", 460 | " this.remoteView.play();\n", 461 | " };\n", 462 | " };\n", 463 | " \n", 464 | " const localStream = this.localView.srcObject;\n", 465 | " console.log('adding local stream');\n", 466 | " await this.peer.addLocalStream(localStream);\n", 467 | " \n", 468 | " await this.peer.waitMessage();\n", 469 | " \n", 470 | " } catch (err) {\n", 471 | " console.error(err);\n", 472 | " }\n", 473 | "};\n", 474 | "\n", 475 | "PeerUI.prototype.disconnect = async function() {\n", 476 | " await this.peer.disconnect();\n", 477 | " this.startButton.style.display = 'inline';\n", 478 | " //this.joinButton.style.display = 'inline';\n", 479 | " this.hangupButton.style.display = 'none';\n", 480 | " this.videoDiv.style.display = 'none'; \n", 481 | "\n", 482 | " trace('Ending call.');\n", 483 | " this.localView.srcObject.getVideoTracks()[0].stop();\n", 484 | "};\n", 485 | "\n", 486 | "// Logs an action (text) and the time when it happened on the console.\n", 487 | "function trace(text) {\n", 488 | " text = text.trim();\n", 489 | " const now = (window.performance.now() / 1000).toFixed(3);\n", 490 | " console.log(now, text);\n", 491 | "} \n", 492 | " var start_js_peer = function(room) { \n", 493 | " new PeerUI(room);\n", 494 | " }\n", 495 | " " 496 | ], 497 | "text/plain": [ 498 | "" 499 | ] 500 | }, 501 | "metadata": { 502 | "tags": [] 503 | } 504 | }, 505 | { 506 | "output_type": "stream", 507 | "text": [ 508 | "INFO:colabrtc.signaling:Peer ID: 6867584017\n", 509 | "INFO:colabrtc.signaling:Room ID: 0792707372\n", 510 | "INFO:colabrtc.signaling:Peer ID: 4412623024\n" 511 | ], 512 | "name": "stderr" 513 | } 514 | ] 515 | }, 516 | { 517 | "cell_type": "markdown", 518 | "metadata": { 519 | "id": "rxEolKJ0-L1W", 520 | "colab_type": "text" 521 | }, 522 | "source": [ 523 | "## Processing frames with OpenCV\n", 524 | "\n" 525 | ] 526 | }, 527 | { 528 | "cell_type": "code", 529 | "metadata": { 530 | "id": "bQmCqcq4zE9M", 531 | "colab_type": "code", 532 | "outputId": "32f9613e-5c76-45b3-a269-4eacb3bbcc42", 533 | "colab": { 534 | "base_uri": "https://localhost:8080/", 535 | "height": 103 536 | } 537 | }, 538 | "source": [ 539 | "import cv2\n", 540 | "def process_frame(frame, frame_idx):\n", 541 | " if frame_idx % 2 == 1:\n", 542 | " edges = cv2.Canny(frame, 100, 200)\n", 543 | " return cv2.cvtColor(edges, cv2.COLOR_GRAY2BGR)\n", 544 | " \n", 545 | "call = ColabCall()\n", 546 | "call.create(frame_transformer=process_frame)\n", 547 | "call.join()" 548 | ], 549 | "execution_count": 25, 550 | "outputs": [ 551 | { 552 | "output_type": "display_data", 553 | "data": { 554 | "application/javascript": [ 555 | "async function invoke_python(room, action, args) {\n", 556 | " // trace('Calling remote function: ' + room + '.colab.signaling.' + action);\n", 557 | " const result = await google.colab.kernel.invokeFunction(\n", 558 | " room + '.colab.signaling.' + action, // The callback name.\n", 559 | " args, // The arguments.\n", 560 | " {}); // kwargs\n", 561 | "\n", 562 | " let json = null;\n", 563 | " if ('application/json' in result.data) {\n", 564 | " json = result.data['application/json'];\n", 565 | " console.log('Result from python:') \n", 566 | " console.log(json);\n", 567 | " }\n", 568 | " return json;\n", 569 | "}\n", 570 | "\n", 571 | "var SignalingChannel = function(room) {\n", 572 | " this.room = room;\n", 573 | "\n", 574 | " // Public callbacks. Keep it sorted.\n", 575 | " this.onerror = null;\n", 576 | " this.onmessage = null;\n", 577 | "};\n", 578 | "\n", 579 | "SignalingChannel.prototype.send = async function(message) {\n", 580 | " await invoke_python(this.room, 'send', [JSON.stringify(message)]);\n", 581 | "};\n", 582 | "\n", 583 | "SignalingChannel.prototype.receive = async function() {\n", 584 | " const message = await invoke_python(this.room, 'receive', []);\n", 585 | " if (this.onmessage) {\n", 586 | " this.onmessage(message);\n", 587 | " }\n", 588 | " return message;\n", 589 | "};\n", 590 | "\n", 591 | "SignalingChannel.prototype.connect = async function() {\n", 592 | " return await invoke_python(this.room, 'connect', []);\n", 593 | "};\n", 594 | "\n", 595 | "SignalingChannel.prototype.close = async function() {\n", 596 | " return await invoke_python(this.room, 'close', []);\n", 597 | "}; var Peer = function(room, configuration, polite=true) {\n", 598 | " this.makingOffer = false;\n", 599 | " this.ignoreOffer = false;\n", 600 | " this.polite = polite;\n", 601 | " this.timeout = null;\n", 602 | "};\n", 603 | "\n", 604 | "Peer.prototype.connect = async function(room, configuration) {\n", 605 | " \n", 606 | " if (this.pc) {\n", 607 | " await this.disconnect();\n", 608 | " }\n", 609 | " \n", 610 | " if (!configuration) {\n", 611 | " configuration = {\n", 612 | " iceServers: [{urls: 'stun:stun.l.google.com:19302'}]\n", 613 | " };\n", 614 | " }\n", 615 | " \n", 616 | " pc = new RTCPeerConnection(configuration);\n", 617 | " signaling = new SignalingChannel(room);\n", 618 | " this.signaling = signaling;\n", 619 | " this.pc = pc;\n", 620 | " \n", 621 | " // send any ice candidates to the other peer\n", 622 | " pc.onicecandidate = async (event) => {\n", 623 | " if (event.candidate) {\n", 624 | " trace('Sending ICE candidate');\n", 625 | " console.log(event);\n", 626 | " event.candidate.type = 'candidate'\n", 627 | " await this.signaling.send(event.candidate);\n", 628 | " }\n", 629 | " }\n", 630 | " \n", 631 | " pc.oniceconnectionstatechange = (event) => {\n", 632 | " const peerConnection = event.target;\n", 633 | " trace(`ICE state change: ${peerConnection.iceConnectionState}.`);\n", 634 | " }\n", 635 | " \n", 636 | " // let the \"negotiationneeded\" event trigger offer generation\n", 637 | " pc.onnegotiationneeded = async () => {\n", 638 | " try {\n", 639 | " trace('making offer');\n", 640 | " this.makingOffer = true;\n", 641 | " await this.pc.setLocalDescription();\n", 642 | " await this.signaling.send(pc.localDescription);\n", 643 | " } catch (err) {\n", 644 | " console.error(err);\n", 645 | " } finally {\n", 646 | " this.makingOffer = false;\n", 647 | " }\n", 648 | " };\n", 649 | " \n", 650 | " // The perfect negotiation logic, separated from the rest of the application\n", 651 | " // from https://w3c.github.io/webrtc-pc/#perfect-negotiation-example\n", 652 | " \n", 653 | " this.signaling.onmessage = async (message) => {\n", 654 | " try {\n", 655 | " if (message == null) {\n", 656 | " return;\n", 657 | " }\n", 658 | "\n", 659 | " if (['offer', 'answer'].includes(message.type)) {\n", 660 | " const offerCollision = message.type == \"offer\" &&\n", 661 | " (this.makingOffer || pc.signalingState != \"stable\");\n", 662 | "\n", 663 | " this.ignoreOffer = !this.polite && offerCollision;\n", 664 | " if (this.ignoreOffer) {\n", 665 | " return;\n", 666 | " }\n", 667 | " await pc.setRemoteDescription(message); // SRD rolls back as needed\n", 668 | " if (message.type == \"offer\") {\n", 669 | " await pc.setLocalDescription();\n", 670 | " await signaling.send(this.pc.localDescription);\n", 671 | " // The Python peer does not send candidates, so we do not expect more messages\n", 672 | " //clearTimeout(this.timeout);\n", 673 | " }\n", 674 | " } else if (message.type == 'candidate') {\n", 675 | " try {\n", 676 | " await pc.addIceCandidate(message);\n", 677 | " } catch (err) {\n", 678 | " if (!this.ignoreOffer) throw err; // Suppress ignored offer's candidates\n", 679 | " }\n", 680 | " } else if (message.type == 'bye') {\n", 681 | " await this.disconnect();\n", 682 | " }\n", 683 | " \n", 684 | " //if (onmessage) {\n", 685 | " // onmessage(message);\n", 686 | " //}\n", 687 | " \n", 688 | " } catch (err) {\n", 689 | " console.error(err);\n", 690 | " }\n", 691 | " }\n", 692 | " \n", 693 | " const params = await this.signaling.connect();\n", 694 | " this.signalingParams = params;\n", 695 | " return pc;\n", 696 | "};\n", 697 | "\n", 698 | "Peer.prototype.addLocalStream = async function(localStream) {\n", 699 | " for (const track of localStream.getTracks()) {\n", 700 | " this.pc.addTrack(track, localStream);\n", 701 | " trace(`Adding device: ${track.label}.`);\n", 702 | " }\n", 703 | "};\n", 704 | "\n", 705 | "Peer.prototype.disconnect = async function() {\n", 706 | " clearTimeout(this.timeout);\n", 707 | " await this.signaling.close();\n", 708 | "\n", 709 | " if (this.pc) {\n", 710 | " await this.pc.close();\n", 711 | " this.pc = null;\n", 712 | " }\n", 713 | "};\n", 714 | "\n", 715 | "Peer.prototype.waitMessage = async function() {\n", 716 | " await this.signaling.receive();\n", 717 | " if (this.pc != null) {\n", 718 | " this.timeout = setTimeout(this.waitMessage, 1000);\n", 719 | " }\n", 720 | "};\n", 721 | "\n", 722 | "// Logs an action (text) and the time when it happened on the console.\n", 723 | "function trace(text) {\n", 724 | " text = text.trim();\n", 725 | " const now = (window.performance.now() / 1000).toFixed(3);\n", 726 | " console.log(now, text);\n", 727 | "}\n", 728 | " var PeerUI = function(room, container_id) {\n", 729 | " // Define initial start time of the call (defined as connection between peers).\n", 730 | " startTime = null;\n", 731 | " constraints = {audio: false, video: true};\n", 732 | " \n", 733 | " let peerDiv = null;\n", 734 | " \n", 735 | " if (container_id) {\n", 736 | " peerDiv = document.getElementById(container_id);\n", 737 | " } else {\n", 738 | " peerDiv = document.createElement('div');\n", 739 | " document.body.appendChild(peerDiv);\n", 740 | " }\n", 741 | "\n", 742 | " var style = document.createElement('style');\n", 743 | " style.type = 'text/css';\n", 744 | " style.innerHTML = `\n", 745 | " .loader {\n", 746 | " position: absolute;\n", 747 | " left: 38%;\n", 748 | " top: 60%;\n", 749 | " z-index: 1;\n", 750 | " width: 50px;\n", 751 | " height: 50px;\n", 752 | " margin: -75px 0 0 -75px;\n", 753 | " border: 16px solid #f3f3f3;\n", 754 | " border-radius: 50%;\n", 755 | " border-top: 16px solid #3498db;\n", 756 | " -webkit-animation: spin 2s linear infinite;\n", 757 | " animation: spin 2s linear infinite;\n", 758 | " }\n", 759 | "\n", 760 | " @keyframes spin {\n", 761 | " 0% { transform: rotate(0deg); }\n", 762 | " 100% { transform: rotate(360deg); }\n", 763 | " }\n", 764 | " `;\n", 765 | " document.getElementsByTagName('head')[0].appendChild(style);\n", 766 | " \n", 767 | " peerDiv.style.width = '70%';\n", 768 | " \n", 769 | " // Define video elements.\n", 770 | " const videoDiv = document.createElement('div');\n", 771 | " videoDiv.style.display = 'none';\n", 772 | " videoDiv.style.textAlign = '-webkit-center';\n", 773 | " const localView = document.createElement('video');\n", 774 | " const remoteView = document.createElement('video');\n", 775 | " remoteView.autoplay = true;\n", 776 | " localView.style.display = 'block';\n", 777 | " remoteView.style.display = 'block';\n", 778 | " localView.height = 240;\n", 779 | " localView.width = 320;\n", 780 | " remoteView.height = 240;\n", 781 | " remoteView.width = 320;\n", 782 | " videoDiv.appendChild(localView);\n", 783 | " videoDiv.appendChild(remoteView);\n", 784 | " const loader = document.createElement('div');\n", 785 | " loader.style.display = 'none';\n", 786 | " loader.className = 'loader';\n", 787 | " videoDiv.appendChild(loader);\n", 788 | " \n", 789 | " // Logs a message with the id and size of a video element.\n", 790 | " function logVideoLoaded(event) {\n", 791 | " const video = event.target;\n", 792 | " trace(`${video.id} videoWidth: ${video.videoWidth}px, ` +\n", 793 | " `videoHeight: ${video.videoHeight}px.`);\n", 794 | "\n", 795 | " localView.style.width = '20%';\n", 796 | " localView.style.position = 'absolute';\n", 797 | " remoteView.style.display = 'block';\n", 798 | " remoteView.style.width = '100%';\n", 799 | " remoteView.style.height = 'auto';\n", 800 | " loader.style.display = 'none';\n", 801 | " }\n", 802 | "\n", 803 | " //localView.addEventListener('loadedmetadata', logVideoLoaded);\n", 804 | " remoteView.addEventListener('loadedmetadata', logVideoLoaded);\n", 805 | " //remoteView.addEventListener('onresize', logResizedVideo);\n", 806 | "\n", 807 | " // Define action buttons.\n", 808 | " const controlDiv = document.createElement('div');\n", 809 | " controlDiv.style.textAlign = 'center';\n", 810 | " const startButton = document.createElement('button');\n", 811 | " //const joinButton = document.createElement('button');\n", 812 | " const hangupButton = document.createElement('button');\n", 813 | " startButton.textContent = 'Join room: ' + room;\n", 814 | " //joinButton.textContent = 'Join';\n", 815 | " hangupButton.textContent = 'Hangup';\n", 816 | " controlDiv.appendChild(startButton);\n", 817 | " //controlDiv.appendChild(joinButton);\n", 818 | " controlDiv.appendChild(hangupButton);\n", 819 | " \n", 820 | " // Set up initial action buttons status: disable call and hangup.\n", 821 | " //callButton.disabled = true;\n", 822 | " hangupButton.style.display = 'none';\n", 823 | " \n", 824 | " peerDiv.appendChild(videoDiv);\n", 825 | " peerDiv.appendChild(controlDiv);\n", 826 | " \n", 827 | " this.localView = localView;\n", 828 | " this.remoteView = remoteView;\n", 829 | " this.peerDiv = peerDiv;\n", 830 | " this.videoDiv = videoDiv;\n", 831 | " this.loader = loader;\n", 832 | " this.startButton = startButton;\n", 833 | " //this.joinButton = joinButton;\n", 834 | " this.hangupButton = hangupButton;\n", 835 | " this.constraints = constraints;\n", 836 | " this.room = room;\n", 837 | " \n", 838 | " self = this;\n", 839 | " async function start() {\n", 840 | " await self.connect(this.room);\n", 841 | " }\n", 842 | " \n", 843 | " // Handles hangup action: ends up call, closes connections and resets peers.\n", 844 | " async function hangup() {\n", 845 | " await self.disconnect();\n", 846 | " }\n", 847 | " \n", 848 | " // Add click event handlers for buttons.\n", 849 | " this.startButton.addEventListener('click', start);\n", 850 | " //this.joinButton.addEventListener('click', join);\n", 851 | " this.hangupButton.addEventListener('click', hangup);\n", 852 | " \n", 853 | "// const workerCode = () => {\n", 854 | "// onmessage = async function(e) {\n", 855 | "// //const data = JSON.parse(e.data);\n", 856 | "// console.log(e.data);\n", 857 | "// console.log(e.data[0].connect);\n", 858 | "// const [async_fn, ...args] = e.data;\n", 859 | "// await async_fn(...args);\n", 860 | "// //self.postMessage('msg from worker');\n", 861 | "// };\n", 862 | "// }\n", 863 | "// const workerCodeStr = workerCode.toString().replace(/^[^{]*{\\s*/,'').replace(/\\s*}[^}]*$/,'');\n", 864 | "// console.log(workerCodeStr);\n", 865 | "// const workerBlob = new Blob([workerCodeStr], { type: \"text/javascript\" })\n", 866 | "// this.worker = new Worker(window.URL.createObjectURL(workerBlob));\n", 867 | "};\n", 868 | "\n", 869 | "\n", 870 | "PeerUI.prototype.connect = async function(room) {\n", 871 | " //startButton.disabled = true;\n", 872 | " const stream = await navigator.mediaDevices.getUserMedia(constraints);\n", 873 | " this.localView.srcObject = stream;\n", 874 | " this.localView.play();\n", 875 | " trace('Received local stream.');\n", 876 | "\n", 877 | " this.loader.style.display = 'block';\n", 878 | " this.startButton.style.display = 'none';\n", 879 | " this.localView.style.width = '100%';\n", 880 | " this.localView.style.height = 'auto';\n", 881 | " this.localView.style.position = 'relative';\n", 882 | " this.remoteView.style.display = 'none';\n", 883 | " this.videoDiv.style.display = 'block';\n", 884 | "\n", 885 | " if (google) {\n", 886 | " // Resize the output to fit the video element.\n", 887 | " google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true);\n", 888 | " }\n", 889 | " \n", 890 | " try {\n", 891 | " //this.joinButton.style.display = 'none';\n", 892 | " this.hangupButton.style.display = 'inline';\n", 893 | " \n", 894 | " trace('Starting call.');\n", 895 | " this.startTime = window.performance.now();\n", 896 | " \n", 897 | " this.peer = new Peer();\n", 898 | " await this.peer.connect(this.room);\n", 899 | " //const obj = JSON.stringify([this.peer.connect, this.room]);\n", 900 | " //this.worker.postMessage([this.peer, this.room]);\n", 901 | " \n", 902 | " this.peer.pc.ontrack = ({track, streams}) => {\n", 903 | " // once media for a remote track arrives, show it in the remote video element\n", 904 | " track.onunmute = () => {\n", 905 | " // don't set srcObject again if it is already set.\n", 906 | " if (this.remoteView.srcObject) return;\n", 907 | " console.log(streams);\n", 908 | " this.remoteView.srcObject = streams[0];\n", 909 | " trace('Remote peer connection received remote stream.');\n", 910 | " this.remoteView.play();\n", 911 | " };\n", 912 | " };\n", 913 | " \n", 914 | " const localStream = this.localView.srcObject;\n", 915 | " console.log('adding local stream');\n", 916 | " await this.peer.addLocalStream(localStream);\n", 917 | " \n", 918 | " await this.peer.waitMessage();\n", 919 | " \n", 920 | " } catch (err) {\n", 921 | " console.error(err);\n", 922 | " }\n", 923 | "};\n", 924 | "\n", 925 | "PeerUI.prototype.disconnect = async function() {\n", 926 | " await this.peer.disconnect();\n", 927 | " this.startButton.style.display = 'inline';\n", 928 | " //this.joinButton.style.display = 'inline';\n", 929 | " this.hangupButton.style.display = 'none';\n", 930 | " this.videoDiv.style.display = 'none'; \n", 931 | "\n", 932 | " trace('Ending call.');\n", 933 | " this.localView.srcObject.getVideoTracks()[0].stop();\n", 934 | "};\n", 935 | "\n", 936 | "// Logs an action (text) and the time when it happened on the console.\n", 937 | "function trace(text) {\n", 938 | " text = text.trim();\n", 939 | " const now = (window.performance.now() / 1000).toFixed(3);\n", 940 | " console.log(now, text);\n", 941 | "} \n", 942 | " var start_js_peer = function(room) { \n", 943 | " new PeerUI(room);\n", 944 | " }\n", 945 | " " 946 | ], 947 | "text/plain": [ 948 | "" 949 | ] 950 | }, 951 | "metadata": { 952 | "tags": [] 953 | } 954 | }, 955 | { 956 | "output_type": "stream", 957 | "text": [ 958 | "INFO:colabrtc.signaling:Room ID: 1409964352\n", 959 | "INFO:colabrtc.signaling:Peer ID: 1680197358\n", 960 | "INFO:colabrtc.signaling:Room ID: 1409964352\n", 961 | "INFO:colabrtc.signaling:Peer ID: 8243864987\n" 962 | ], 963 | "name": "stderr" 964 | } 965 | ] 966 | }, 967 | { 968 | "cell_type": "markdown", 969 | "metadata": { 970 | "id": "zvbAzz7N-vsP", 971 | "colab_type": "text" 972 | }, 973 | "source": [ 974 | "## Using a model that requires a GPU\n", 975 | "\n", 976 | "One motivation for this project is to leverage a remote GPU testing and prototyping. In this example, we show how to use a [custom FrameTransformer](https://github.com/thefonseca/colabrtc/blob/master/colabrtc/examples/avatarify_colab.py) to [Avatarify](https://github.com/alievk/avatarify) yourself. Performance is still poor on the free K80 GPUs, and the WebRTC connection eventually breaks. Maybe with the P100 GPU (Colab Pro) things will run \n", 977 | "more smoothly." 978 | ] 979 | }, 980 | { 981 | "cell_type": "code", 982 | "metadata": { 983 | "id": "3eNQA-OqBiOl", 984 | "colab_type": "code", 985 | "colab": { 986 | "base_uri": "https://localhost:8080/", 987 | "height": 85 988 | }, 989 | "outputId": "225562b5-61cb-45a9-b417-eba08d9ab530" 990 | }, 991 | "source": [ 992 | "# Make sure you have a GPU accelerator enabled\n", 993 | "!chmod +x ./colabrtc/examples/install_avatarify.sh && ./colabrtc/examples/install_avatarify.sh" 994 | ], 995 | "execution_count": 7, 996 | "outputs": [ 997 | { 998 | "output_type": "stream", 999 | "text": [ 1000 | "Cloning repository https://github.com/alievk/avatarify.git...\n", 1001 | "Downloading model checkpoints from repository https://www.dropbox.com/s/t7h24l6wx9vreto/vox-adv-cpk.pth.tar?dl=1...\n", 1002 | "Installing FOMM submodule...\n", 1003 | "Avatarify setup complete!\n" 1004 | ], 1005 | "name": "stdout" 1006 | } 1007 | ] 1008 | }, 1009 | { 1010 | "cell_type": "code", 1011 | "metadata": { 1012 | "id": "dDbps1K74SiB", 1013 | "colab_type": "code", 1014 | "outputId": "7ef1b57c-b80c-41cb-e1a2-f05b0c735437", 1015 | "colab": { 1016 | "base_uri": "https://localhost:8080/", 1017 | "height": 34 1018 | } 1019 | }, 1020 | "source": [ 1021 | "import random\n", 1022 | "room = \"\".join([random.choice(\"0123456789\") for x in range(10)])\n", 1023 | "print('Starting call in room', room)\n", 1024 | "\n", 1025 | "# Due to multiprocessing support limitations, we need to run the Python peer via commandline.\n", 1026 | "!nohup python3 /content/colabrtc/examples/avatarify_colab.py \\\n", 1027 | "--room $room --avatar 0 > nohup.txt 2>&1 &\n", 1028 | "\n", 1029 | "# Default avatar options\n", 1030 | "# 0: ./avatarify/avatars/einstein.jpg\n", 1031 | "# 1: ./avatarify/avatars/eminem.jpg\n", 1032 | "# 2: ./avatarify/avatars/jobs.jpg\n", 1033 | "# 3: ./avatarify/avatars/mona.jpg\n", 1034 | "# 4: ./avatarify/avatars/obama.jpg\n", 1035 | "# 5: ./avatarify/avatars/potter.jpg\n", 1036 | "# 6: ./avatarify/avatars/ronaldo.png\n", 1037 | "# 7: ./avatarify/avatars/schwarzenegger.png" 1038 | ], 1039 | "execution_count": 34, 1040 | "outputs": [ 1041 | { 1042 | "output_type": "stream", 1043 | "text": [ 1044 | "Starting call in room 2267659443\n" 1045 | ], 1046 | "name": "stdout" 1047 | } 1048 | ] 1049 | }, 1050 | { 1051 | "cell_type": "code", 1052 | "metadata": { 1053 | "id": "SKpYn7IesIXM", 1054 | "colab_type": "code", 1055 | "colab": { 1056 | "base_uri": "https://localhost:8080/", 1057 | "height": 69 1058 | }, 1059 | "outputId": "c8b6508c-cf75-4b22-9325-931fd68976dc" 1060 | }, 1061 | "source": [ 1062 | "# Check out the nohup.txt file contents before running this cell\n", 1063 | "# You should see at the end something like (it may take a while):\n", 1064 | "# INFO:colabrtc.signaling:Room ID: 0065888135\n", 1065 | "# INFO:colabrtc.signaling:Peer ID: 9336811461\n", 1066 | "\n", 1067 | "from call import ColabCall\n", 1068 | "call = ColabCall()\n", 1069 | "call.join(room=room)" 1070 | ], 1071 | "execution_count": 35, 1072 | "outputs": [ 1073 | { 1074 | "output_type": "display_data", 1075 | "data": { 1076 | "application/javascript": [ 1077 | "async function invoke_python(room, action, args) {\n", 1078 | " // trace('Calling remote function: ' + room + '.colab.signaling.' + action);\n", 1079 | " const result = await google.colab.kernel.invokeFunction(\n", 1080 | " room + '.colab.signaling.' + action, // The callback name.\n", 1081 | " args, // The arguments.\n", 1082 | " {}); // kwargs\n", 1083 | "\n", 1084 | " let json = null;\n", 1085 | " if ('application/json' in result.data) {\n", 1086 | " json = result.data['application/json'];\n", 1087 | " console.log('Result from python:') \n", 1088 | " console.log(json);\n", 1089 | " }\n", 1090 | " return json;\n", 1091 | "}\n", 1092 | "\n", 1093 | "var SignalingChannel = function(room) {\n", 1094 | " this.room = room;\n", 1095 | "\n", 1096 | " // Public callbacks. Keep it sorted.\n", 1097 | " this.onerror = null;\n", 1098 | " this.onmessage = null;\n", 1099 | "};\n", 1100 | "\n", 1101 | "SignalingChannel.prototype.send = async function(message) {\n", 1102 | " await invoke_python(this.room, 'send', [JSON.stringify(message)]);\n", 1103 | "};\n", 1104 | "\n", 1105 | "SignalingChannel.prototype.receive = async function() {\n", 1106 | " const message = await invoke_python(this.room, 'receive', []);\n", 1107 | " if (this.onmessage) {\n", 1108 | " this.onmessage(message);\n", 1109 | " }\n", 1110 | " return message;\n", 1111 | "};\n", 1112 | "\n", 1113 | "SignalingChannel.prototype.connect = async function() {\n", 1114 | " return await invoke_python(this.room, 'connect', []);\n", 1115 | "};\n", 1116 | "\n", 1117 | "SignalingChannel.prototype.close = async function() {\n", 1118 | " return await invoke_python(this.room, 'close', []);\n", 1119 | "}; var Peer = function(room, configuration, polite=true) {\n", 1120 | " this.makingOffer = false;\n", 1121 | " this.ignoreOffer = false;\n", 1122 | " this.polite = polite;\n", 1123 | " this.timeout = null;\n", 1124 | "};\n", 1125 | "\n", 1126 | "Peer.prototype.connect = async function(room, configuration) {\n", 1127 | " \n", 1128 | " if (this.pc) {\n", 1129 | " await this.disconnect();\n", 1130 | " }\n", 1131 | " \n", 1132 | " if (!configuration) {\n", 1133 | " configuration = {\n", 1134 | " iceServers: [{urls: 'stun:stun.l.google.com:19302'}]\n", 1135 | " };\n", 1136 | " }\n", 1137 | " \n", 1138 | " pc = new RTCPeerConnection(configuration);\n", 1139 | " signaling = new SignalingChannel(room);\n", 1140 | " this.signaling = signaling;\n", 1141 | " this.pc = pc;\n", 1142 | " \n", 1143 | " // send any ice candidates to the other peer\n", 1144 | " pc.onicecandidate = async (event) => {\n", 1145 | " if (event.candidate) {\n", 1146 | " trace('Sending ICE candidate');\n", 1147 | " console.log(event);\n", 1148 | " event.candidate.type = 'candidate'\n", 1149 | " await this.signaling.send(event.candidate);\n", 1150 | " }\n", 1151 | " }\n", 1152 | " \n", 1153 | " pc.oniceconnectionstatechange = (event) => {\n", 1154 | " const peerConnection = event.target;\n", 1155 | " trace(`ICE state change: ${peerConnection.iceConnectionState}.`);\n", 1156 | " }\n", 1157 | " \n", 1158 | " // let the \"negotiationneeded\" event trigger offer generation\n", 1159 | " pc.onnegotiationneeded = async () => {\n", 1160 | " try {\n", 1161 | " trace('making offer');\n", 1162 | " this.makingOffer = true;\n", 1163 | " await this.pc.setLocalDescription();\n", 1164 | " await this.signaling.send(pc.localDescription);\n", 1165 | " } catch (err) {\n", 1166 | " console.error(err);\n", 1167 | " } finally {\n", 1168 | " this.makingOffer = false;\n", 1169 | " }\n", 1170 | " };\n", 1171 | " \n", 1172 | " // The perfect negotiation logic, separated from the rest of the application\n", 1173 | " // from https://w3c.github.io/webrtc-pc/#perfect-negotiation-example\n", 1174 | " \n", 1175 | " this.signaling.onmessage = async (message) => {\n", 1176 | " try {\n", 1177 | " if (message == null) {\n", 1178 | " return;\n", 1179 | " }\n", 1180 | "\n", 1181 | " if (['offer', 'answer'].includes(message.type)) {\n", 1182 | " const offerCollision = message.type == \"offer\" &&\n", 1183 | " (this.makingOffer || pc.signalingState != \"stable\");\n", 1184 | "\n", 1185 | " this.ignoreOffer = !this.polite && offerCollision;\n", 1186 | " if (this.ignoreOffer) {\n", 1187 | " return;\n", 1188 | " }\n", 1189 | " await pc.setRemoteDescription(message); // SRD rolls back as needed\n", 1190 | " if (message.type == \"offer\") {\n", 1191 | " await pc.setLocalDescription();\n", 1192 | " await signaling.send(this.pc.localDescription);\n", 1193 | " // The Python peer does not send candidates, so we do not expect more messages\n", 1194 | " //clearTimeout(this.timeout);\n", 1195 | " }\n", 1196 | " } else if (message.type == 'candidate') {\n", 1197 | " try {\n", 1198 | " await pc.addIceCandidate(message);\n", 1199 | " } catch (err) {\n", 1200 | " if (!this.ignoreOffer) throw err; // Suppress ignored offer's candidates\n", 1201 | " }\n", 1202 | " } else if (message.type == 'bye') {\n", 1203 | " await this.disconnect();\n", 1204 | " }\n", 1205 | " \n", 1206 | " //if (onmessage) {\n", 1207 | " // onmessage(message);\n", 1208 | " //}\n", 1209 | " \n", 1210 | " } catch (err) {\n", 1211 | " console.error(err);\n", 1212 | " }\n", 1213 | " }\n", 1214 | " \n", 1215 | " const params = await this.signaling.connect();\n", 1216 | " this.signalingParams = params;\n", 1217 | " return pc;\n", 1218 | "};\n", 1219 | "\n", 1220 | "Peer.prototype.addLocalStream = async function(localStream) {\n", 1221 | " for (const track of localStream.getTracks()) {\n", 1222 | " this.pc.addTrack(track, localStream);\n", 1223 | " trace(`Adding device: ${track.label}.`);\n", 1224 | " }\n", 1225 | "};\n", 1226 | "\n", 1227 | "Peer.prototype.disconnect = async function() {\n", 1228 | " clearTimeout(this.timeout);\n", 1229 | " await this.signaling.close();\n", 1230 | "\n", 1231 | " if (this.pc) {\n", 1232 | " await this.pc.close();\n", 1233 | " this.pc = null;\n", 1234 | " }\n", 1235 | "};\n", 1236 | "\n", 1237 | "Peer.prototype.waitMessage = async function() {\n", 1238 | " await this.signaling.receive();\n", 1239 | " if (this.pc != null) {\n", 1240 | " this.timeout = setTimeout(this.waitMessage, 1000);\n", 1241 | " }\n", 1242 | "};\n", 1243 | "\n", 1244 | "// Logs an action (text) and the time when it happened on the console.\n", 1245 | "function trace(text) {\n", 1246 | " text = text.trim();\n", 1247 | " const now = (window.performance.now() / 1000).toFixed(3);\n", 1248 | " console.log(now, text);\n", 1249 | "}\n", 1250 | " var PeerUI = function(room, container_id) {\n", 1251 | " // Define initial start time of the call (defined as connection between peers).\n", 1252 | " startTime = null;\n", 1253 | " constraints = {audio: false, video: true};\n", 1254 | " \n", 1255 | " let peerDiv = null;\n", 1256 | " \n", 1257 | " if (container_id) {\n", 1258 | " peerDiv = document.getElementById(container_id);\n", 1259 | " } else {\n", 1260 | " peerDiv = document.createElement('div');\n", 1261 | " document.body.appendChild(peerDiv);\n", 1262 | " }\n", 1263 | "\n", 1264 | " var style = document.createElement('style');\n", 1265 | " style.type = 'text/css';\n", 1266 | " style.innerHTML = `\n", 1267 | " .loader {\n", 1268 | " position: absolute;\n", 1269 | " left: 38%;\n", 1270 | " top: 60%;\n", 1271 | " z-index: 1;\n", 1272 | " width: 50px;\n", 1273 | " height: 50px;\n", 1274 | " margin: -75px 0 0 -75px;\n", 1275 | " border: 16px solid #f3f3f3;\n", 1276 | " border-radius: 50%;\n", 1277 | " border-top: 16px solid #3498db;\n", 1278 | " -webkit-animation: spin 2s linear infinite;\n", 1279 | " animation: spin 2s linear infinite;\n", 1280 | " }\n", 1281 | "\n", 1282 | " @keyframes spin {\n", 1283 | " 0% { transform: rotate(0deg); }\n", 1284 | " 100% { transform: rotate(360deg); }\n", 1285 | " }\n", 1286 | " `;\n", 1287 | " document.getElementsByTagName('head')[0].appendChild(style);\n", 1288 | " \n", 1289 | " peerDiv.style.width = '70%';\n", 1290 | " \n", 1291 | " // Define video elements.\n", 1292 | " const videoDiv = document.createElement('div');\n", 1293 | " videoDiv.style.display = 'none';\n", 1294 | " videoDiv.style.textAlign = '-webkit-center';\n", 1295 | " const localView = document.createElement('video');\n", 1296 | " const remoteView = document.createElement('video');\n", 1297 | " remoteView.autoplay = true;\n", 1298 | " localView.style.display = 'block';\n", 1299 | " remoteView.style.display = 'block';\n", 1300 | " localView.height = 240;\n", 1301 | " localView.width = 320;\n", 1302 | " remoteView.height = 240;\n", 1303 | " remoteView.width = 320;\n", 1304 | " videoDiv.appendChild(localView);\n", 1305 | " videoDiv.appendChild(remoteView);\n", 1306 | " const loader = document.createElement('div');\n", 1307 | " loader.style.display = 'none';\n", 1308 | " loader.className = 'loader';\n", 1309 | " videoDiv.appendChild(loader);\n", 1310 | " \n", 1311 | " // Logs a message with the id and size of a video element.\n", 1312 | " function logVideoLoaded(event) {\n", 1313 | " const video = event.target;\n", 1314 | " trace(`${video.id} videoWidth: ${video.videoWidth}px, ` +\n", 1315 | " `videoHeight: ${video.videoHeight}px.`);\n", 1316 | "\n", 1317 | " localView.style.width = '20%';\n", 1318 | " localView.style.position = 'absolute';\n", 1319 | " remoteView.style.display = 'block';\n", 1320 | " remoteView.style.width = '100%';\n", 1321 | " remoteView.style.height = 'auto';\n", 1322 | " loader.style.display = 'none';\n", 1323 | " }\n", 1324 | "\n", 1325 | " //localView.addEventListener('loadedmetadata', logVideoLoaded);\n", 1326 | " remoteView.addEventListener('loadedmetadata', logVideoLoaded);\n", 1327 | " //remoteView.addEventListener('onresize', logResizedVideo);\n", 1328 | "\n", 1329 | " // Define action buttons.\n", 1330 | " const controlDiv = document.createElement('div');\n", 1331 | " controlDiv.style.textAlign = 'center';\n", 1332 | " const startButton = document.createElement('button');\n", 1333 | " //const joinButton = document.createElement('button');\n", 1334 | " const hangupButton = document.createElement('button');\n", 1335 | " startButton.textContent = 'Join room: ' + room;\n", 1336 | " //joinButton.textContent = 'Join';\n", 1337 | " hangupButton.textContent = 'Hangup';\n", 1338 | " controlDiv.appendChild(startButton);\n", 1339 | " //controlDiv.appendChild(joinButton);\n", 1340 | " controlDiv.appendChild(hangupButton);\n", 1341 | " \n", 1342 | " // Set up initial action buttons status: disable call and hangup.\n", 1343 | " //callButton.disabled = true;\n", 1344 | " hangupButton.style.display = 'none';\n", 1345 | " \n", 1346 | " peerDiv.appendChild(videoDiv);\n", 1347 | " peerDiv.appendChild(controlDiv);\n", 1348 | " \n", 1349 | " this.localView = localView;\n", 1350 | " this.remoteView = remoteView;\n", 1351 | " this.peerDiv = peerDiv;\n", 1352 | " this.videoDiv = videoDiv;\n", 1353 | " this.loader = loader;\n", 1354 | " this.startButton = startButton;\n", 1355 | " //this.joinButton = joinButton;\n", 1356 | " this.hangupButton = hangupButton;\n", 1357 | " this.constraints = constraints;\n", 1358 | " this.room = room;\n", 1359 | " \n", 1360 | " self = this;\n", 1361 | " async function start() {\n", 1362 | " await self.connect(this.room);\n", 1363 | " }\n", 1364 | " \n", 1365 | " // Handles hangup action: ends up call, closes connections and resets peers.\n", 1366 | " async function hangup() {\n", 1367 | " await self.disconnect();\n", 1368 | " }\n", 1369 | " \n", 1370 | " // Add click event handlers for buttons.\n", 1371 | " this.startButton.addEventListener('click', start);\n", 1372 | " //this.joinButton.addEventListener('click', join);\n", 1373 | " this.hangupButton.addEventListener('click', hangup);\n", 1374 | " \n", 1375 | "// const workerCode = () => {\n", 1376 | "// onmessage = async function(e) {\n", 1377 | "// //const data = JSON.parse(e.data);\n", 1378 | "// console.log(e.data);\n", 1379 | "// console.log(e.data[0].connect);\n", 1380 | "// const [async_fn, ...args] = e.data;\n", 1381 | "// await async_fn(...args);\n", 1382 | "// //self.postMessage('msg from worker');\n", 1383 | "// };\n", 1384 | "// }\n", 1385 | "// const workerCodeStr = workerCode.toString().replace(/^[^{]*{\\s*/,'').replace(/\\s*}[^}]*$/,'');\n", 1386 | "// console.log(workerCodeStr);\n", 1387 | "// const workerBlob = new Blob([workerCodeStr], { type: \"text/javascript\" })\n", 1388 | "// this.worker = new Worker(window.URL.createObjectURL(workerBlob));\n", 1389 | "};\n", 1390 | "\n", 1391 | "\n", 1392 | "PeerUI.prototype.connect = async function(room) {\n", 1393 | " //startButton.disabled = true;\n", 1394 | " const stream = await navigator.mediaDevices.getUserMedia(constraints);\n", 1395 | " this.localView.srcObject = stream;\n", 1396 | " this.localView.play();\n", 1397 | " trace('Received local stream.');\n", 1398 | "\n", 1399 | " this.loader.style.display = 'block';\n", 1400 | " this.startButton.style.display = 'none';\n", 1401 | " this.localView.style.width = '100%';\n", 1402 | " this.localView.style.height = 'auto';\n", 1403 | " this.localView.style.position = 'relative';\n", 1404 | " this.remoteView.style.display = 'none';\n", 1405 | " this.videoDiv.style.display = 'block';\n", 1406 | "\n", 1407 | " if (google) {\n", 1408 | " // Resize the output to fit the video element.\n", 1409 | " google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true);\n", 1410 | " }\n", 1411 | " \n", 1412 | " try {\n", 1413 | " //this.joinButton.style.display = 'none';\n", 1414 | " this.hangupButton.style.display = 'inline';\n", 1415 | " \n", 1416 | " trace('Starting call.');\n", 1417 | " this.startTime = window.performance.now();\n", 1418 | " \n", 1419 | " this.peer = new Peer();\n", 1420 | " await this.peer.connect(this.room);\n", 1421 | " //const obj = JSON.stringify([this.peer.connect, this.room]);\n", 1422 | " //this.worker.postMessage([this.peer, this.room]);\n", 1423 | " \n", 1424 | " this.peer.pc.ontrack = ({track, streams}) => {\n", 1425 | " // once media for a remote track arrives, show it in the remote video element\n", 1426 | " track.onunmute = () => {\n", 1427 | " // don't set srcObject again if it is already set.\n", 1428 | " if (this.remoteView.srcObject) return;\n", 1429 | " console.log(streams);\n", 1430 | " this.remoteView.srcObject = streams[0];\n", 1431 | " trace('Remote peer connection received remote stream.');\n", 1432 | " this.remoteView.play();\n", 1433 | " };\n", 1434 | " };\n", 1435 | " \n", 1436 | " const localStream = this.localView.srcObject;\n", 1437 | " console.log('adding local stream');\n", 1438 | " await this.peer.addLocalStream(localStream);\n", 1439 | " \n", 1440 | " await this.peer.waitMessage();\n", 1441 | " \n", 1442 | " } catch (err) {\n", 1443 | " console.error(err);\n", 1444 | " }\n", 1445 | "};\n", 1446 | "\n", 1447 | "PeerUI.prototype.disconnect = async function() {\n", 1448 | " await this.peer.disconnect();\n", 1449 | " this.startButton.style.display = 'inline';\n", 1450 | " //this.joinButton.style.display = 'inline';\n", 1451 | " this.hangupButton.style.display = 'none';\n", 1452 | " this.videoDiv.style.display = 'none'; \n", 1453 | "\n", 1454 | " trace('Ending call.');\n", 1455 | " this.localView.srcObject.getVideoTracks()[0].stop();\n", 1456 | "};\n", 1457 | "\n", 1458 | "// Logs an action (text) and the time when it happened on the console.\n", 1459 | "function trace(text) {\n", 1460 | " text = text.trim();\n", 1461 | " const now = (window.performance.now() / 1000).toFixed(3);\n", 1462 | " console.log(now, text);\n", 1463 | "} \n", 1464 | " var start_js_peer = function(room) { \n", 1465 | " new PeerUI(room);\n", 1466 | " }\n", 1467 | " " 1468 | ], 1469 | "text/plain": [ 1470 | "" 1471 | ] 1472 | }, 1473 | "metadata": { 1474 | "tags": [] 1475 | } 1476 | }, 1477 | { 1478 | "output_type": "stream", 1479 | "text": [ 1480 | "INFO:colabrtc.signaling:Room ID: 2267659443\n", 1481 | "INFO:colabrtc.signaling:Peer ID: 1144267468\n" 1482 | ], 1483 | "name": "stderr" 1484 | } 1485 | ] 1486 | }, 1487 | { 1488 | "cell_type": "markdown", 1489 | "metadata": { 1490 | "id": "BJBdu9fy_rv2", 1491 | "colab_type": "text" 1492 | }, 1493 | "source": [ 1494 | "This is one easy way to kill the background process." 1495 | ] 1496 | }, 1497 | { 1498 | "cell_type": "code", 1499 | "metadata": { 1500 | "id": "75nrlAoCsM5D", 1501 | "colab_type": "code", 1502 | "outputId": "9430dd24-5890-410e-fdf4-628dbc0ccf38", 1503 | "colab": { 1504 | "base_uri": "https://localhost:8080/", 1505 | "height": 51 1506 | } 1507 | }, 1508 | "source": [ 1509 | "!pkill -f avatarify_colab.py\n", 1510 | "!ps aux | grep avatarify_colab.py" 1511 | ], 1512 | "execution_count": 33, 1513 | "outputs": [ 1514 | { 1515 | "output_type": "stream", 1516 | "text": [ 1517 | "root 2318 0.0 0.0 39192 6640 ? S 21:23 0:00 /bin/bash -c ps aux | grep avatarify_colab.py\n", 1518 | "root 2320 0.0 0.0 38572 5580 ? S 21:23 0:00 grep avatarify_colab.py\n" 1519 | ], 1520 | "name": "stdout" 1521 | } 1522 | ] 1523 | } 1524 | ] 1525 | } -------------------------------------------------------------------------------- /examples/install_avatarify.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export DEBIAN_FRONTEND=noninteractive 4 | 5 | echo 'Cloning repository https://github.com/alievk/avatarify.git...' 6 | git clone -q https://github.com/alievk/avatarify.git 7 | cd avatarify && git reset -q --hard 27596677163b2fa86e97b57bcc20088872c1d05a 8 | 9 | echo 'Downloading model checkpoints from repository https://www.dropbox.com/s/t7h24l6wx9vreto/vox-adv-cpk.pth.tar?dl=1...' 10 | wget -q -O vox-adv-cpk.pth.tar https://www.dropbox.com/s/t7h24l6wx9vreto/vox-adv-cpk.pth.tar?dl=1 11 | 12 | pip -q install face-alignment pyfakewebcam < /dev/null > /dev/null 13 | 14 | # FOMM 15 | echo 'Installing FOMM submodule...' 16 | cd /content/avatarify && git submodule update -q --init 17 | cd /content/avatarify && pip -q install -r fomm/requirements.txt < /dev/null > /dev/null 18 | pip -q install requests < /dev/null > /dev/null 19 | 20 | echo 'Avatarify setup complete!' -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export DEBIAN_FRONTEND=noninteractive 4 | 5 | echo 'Updating ffmpeg...' 6 | # We need this to update ffmpeg to version 4.x 7 | add-apt-repository -qy ppa:savoury1/ffmpeg4 > /dev/null 2>&1 8 | apt-get -qq install ffmpeg < /dev/null > /dev/null 9 | 10 | echo 'Installing dependencies...' 11 | apt-get -qq install libavdevice-dev libavfilter-dev libopus-dev libvpx-dev \ 12 | pkg-config libsrtp2-dev libavformat-dev libavcodec-dev libavutil-dev \ 13 | libswscale-dev libswresample-dev < /dev/null > /dev/null 14 | 15 | pip -qq install -r /content/colabrtc/requirements.txt < /dev/null > /dev/null 16 | 17 | echo 'ColabRTC setup complete!' -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.7.4 2 | aiortc==0.9.28 3 | fire==0.3.1 4 | nest-asyncio==1.3.3 5 | opencv-python==4.1.2.30 --------------------------------------------------------------------------------