yak.nu

├── frontend ├── src │ ├── global.d.ts │ ├── index.tsx │ └── umbrella │ │ └── SfuApp.tsx ├── html │ ├── index.html │ └── generic.html ├── package.json ├── tsconfig.json ├── public │ └── styles.css └── build.js ├── .gitignore ├── deployment ├── Dockerfile ├── build_docker.sh └── docker-compose-example.yml ├── razor ├── README.md ├── common.go ├── lockandnotify.go ├── logger.go ├── messagequeue.go └── messagehandler.go ├── sfu ├── descriptors.go ├── clients.go ├── tracks.go ├── peerconnection.go ├── client_rtsp.go ├── sfu.go └── client_webrtc.go ├── LICENSE ├── go.mod ├── docs ├── docker.md ├── openwrt.md ├── KEYS_SETUP.md ├── sfu-diagram-2.svg └── sfu-diagram-1.svg ├── proto └── sfu.proto ├── go.sum ├── README.md └── main.go /frontend/src/global.d.ts: -------------------------------------------------------------------------------- 1 | interface UmbrellaInjectedParameters { 2 | HttpPrefix: string; 3 | } 4 | 5 | interface Window { 6 | __injected__: UmbrellaInjectedParameters; 7 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | frontend/dist/ 2 | frontend/node_modules/ 3 | frontend/src/generated/ 4 | /umbrella 5 | deployment/tmp 6 | 7 | *.pb.go 8 | 9 | myCA.* 10 | service.* 11 | *.key 12 | *.pem 13 | *.srl 14 | *.crt -------------------------------------------------------------------------------- /deployment/Dockerfile: -------------------------------------------------------------------------------- 1 | # Run Stage 2 | FROM ubuntu:22.04 3 | 4 | COPY tmp/umbrella umbrella 5 | 6 | # Expose application ports 7 | EXPOSE 8081 8 | EXPOSE 9091 9 | EXPOSE 50000-60000 10 | 11 | # Start app 12 | CMD ./umbrella 13 | -------------------------------------------------------------------------------- /deployment/build_docker.sh: -------------------------------------------------------------------------------- 1 | rm -rf tmp 2 | mkdir tmp 3 | 4 | rm -rf ../frontend/dist 5 | 6 | set -e 7 | 8 | echo "Build" 9 | 10 | pushd ../ 11 | 12 | ./build_amd64_linux.sh 13 | 14 | popd 15 | 16 | cp -r ../umbrella tmp/umbrella 17 | 18 | docker build --no-cache -t umbrella-sfu . 19 | -------------------------------------------------------------------------------- /razor/README.md: -------------------------------------------------------------------------------- 1 | ### razor 2 | 3 | This is a set of opinionated utility libraries which follow me around in different languages. This is the first rough implementation in golang, so is not as well battle tested as other things yet. 4 | 5 | This will likely be spun out somewhere else when it stabilizes. -------------------------------------------------------------------------------- /frontend/html/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | yak.nu 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |

15 |

16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /sfu/descriptors.go: -------------------------------------------------------------------------------- 1 | package sfu 2 | 3 | import "github.com/pion/webrtc/v4" 4 | 5 | func trackKindToWebrtcKind(kind TrackKind) webrtc.RTPCodecType { 6 | switch kind { 7 | case TrackKind_Audio: 8 | return webrtc.RTPCodecTypeAudio 9 | case TrackKind_Video: 10 | return webrtc.RTPCodecTypeVideo 11 | } 12 | 13 | return webrtc.RTPCodecTypeUnknown 14 | } 15 | 16 | func trackKindFromWebrtcKind(kind webrtc.RTPCodecType) TrackKind { 17 | switch kind { 18 | case webrtc.RTPCodecTypeAudio: 19 | return TrackKind_Audio 20 | case webrtc.RTPCodecTypeVideo: 21 | return TrackKind_Video 22 | } 23 | 24 | return TrackKind_Unknown 25 | } 26 | -------------------------------------------------------------------------------- /razor/common.go: -------------------------------------------------------------------------------- 1 | package razor 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | "time" 7 | ) 8 | 9 | // HORRIFYING HACK 10 | var EndOfTime = time.Unix((1<<62)-1, 0) 11 | 12 | type Optional[T any] struct { 13 | Value T 14 | IsSet bool 15 | } 16 | 17 | func NewOptional[T any](value T) Optional[T] { 18 | return Optional[T]{Value: value, IsSet: true} 19 | } 20 | 21 | func NilOptional[T any]() Optional[T] { 22 | return Optional[T]{IsSet: false} 23 | } 24 | 25 | func Swap[T any](a, b *T) { 26 | *a, *b = *b, *a 27 | } 28 | 29 | func Assert(cond bool) { 30 | if !cond { 31 | panic("Assertion failure") 32 | } 33 | } 34 | 35 | func WaitForOsInterruptSignal() { 36 | stop := make(chan os.Signal, 1) 37 | signal.Notify(stop, os.Interrupt) 38 | <-stop 39 | } 40 | -------------------------------------------------------------------------------- /frontend/html/generic.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{ .Title }} 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |

18 |

19 | 20 | 21 | 22 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frontend", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "type": "module", 7 | "scripts": { 8 | "test": "echo \"Error: no test specified\" && exit 1", 9 | "build": "node build.js" 10 | }, 11 | "keywords": [], 12 | "author": "", 13 | "license": "ISC", 14 | "devDependencies": { 15 | "@types/serviceworker": "^0.0.102", 16 | "esbuild": "^0.24.0", 17 | "typescript": "^5.6.3", 18 | "fs-extra": "^11.2.0", 19 | "protoc-gen-js": "^3.21.2", 20 | "ts-protoc-gen": "^0.15.0", 21 | "@protobuf-ts/plugin": "^2.9.4" 22 | }, 23 | "dependencies": { 24 | "@types/react": "^18.3.12", 25 | "@types/react-dom": "^18.3.1", 26 | "react": "^18.3.1", 27 | "react-dom": "^18.3.1" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "jsx": "react-jsx", // Use "react" if you're using React 16 or earlier 4 | "target": "es2016", // Specify ECMAScript target 5 | "module": "commonjs", // Specify module code generation 6 | "outDir": "./dist", // Redirect output structure to the 'dist' directory 7 | "strict": true, // Enable all strict type-checking options 8 | "esModuleInterop": true, // Enables emit interoperability between CommonJS and ES Modules 9 | "skipLibCheck": true // Skip type checking of declaration files 10 | }, 11 | "include": ["src/**/*.ts", "src/index.tsx", "src/**/*.js"], // Include all .ts files in the 'src' directory 12 | "exclude": ["node_modules"] // Exclude the 'node_modules' directory 13 | } 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Nigel Birkenshaw 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /razor/lockandnotify.go: -------------------------------------------------------------------------------- 1 | package razor 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | type LockAndNotify struct { 9 | mutex sync.Mutex 10 | 11 | waker chan struct{} 12 | waiting chan bool 13 | waiter chan struct{} 14 | } 15 | 16 | func NewLockAndNotify() *LockAndNotify { 17 | waker := make(chan struct{}, 16) 18 | waiting := make(chan bool, 1) 19 | waiter := make(chan struct{}, 1) 20 | 21 | go func() { 22 | iswaiting := false 23 | for { 24 | select { 25 | case is := <-waiting: 26 | iswaiting = is 27 | case <-waker: 28 | if iswaiting { 29 | iswaiting = false 30 | waiter <- struct{}{} 31 | } 32 | } 33 | } 34 | }() 35 | 36 | return &LockAndNotify{ 37 | waker: waker, 38 | waiting: waiting, 39 | waiter: waiter, 40 | } 41 | } 42 | 43 | func (l *LockAndNotify) Enter() { 44 | l.mutex.Lock() 45 | } 46 | 47 | func (l *LockAndNotify) Leave() { 48 | l.mutex.Unlock() 49 | } 50 | 51 | // This is really dangerous if we've aborted! 52 | func (l *LockAndNotify) Notify() { 53 | l.waker <- struct{}{} 54 | } 55 | 56 | func (l *LockAndNotify) Wait(delay time.Duration) { 57 | l.waiting <- true 58 | 59 | select { 60 | case <-time.After(delay): 61 | case <-l.waiter: 62 | } 63 | 64 | l.waiting <- false 65 | } 66 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module atomirex.com/umbrella 2 | 3 | go 1.22.5 4 | 5 | require ( 6 | github.com/atomirex/mdns v0.0.13-0.20241125201824-943da8fa1231 7 | github.com/bluenviron/gortsplib/v4 v4.11.2 8 | github.com/google/uuid v1.6.0 9 | github.com/gorilla/websocket v1.5.3 10 | github.com/pion/interceptor v0.1.37 11 | github.com/pion/logging v0.2.2 12 | github.com/pion/rtcp v1.2.14 13 | github.com/pion/rtp v1.8.9 14 | github.com/pion/webrtc/v4 v4.0.1 15 | golang.org/x/net v0.31.0 16 | google.golang.org/protobuf v1.35.1 17 | ) 18 | 19 | require ( 20 | github.com/bluenviron/mediacommon v1.13.1 // indirect 21 | github.com/pion/datachannel v1.5.9 // indirect 22 | github.com/pion/dtls/v3 v3.0.3 // indirect 23 | github.com/pion/ice/v4 v4.0.2 // indirect 24 | github.com/pion/mdns/v2 v2.0.7 // indirect 25 | github.com/pion/randutil v0.1.0 // indirect 26 | github.com/pion/sctp v1.8.33 // indirect 27 | github.com/pion/sdp/v3 v3.0.9 // indirect 28 | github.com/pion/srtp/v3 v3.0.4 // indirect 29 | github.com/pion/stun/v3 v3.0.0 // indirect 30 | github.com/pion/transport/v3 v3.0.7 // indirect 31 | github.com/pion/turn/v4 v4.0.0 // indirect 32 | github.com/wlynxg/anet v0.0.3 // indirect 33 | golang.org/x/crypto v0.29.0 // indirect 34 | golang.org/x/sys v0.27.0 // indirect 35 | ) 36 | -------------------------------------------------------------------------------- /frontend/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect } from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import { useRef, useState } from 'react'; 4 | import { createRoot } from 'react-dom/client'; 5 | import { SfuApp, StatusApp, ServersApp } from './umbrella/SfuApp' 6 | 7 | const EmptyApp = () => { 8 | return ( 9 |

10 |

This is the default empty page, shouldn't be seen.

11 |

12 | ); 13 | }; 14 | 15 | const injected = window.__injected__; 16 | 17 | // Prefix when deployed 18 | const httpPrefix = injected.HttpPrefix; 19 | 20 | function removePrefix(path: string) : string { 21 | if(httpPrefix.length > 0 && path.startsWith(httpPrefix)) { 22 | path = path.substring(httpPrefix.length, path.length); 23 | } 24 | 25 | return path; 26 | } 27 | 28 | export function runApp() { 29 | const root = createRoot(document.getElementById('main-container')!); 30 | 31 | console.log("Creating app on page "+window.location.pathname); 32 | 33 | switch(removePrefix(window.location.pathname)) { 34 | case "/sfu": 35 | root.render(); 36 | break; 37 | case "/servers": 38 | root.render(); 39 | break; 40 | case "/status": 41 | root.render(); 42 | break; 43 | default: 44 | root.render(); 45 | break; 46 | } 47 | }; 48 | 49 | runApp(); -------------------------------------------------------------------------------- /docs/docker.md: -------------------------------------------------------------------------------- 1 | #### Docker/traefik 2 | For running on a public webserver something like docker compose with traefik will take away a lot of headaches. However, it introduces at least one big one. WebRTC uses a lot of ephemeral ports for communication, and these need to be exposed directly by the container. This means you have to run the container with host networking. 3 | 4 | There are several environment variables which configure the SFU to run in cloud mode. 5 | * UMBRELLA_CLOUD=1 - set to cloud mode 6 | * UMBRELLA_HTTP_PREFIX - the path from the base url to the sfu, including forward slash, so "/umbrella" serves at https://domain:port/umbrella/sfu 7 | * UMBRELLA_HTTP_SERVE_ADDR - the addr the server serves on, as host and port. To override just the default port 8081 set to :PORT, e.g. ":9000". 8 | * UMBRELLA_PUBLIC_IP= - set to the public IP of the server. i.e. 245.234.244.122 9 | * UMBRELLA_PUBLIC_HOST= - set to the public host of the server. i.e. www.atomirex.com 10 | * UMBRELLA_MIN_PORT= , UMBRELLA_MAX_PORT= - set to the minimum and maximum ephemeral ports to allocate - e.g. UMBRELLA_MIN_PORT=50000, UMBRELLA_MAX_PORT=55000 11 | 12 | The frontend is served on 8081, unless you override UMBRELLA_HTTP_SERVE_ADDR, and will need proxying for https for the public internet. You probably want to block whatever port you use from the public internet (here assumed to be on eth0) with something like: 13 | ``` 14 | iptables -A INPUT -p tcp --dport 8081 -i eth0 -j REJECT 15 | ``` 16 | 17 | This won't persist across reboots by default though, which can be done with the package iptables-persistent. 18 | 19 | An example docker-compose file is in the deployment directory. -------------------------------------------------------------------------------- /frontend/public/styles.css: -------------------------------------------------------------------------------- 1 | 2 | * { 3 | -webkit-user-select: none; 4 | user-select: none; 5 | color: white; 6 | font-family: Arial, Helvetica, sans-serif; 7 | margin: 0px; 8 | } 9 | 10 | html { 11 | height: 100%; 12 | margin: 0; 13 | } 14 | 15 | body { 16 | background-color: #1a1a1a; 17 | height: 100%; 18 | margin: 0; 19 | } 20 | 21 | input { 22 | color: black; 23 | font-size: 16px; 24 | padding: 12px; 25 | margin: 12px; 26 | } 27 | 28 | button { 29 | color: black; 30 | margin: 12px; 31 | padding: 12px; 32 | font-size: 16px; 33 | font-weight: bold; 34 | border-radius: 24px; 35 | border: white solid 2px; 36 | background-color: white; 37 | } 38 | 39 | .centering-container { 40 | display: flex; 41 | justify-content: center; 42 | align-content: center; 43 | align-items: center; 44 | height: 100vh; 45 | flex-wrap: wrap; 46 | gap: 12px; 47 | } 48 | 49 | .video-container { 50 | display: inline-block; 51 | position: relative; 52 | } 53 | 54 | .video-overlay { 55 | position: absolute; 56 | z-index: 1; 57 | top:0; 58 | left: 0; 59 | padding: 16px; 60 | margin: 12px; 61 | font-weight: bold; 62 | text-shadow: 0px 3px 6px rgba(0, 0, 0, 0.85); 63 | } 64 | 65 | .local-video { 66 | display: block; 67 | max-width: 30vmax; 68 | max-height: 30vmax; 69 | border-radius: 16px; 70 | border: white solid 2px; 71 | margin: 12px; 72 | } 73 | 74 | .remote-video { 75 | display: block; 76 | max-width: 30vmax; 77 | max-height: 30vmax; 78 | border-radius: 16px; 79 | border: white solid 2px; 80 | margin: 12px; 81 | } 82 | 83 | p { 84 | font-size: 16px; 85 | font-family: Arial, Helvetica, sans-serif; 86 | margin: 0px; 87 | padding: 0px; 88 | } 89 | -------------------------------------------------------------------------------- /docs/openwrt.md: -------------------------------------------------------------------------------- 1 | #### OpenWrt 2 | While it can run on OpenWrt it will not run on every OpenWrt device, and it won't run well on many it could run on. The target device has been a D-Link AX3200 M32, which has a dual core AArch64 (Arm) CPU and 512MB of RAM. Thanks to the golang cross compilation support this is a basic arm64 linux target and can be built with build_arm64_linux.sh . 3 | 4 | Once setup the steps are the same as for running locally on Linux. (See below). 5 | 6 | OpenWrt is a generally well behaved small Linux distro, with the exceptions here being aspects related to the security requirements for browser WebRTC support. OpenWrt uses different things for core services than you may expect, such as the package manager being opkg. Assuming you have OpenWrt installed, working, and ssh access to the device . . . 7 | 8 | ##### umdns and all lower case hostname 9 | Out of the box OpenWrt sets the hostname "OpenWrt" and does not include mdns support. Luckily there is a convenient package umdns: https://openwrt.org/docs/guide-developer/mdns 10 | 11 | Unfortunately, at least with Apple devices as clients, umdns causes confusion with letter case in hostnames, which will then cause the certificate checks to fail, so the security prerequisites are no longer valid. The workaround is to set the OpenWrt device hostname to all lower case letters. (All the certs will need to reflect this). 12 | 13 | It's useful to access the services of one AP from machines connected to other APs. To advertise mdns back out to the WAN to do this /etc/config/umdns has to include something like: 14 | ``` 15 | config umdns 16 | option jail 1 17 | list network lan 18 | list network wan 19 | ``` 20 | 21 | ##### Firewall 22 | This also required allowing mdns, so /etc/config/firewall has this in it for multicast on the usual IP/port combo: 23 | ``` 24 | config rule 25 | option src_port '5353' 26 | option src '*' 27 | option name 'Allow-mDNS' 28 | option target 'ACCEPT' 29 | option dest_ip '224.0.0.251' 30 | option dest_port '5353' 31 | option proto 'udp' 32 | ``` 33 | -------------------------------------------------------------------------------- /sfu/clients.go: -------------------------------------------------------------------------------- 1 | package sfu 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "atomirex.com/umbrella/razor" 8 | "github.com/gorilla/websocket" 9 | ) 10 | 11 | type RemoteClient interface { 12 | Label() string 13 | getStatus() *SFUStatusClient 14 | stop() 15 | AddOutgoingTracksForIncomingTrack(*incomingTrack) 16 | RemoveOutgoingTracksForIncomingTrack(*incomingTrack) 17 | RequestEvalState() 18 | } 19 | 20 | type BaseClient struct { 21 | label string 22 | logger *razor.Logger 23 | } 24 | 25 | func (bc *BaseClient) Label() string { 26 | return bc.label 27 | } 28 | 29 | type RemoteClientParameters struct { 30 | logger *razor.Logger 31 | trunkurl string 32 | s *Sfu 33 | ws *websocket.Conn 34 | } 35 | 36 | type RemoteClientFactory interface { 37 | NewClient(params *RemoteClientParameters) RemoteClient 38 | } 39 | 40 | type DefaultRemoteClientFactory struct{} 41 | 42 | // This can actually block ever returning . . . thanks to looping on the websocket 43 | // not great 44 | func (rcf *DefaultRemoteClientFactory) NewClient(params *RemoteClientParameters) RemoteClient { 45 | if params.trunkurl == "" { 46 | c := &client{ 47 | BaseClient: BaseClient{ 48 | label: fmt.Sprintf("Incoming client from %s", params.ws.UnderlyingConn().RemoteAddr()), 49 | logger: params.logger, 50 | }, 51 | 52 | websocket: params.ws, 53 | } 54 | 55 | c.run(params.ws, params.s) 56 | 57 | c.continueWebsocket(params.s) 58 | 59 | return c 60 | } else { 61 | if strings.HasPrefix(params.trunkurl, "rtsp") { 62 | c := &RtspClient{ 63 | BaseClient: BaseClient{ 64 | label: fmt.Sprintf("RTSP client of %s", params.trunkurl), 65 | logger: params.logger, 66 | }, 67 | 68 | url: params.trunkurl, 69 | } 70 | 71 | c.run(params.s) 72 | 73 | return c 74 | } else { 75 | c := &client{ 76 | BaseClient: BaseClient{ 77 | label: fmt.Sprintf("Trunking client to %s", params.trunkurl), 78 | logger: params.logger, 79 | }, 80 | 81 | trunkurl: params.trunkurl, 82 | } 83 | 84 | c.run(nil, params.s) 85 | 86 | return c 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /deployment/docker-compose-example.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | name: umbrella-sfu 4 | 5 | services: 6 | umbrella-sfu: 7 | image: ${VLAN_BASE_IP}:5000/umbrella-sfu:latest 8 | container_name: umbrella-sfu 9 | restart: unless-stopped 10 | network_mode: host 11 | 12 | extra_hosts: 13 | - "host.docker.internal:${VLAN_LOCAL_IP}" 14 | environment: 15 | - UMBRELLA_CLOUD=1 16 | - UMBRELLA_HTTP_PREFIX=/umbrella 17 | - UMBRELLA_PUBLIC_IP=211.72.93.112 18 | - UMBRELLA_PUBLIC_HOST=www.atomirex.com 19 | - UMBRELLA_MIN_PORT=50000 20 | - UMBRELLA_MAX_PORT=54000 21 | labels: 22 | - "traefik.enable=true" 23 | 24 | - "traefik.http.routers.umbrella-sfu.entrypoints=http" 25 | - "traefik.http.routers.umbrella-sfu.rule=(Host(`atomirex.com`) || Host(`www.atomirex.com`)) && PathPrefix(`/umbrella`)" 26 | - "traefik.http.middlewares.umbrella-sfu-https-redirect.redirectscheme.scheme=https" 27 | - "traefik.http.middlewares.umbrella-sfu-https-redirect.redirectscheme.permanent=true" 28 | - "traefik.http.routers.umbrella-sfu.middlewares=umbrella-sfu-https-redirect" 29 | 30 | - "traefik.http.routers.umbrella-sfu-secure.entrypoints=https" 31 | - "traefik.http.routers.umbrella-sfu-secure.rule=(Host(`atomirex.com`) || Host(`www.atomirex.com`)) && PathPrefix(`/umbrella`)" 32 | - "traefik.http.routers.umbrella-sfu-secure.tls=true" 33 | - "traefik.http.routers.umbrella-sfu-secure.tls.certresolver=myresolver" 34 | 35 | - "traefik.http.services.umbrella-sfu.loadbalancer.server.port=8081" 36 | - "traefik.http.services.umbrella-sfu.loadbalancer.server.url=http://host.docker.internal:8081" 37 | 38 | - "traefik.http.middlewares.umbrella-sfu-nowww.redirectregex.regex=^https://atomirex.com/(.*)$$" 39 | - "traefik.http.middlewares.umbrella-sfu-nowww.redirectregex.replacement=https://www.atomirex.com/$${2}" 40 | - "traefik.http.middlewares.umbrella-sfu-nowww.redirectregex.permanent=true" 41 | 42 | - "traefik.http.routers.umbrella-sfu-secure.middlewares=umbrella-sfu-nowww" 43 | ports: 44 | - "${VLAN_LOCAL_IP}:8081:8081" 45 | - "50000-54000" -------------------------------------------------------------------------------- /sfu/tracks.go: -------------------------------------------------------------------------------- 1 | package sfu 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/pion/webrtc/v4" 7 | ) 8 | 9 | type incomingTrack struct { 10 | descriptor *TrackDescriptor 11 | remote *webrtc.TrackRemote 12 | relay *webrtc.TrackLocalStaticRTP 13 | receiver *webrtc.RTPReceiver 14 | } 15 | 16 | func (it *incomingTrack) String() string { 17 | return fmt.Sprintf("{IncomingTrack id: %s}", it.descriptor.UmbrellaId) 18 | } 19 | 20 | type incomingTrackWithClientState struct { 21 | track *incomingTrack 22 | 23 | transceiverMid string // The mid of the transceiver when the track is received, until then "" 24 | } 25 | 26 | func (it *incomingTrackWithClientState) String() string { 27 | return fmt.Sprintf("{IncomingTrackWithClientState id: %s mid: %s}", it.track.descriptor.UmbrellaId, it.transceiverMid) 28 | } 29 | 30 | type outgoingTrack struct { 31 | descriptor *TrackDescriptor 32 | } 33 | 34 | func (ot *outgoingTrack) String() string { 35 | return fmt.Sprintf("{OutgoingTrack id: %s}", ot.descriptor.UmbrellaId) 36 | } 37 | 38 | type outgoingTrackWithClientState struct { 39 | track *outgoingTrack 40 | 41 | source *incomingTrack 42 | remoteNotified bool // Indicates we have sent upstreamTracks including this track to the remote device 43 | remoteAccepted bool // Indicates we have received confirmation this track is expected by the remote device 44 | } 45 | 46 | func (ot *outgoingTrackWithClientState) String() string { 47 | return fmt.Sprintf("{CutgoingTrackWithClientState id: %s}", ot.track.descriptor.UmbrellaId) 48 | } 49 | 50 | // This points to the definition of descriptor being wrong 51 | // i.e. the descriptor needs to include separate fields for the mid, the id, the src mid etc. 52 | // then need to create an "incomingID" of clientid-srcmid for each track 53 | func (t *incomingTrack) UmbrellaID() string { 54 | return t.descriptor.UmbrellaId 55 | } 56 | 57 | func (t *outgoingTrack) UmbrellaID() string { 58 | return t.descriptor.UmbrellaId 59 | } 60 | 61 | func (t *incomingTrackWithClientState) UmbrellaID() string { 62 | return t.track.UmbrellaID() 63 | } 64 | 65 | func (t *outgoingTrackWithClientState) UmbrellaID() string { 66 | return t.track.UmbrellaID() 67 | } 68 | -------------------------------------------------------------------------------- /razor/logger.go: -------------------------------------------------------------------------------- 1 | package razor 2 | 3 | import ( 4 | "log" 5 | ) 6 | 7 | type LoggingLevel int 8 | 9 | const ( 10 | LoggingLevelOff LoggingLevel = iota 11 | LogLevelError 12 | LogLevelWarn 13 | LogLevelInfo 14 | LogLevelVerbose 15 | LogLevelDebug 16 | LogLevelTrace 17 | ) 18 | 19 | func (l LoggingLevel) String() string { 20 | switch l { 21 | case LoggingLevelOff: 22 | return "OFF" 23 | case LogLevelError: 24 | return "ERR" 25 | case LogLevelWarn: 26 | return "WRN" 27 | case LogLevelInfo: 28 | return "IFO" 29 | case LogLevelVerbose: 30 | return "VRB" 31 | case LogLevelDebug: 32 | return "DBG" 33 | case LogLevelTrace: 34 | return "TRC" 35 | } 36 | 37 | return "UNK" 38 | } 39 | 40 | type Logger struct { 41 | level LoggingLevel 42 | panicOnAssertFailure bool 43 | } 44 | 45 | func NewLogger(level LoggingLevel, panicOnAssertFailure bool) *Logger { 46 | return &Logger{} 47 | } 48 | 49 | func (l *Logger) log(level LoggingLevel, component string, msg string) { 50 | if level <= l.level { 51 | log.Println(level, component, "--", msg) 52 | } 53 | } 54 | 55 | func (l *Logger) NilErrCheck(component string, msg string, err error) bool { 56 | if err != nil { 57 | if l.panicOnAssertFailure { 58 | panic("Error is not nil " + component + " " + msg + " " + err.Error()) 59 | } else { 60 | l.log(LogLevelError, component, msg) 61 | } 62 | return true 63 | } 64 | return false 65 | } 66 | 67 | func (l *Logger) Assert(component string, msg string, condition bool) { 68 | if !condition { 69 | if l.panicOnAssertFailure { 70 | panic("Assertion failure in " + component + " " + msg) 71 | } else { 72 | l.log(LogLevelError, component, "Assertion failure: "+msg) 73 | } 74 | } 75 | } 76 | 77 | func (l *Logger) Error(component string, msg string) { 78 | l.log(LogLevelError, component, msg) 79 | } 80 | 81 | func (l *Logger) Warn(component string, msg string) { 82 | l.log(LogLevelWarn, component, msg) 83 | } 84 | 85 | func (l *Logger) Info(component string, msg string) { 86 | l.log(LogLevelInfo, component, msg) 87 | } 88 | 89 | func (l *Logger) Verbose(component string, msg string) { 90 | l.log(LogLevelVerbose, component, msg) 91 | } 92 | 93 | func (l *Logger) Debug(component string, msg string) { 94 | l.log(LogLevelDebug, component, msg) 95 | } 96 | 97 | func (l *Logger) Trace(component string, msg string) { 98 | l.log(LogLevelTrace, component, msg) 99 | } 100 | -------------------------------------------------------------------------------- /frontend/build.js: -------------------------------------------------------------------------------- 1 | import { build } from 'esbuild'; 2 | import fs from 'fs'; 3 | import fsextra from 'fs-extra'; 4 | import { exec, execSync } from 'child_process'; 5 | import path from 'path'; 6 | 7 | console.log("Starting node build"); 8 | 9 | const __dirname = new URL('.', import.meta.url).pathname; 10 | 11 | function mkdir(dir) { 12 | if (!fs.existsSync(dir)) { 13 | fs.mkdirSync(dir, { recursive: true }); 14 | } 15 | } 16 | 17 | const srcDir = path.join(__dirname, 'src'); 18 | const distDir = path.join(__dirname, 'dist'); 19 | mkdir(distDir); 20 | mkdir(path.join(distDir, 'static')); 21 | mkdir(path.join(distDir, 'templates')); 22 | mkdir(path.join(srcDir, 'generated')); 23 | 24 | const version = Date.now(); // or use any unique versioning scheme 25 | 26 | const protocCommand = ` 27 | protoc --ts_out src/generated --proto_path ../proto ../proto/*.proto 28 | `; 29 | 30 | function runCommand(cmd) { 31 | try { 32 | // Execute the command synchronously 33 | const stdout = execSync(cmd, { stdio: 'pipe' }); // 'pipe' captures output 34 | console.log(`stdout: ${stdout.toString()}`); // Convert Buffer to string 35 | } catch (error) { 36 | // Handle errors 37 | console.error(`exec error: ${error.message}`); 38 | if (error.stdout) { 39 | console.error(`stdout: ${error.stdout.toString()}`); 40 | } 41 | if (error.stderr) { 42 | console.error(`stderr: ${error.stderr.toString()}`); 43 | } 44 | process.exit(1); // Exit with an error code 45 | } 46 | } 47 | 48 | console.log("Generating protobufs"); 49 | runCommand(protocCommand); 50 | 51 | console.log("Checking typescript types"); 52 | runCommand(`tsc --noEmit`); 53 | 54 | function updateHtml(input, output) { 55 | const indexInPath = path.join(__dirname, input); 56 | const indexOutPath = path.join(__dirname, output); 57 | const indexContent = fs.readFileSync(indexInPath, 'utf-8'); 58 | 59 | const updatedContent = indexContent 60 | .replace(/bundle\.js/g, `bundle.js?v=${version}`) 61 | .replace(/styles\.css/g, `styles.css?v=${version}`); 62 | 63 | fs.writeFileSync(indexOutPath, updatedContent); 64 | } 65 | 66 | console.log("Preparing html static and templates"); 67 | updateHtml('html/index.html', 'dist/static/index.html'); 68 | updateHtml('html/generic.html', 'dist/templates/generic.html'); 69 | 70 | console.log("Copying static resources"); 71 | fsextra.copy('public', 'dist/static', { overwrite: true }); 72 | 73 | console.log("esbuild of frontend"); 74 | build({ 75 | entryPoints: ['./src/index.tsx'], 76 | loader: {'.ts': 'ts', '.tsx': 'tsx', '.js': 'js'}, 77 | tsconfig: 'tsconfig.json', 78 | jsx: 'automatic', 79 | bundle: true, 80 | outfile: 'dist/static/bundle.js', 81 | minify: true, 82 | sourcemap: true, 83 | target: ['es2016'], 84 | }).catch(() => process.exit(1)); 85 | 86 | console.log("Finished node build"); -------------------------------------------------------------------------------- /docs/KEYS_SETUP.md: -------------------------------------------------------------------------------- 1 | # Keys setup 2 | This uses WebRTC features of web browsers. For the API for this to work browsers must be persuaded they are operating in a secure context. https://developer.mozilla.org/en-US/docs/Web/Security/Secure_Contexts 3 | 4 | ## Cloud deployments 5 | For securing the cloud deployment the easiest way is to deploy a docker container behind something like https://github.com/traefik/traefik configured to use https://letsencrypt.org as the certificate authority. The docker-compose-example.yml in deployment shows how to use a traefik wide certificate resolver as is normal for such things. 6 | 7 | ## Local subnetworks 8 | Some notes in advance: 9 | * For OpenWrt servers having all lower case host names helps avoid problems with umdns. 10 | * Recent versions of desktop Chrome seem to completely ignore mdns. You can work around this by using the actual device IP instead, and going through the security warnings. (To get a device IP from a Mac open a terminal and run ```dns-sd -q target-machine-name.local``` and it will list the IPs of that device on your subnet). 11 | 12 | ### Server setup 13 | You will need to know the host name of each server device. 14 | 15 | Create a private key for this server: 16 | ``` 17 | openssl genrsa -out service.key 2048 18 | ``` 19 | 20 | Create a server certificate: 21 | ``` 22 | openssl req -x509 -new -key service.key -sha256 -days 365 -out service.crt 23 | ``` 24 | 25 | When prompted you will need to enter the host name and domain as the "Common Name" field, meaning it should be all lower case and with ".local" at the end. For example, for a machine called "Atomirex-Machine" enter "atomirex-machine.local". (OpenWrt servers should have hostnames all lowercase because the umdns responder may not respond properly). 26 | 27 | The resulting service.crt and service.key files need to be in the directory alongside umbrella when it is launched. 28 | 29 | The local subnetwork mode sets up the sfu page to serve at /sfu . 30 | 31 | ### Client setup 32 | When the server is run it will output a line equivalent to "Starting server at https://atomirex-machine.local:8081/" which tells you the base url. The actual sfu is at /sfu for local deployments, and in the cloud likely /umbrella/sfu if following the default docker setup. 33 | 34 | For cloud only deployments no client specific incantations are needed. For local subnetworks some of the notes here might help: 35 | 36 | #### Safari 37 | Vist the page https://[[hostname:port]]/sfu . The first time you will get "This Connection Is Not Private". Select "Show Details" then "Visit Website". It should load successfully. 38 | #### Firefox (all platforms) 39 | In Settings disable DNS over HTTPS, then assuming your operating system is setup to use mdns properly you should be able to vist the page at the url. It will tell you it's not secure - go through more details and visit anyway, after which it will work. 40 | #### Android 41 | You need to disable "Private DNS" in system settings. (Easiest to search for DNS). Set it to "Off". 42 | 43 | Visit the page. Press "Show Advanced" and click on "Proceed to . . . " and it will load as normal. -------------------------------------------------------------------------------- /proto/sfu.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package sfu; 4 | 5 | option go_package = "./sfu"; 6 | 7 | enum TrackKind { 8 | Unknown = 0; 9 | Audio = 1; 10 | Video = 2; 11 | } 12 | 13 | message TrackDescriptor { 14 | string id = 1; 15 | TrackKind kind = 2; 16 | string streamId = 3; 17 | string umbrellaId = 4; 18 | } 19 | 20 | message CandidateMessage { 21 | string candidate = 1; 22 | bool incoming = 2; // Indicates the direction of the pc, relative to the sender of the message (i.e. true if the sender PC is incoming) 23 | } 24 | 25 | message OfferMessage { 26 | // Complete JSON of offer object, not just the sdp field 27 | string offer = 1; 28 | } 29 | 30 | message AnswerMessage { 31 | // Complete JSON of answer object, not just the sdp field 32 | string answer = 1; 33 | } 34 | 35 | // client->server this is a request saying what is available 36 | message SetUpstreamTracks { 37 | repeated TrackDescriptor tracks = 1; 38 | } 39 | 40 | // server->client it is information about what it is ready for accepting 41 | message AcceptUpstreamTracks { 42 | repeated TrackDescriptor tracks = 1; 43 | } 44 | 45 | // client->server - when the mapping for ID to mid is known notify the server 46 | message MidToUmbrellaIDMapping { 47 | string umbrellaId = 1; 48 | string mid = 2; 49 | } 50 | 51 | message MidToUmbrellaIDMappings { 52 | repeated MidToUmbrellaIDMapping mapping = 1; 53 | } 54 | 55 | // Possibly the dumbest conceivable almost symmetrical signalling protocol 56 | message RemoteNodeMessage { 57 | CandidateMessage candidate = 1; 58 | OfferMessage offer = 2; 59 | AnswerMessage answer = 3; 60 | SetUpstreamTracks upstreamTracks = 4; 61 | AcceptUpstreamTracks acceptTracks = 5; 62 | MidToUmbrellaIDMappings midMappings = 6; 63 | } 64 | 65 | // Returned from the /servers endpoint with content-type application/x-protobuf 66 | message CurrentServers { 67 | repeated string servers = 1; // urls of the servers 68 | } 69 | 70 | // Returned by the /status endpoint with content-type application/x-protobuf 71 | message SFUStatus { 72 | repeated TrackDescriptor relayingTracks = 1; 73 | repeated SFUStatusClient clients = 2; 74 | repeated string servers = 3; 75 | } 76 | 77 | 78 | // TODO could use the pc getstats interface for more info later 79 | message SFUStatusPeerConnection { 80 | string connectionState = 1; 81 | string signalingState = 2; 82 | string iceConnectionState = 3; 83 | string iceGatheringState = 4; 84 | 85 | int32 transceiverCount = 5; 86 | int32 senderCount = 6; 87 | int32 receiverCount = 7; 88 | } 89 | 90 | message SFUStatusSender { 91 | bool hasTrack = 1; 92 | string trackIdIfSet = 2; 93 | string umbrellaId = 3; 94 | } 95 | 96 | message SFUStatusStagedIncomingTrack { 97 | string streamId = 1; 98 | string trackId = 2; 99 | string mid = 3; 100 | } 101 | 102 | message SFUStatusClient { 103 | string label = 1; 104 | string trunkUrl = 2; 105 | SFUStatusPeerConnection incomingPC = 3; 106 | SFUStatusPeerConnection outgoingPC = 4; 107 | repeated TrackDescriptor incomingTracks = 5; 108 | repeated TrackDescriptor outgoingTracks = 6; 109 | repeated SFUStatusSender senders = 7; 110 | repeated MidToUmbrellaIDMapping midMapping = 8; 111 | repeated SFUStatusStagedIncomingTrack stagedIncomingTracks = 9; 112 | } -------------------------------------------------------------------------------- /razor/messagequeue.go: -------------------------------------------------------------------------------- 1 | package razor 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type MessageQueueWhat comparable 8 | 9 | type MessageQueueItem[MessageQueueWhat, PayloadType any] struct { 10 | at time.Time 11 | what MessageQueueWhat 12 | payload *PayloadType 13 | } 14 | 15 | type MessageQueue[What MessageQueueWhat, PayloadType any] struct { 16 | heap []MessageQueueItem[What, PayloadType] 17 | size uint16 18 | capacity uint16 19 | } 20 | 21 | func NewMessageQueue[MessageQueueWhat comparable, PayloadType any](capacity uint16) *MessageQueue[MessageQueueWhat, PayloadType] { 22 | return &MessageQueue[MessageQueueWhat, PayloadType]{ 23 | heap: make([]MessageQueueItem[MessageQueueWhat, PayloadType], capacity), 24 | size: 0, 25 | capacity: capacity, 26 | } 27 | } 28 | 29 | func (mq *MessageQueue[MessageQueueWhat, PayloadType]) CurrentSize() uint16 { 30 | return mq.size 31 | } 32 | 33 | func (mq *MessageQueue[MessageQueueWhat, PayloadType]) Capacity() uint16 { 34 | return mq.capacity 35 | } 36 | 37 | func (mq *MessageQueue[MessageQueueWhat, PayloadType]) TryPush(at time.Time, what MessageQueueWhat, payload *PayloadType) bool { 38 | Assert(mq.size < mq.capacity) 39 | 40 | if mq.size >= mq.capacity { 41 | return false 42 | } 43 | 44 | mq.heap[mq.size] = MessageQueueItem[MessageQueueWhat, PayloadType]{at: at, what: what, payload: payload} 45 | mq.size++ 46 | 47 | mq.heapifyUp(mq.size - 1) 48 | return true 49 | } 50 | 51 | func (mq *MessageQueue[MessageQueueWhat, PayloadType]) TryPop() Optional[MessageQueueItem[MessageQueueWhat, PayloadType]] { 52 | if mq.size == 0 { 53 | return NilOptional[MessageQueueItem[MessageQueueWhat, PayloadType]]() 54 | } 55 | 56 | item := mq.heap[0] 57 | mq.size-- 58 | mq.heap[0] = mq.heap[mq.size] 59 | mq.heapifyDown(0) 60 | 61 | return NewOptional(item) 62 | } 63 | 64 | func (mq *MessageQueue[MessageQueueWhat, PayloadType]) TryPopAtTime(at time.Time) Optional[MessageQueueItem[MessageQueueWhat, PayloadType]] { 65 | if mq.size == 0 { 66 | return NilOptional[MessageQueueItem[MessageQueueWhat, PayloadType]]() 67 | } 68 | 69 | item := mq.heap[0] 70 | if at.After(item.at) { 71 | mq.size-- 72 | mq.heap[0] = mq.heap[mq.size] 73 | mq.heapifyDown(0) 74 | 75 | return NewOptional(item) 76 | } else { 77 | return NilOptional[MessageQueueItem[MessageQueueWhat, PayloadType]]() 78 | } 79 | } 80 | 81 | func (mq *MessageQueue[MessageQueueWhat, PayloadType]) TryPeek() Optional[MessageQueueItem[MessageQueueWhat, PayloadType]] { 82 | if mq.size == 0 { 83 | return NilOptional[MessageQueueItem[MessageQueueWhat, PayloadType]]() 84 | } 85 | 86 | item := mq.heap[0] 87 | return NewOptional(item) 88 | } 89 | 90 | func (mq *MessageQueue[MessageQueueWhat, PayloadType]) Clear() { 91 | // Delete all buffers 92 | for i := uint16(0); i < mq.size; i++ { 93 | if mq.heap[i].payload != nil { 94 | // delete heap[i].payload; 95 | mq.heap[i].payload = nil 96 | } 97 | } 98 | 99 | // Set size to 0 100 | mq.size = 0 101 | } 102 | 103 | func (mq *MessageQueue[WHAT, PayloadType]) RemoveMessagesOfWhat(what WHAT) { 104 | for i := uint16(0); i < mq.size; { 105 | if mq.heap[i].what == what { 106 | // Delete the payload!! 107 | item := mq.heap[i] 108 | if item.payload != nil { 109 | // delete item.payload; 110 | mq.heap[i].payload = nil 111 | } 112 | 113 | // Replace the item with the last item 114 | mq.size-- 115 | if mq.size > 0 { 116 | mq.heap[i] = mq.heap[mq.size] 117 | mq.heapifyDown(i) // Restore the heap property 118 | } 119 | } else { 120 | i++ // Only move to the next item if no removal happened 121 | } 122 | } 123 | } 124 | 125 | func (mq *MessageQueue[MessageQueueWhat, PayloadType]) heapifyUp(index uint16) { 126 | for index > 0 { 127 | parent := (index - 1) / 2 128 | if mq.heap[index].at.Before(mq.heap[parent].at) { 129 | Swap(&mq.heap[index], &mq.heap[parent]) 130 | index = parent 131 | } else { 132 | return 133 | } 134 | } 135 | 136 | } 137 | 138 | func (mq *MessageQueue[MessageQueueWhat, PayloadType]) heapifyDown(index uint16) { 139 | for { 140 | left := 2*index + 1 141 | right := 2*index + 2 142 | smallest := index 143 | 144 | if left < mq.size && mq.heap[left].at.Before(mq.heap[smallest].at) { 145 | smallest = left 146 | } 147 | 148 | if right < mq.size && mq.heap[right].at.Before(mq.heap[smallest].at) { 149 | smallest = right 150 | } 151 | 152 | if smallest == index { 153 | // Completed! 154 | return 155 | } 156 | 157 | Swap(&mq.heap[index], &mq.heap[smallest]) 158 | index = smallest 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /sfu/peerconnection.go: -------------------------------------------------------------------------------- 1 | package sfu 2 | 3 | import ( 4 | "atomirex.com/umbrella/razor" 5 | "github.com/pion/rtcp" 6 | "github.com/pion/webrtc/v4" 7 | ) 8 | 9 | type PeerConnection struct { 10 | label string 11 | wrapped *webrtc.PeerConnection 12 | logger *razor.Logger 13 | 14 | OnICECandidate func(w *webrtc.ICECandidate) 15 | OnICEConnectionStateChange func(is webrtc.ICEConnectionState) 16 | OnSignalingStateChange func(ss webrtc.SignalingState) 17 | OnConnectionStateChange func(pcs webrtc.PeerConnectionState) 18 | OnTrack func(tr *webrtc.TrackRemote, r *webrtc.RTPReceiver) 19 | OnNegotiationNeeded func() 20 | } 21 | 22 | type PeerConnectionFactory interface { 23 | NewPeerConnection(label string) (*PeerConnection, error) 24 | } 25 | 26 | type PionPeerConnectionFactory struct { 27 | webrtcApi *webrtc.API 28 | pcConfig *webrtc.Configuration 29 | logger *razor.Logger 30 | } 31 | 32 | func (p *PionPeerConnectionFactory) NewPeerConnection(label string) (*PeerConnection, error) { 33 | pc, err := p.webrtcApi.NewPeerConnection(*p.pcConfig) 34 | 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | newPc := &PeerConnection{ 40 | label: label, 41 | wrapped: pc, 42 | logger: p.logger, 43 | } 44 | 45 | pc.OnICECandidate(func(i *webrtc.ICECandidate) { 46 | p.logger.Verbose(label, "Ice candidate") 47 | if newPc.OnICECandidate != nil { 48 | newPc.OnICECandidate(i) 49 | } 50 | }) 51 | 52 | pc.OnSignalingStateChange(func(ss webrtc.SignalingState) { 53 | p.logger.Info(label, "OnSignalingStateChange"+ss.String()) 54 | if newPc.OnSignalingStateChange != nil { 55 | newPc.OnSignalingStateChange(ss) 56 | } 57 | }) 58 | 59 | pc.OnICEConnectionStateChange(func(is webrtc.ICEConnectionState) { 60 | p.logger.Info(label, "OnICEConnectionStateChange"+is.String()) 61 | if newPc.OnICEConnectionStateChange != nil { 62 | newPc.OnICEConnectionStateChange(is) 63 | } 64 | }) 65 | 66 | pc.OnConnectionStateChange(func(pcs webrtc.PeerConnectionState) { 67 | p.logger.Info(label, "OnConnectionStateChange"+pcs.String()) 68 | if newPc.OnConnectionStateChange != nil { 69 | newPc.OnConnectionStateChange(pcs) 70 | } 71 | }) 72 | 73 | pc.OnTrack(func(tr *webrtc.TrackRemote, r *webrtc.RTPReceiver) { 74 | p.logger.Info(label, "OnTrack") 75 | if newPc.OnTrack != nil { 76 | newPc.OnTrack(tr, r) 77 | } 78 | }) 79 | 80 | pc.OnNegotiationNeeded(func() { 81 | p.logger.Info(label, "OnNegotiationNeeded") 82 | if newPc.OnNegotiationNeeded != nil { 83 | newPc.OnNegotiationNeeded() 84 | } 85 | }) 86 | 87 | return newPc, nil 88 | } 89 | 90 | func (pc *PeerConnection) GetStatus() *SFUStatusPeerConnection { 91 | return &SFUStatusPeerConnection{ 92 | ConnectionState: pc.wrapped.ConnectionState().String(), 93 | SignalingState: pc.wrapped.SignalingState().String(), 94 | IceConnectionState: pc.wrapped.ICEConnectionState().String(), 95 | IceGatheringState: pc.wrapped.ICEGatheringState().String(), 96 | 97 | TransceiverCount: int32(len(pc.wrapped.GetTransceivers())), 98 | SenderCount: int32(len(pc.wrapped.GetSenders())), 99 | ReceiverCount: int32(len(pc.wrapped.GetReceivers())), 100 | } 101 | } 102 | 103 | func (pc *PeerConnection) Close() error { 104 | pc.logger.Info(pc.label, "Closing") 105 | 106 | err := pc.wrapped.Close() 107 | 108 | pc.logger.NilErrCheck(pc.label, "Error when closing", err) 109 | 110 | return err 111 | } 112 | 113 | func (pc *PeerConnection) CreateDataChannel(name string, params *webrtc.DataChannelInit) (*webrtc.DataChannel, error) { 114 | return pc.wrapped.CreateDataChannel(name, params) 115 | } 116 | 117 | func (pc *PeerConnection) GetReceivers() []*webrtc.RTPReceiver { 118 | return pc.wrapped.GetReceivers() 119 | } 120 | 121 | func (pc *PeerConnection) GetSenders() []*webrtc.RTPSender { 122 | return pc.wrapped.GetSenders() 123 | } 124 | 125 | func (pc *PeerConnection) GetTransceivers() []*webrtc.RTPTransceiver { 126 | return pc.wrapped.GetTransceivers() 127 | } 128 | 129 | func (pc *PeerConnection) WriteRTCP(pkts []rtcp.Packet) error { 130 | return pc.wrapped.WriteRTCP(pkts) 131 | } 132 | 133 | func (pc *PeerConnection) AddTransceiverFromKind(kind webrtc.RTPCodecType, params webrtc.RTPTransceiverInit) (*webrtc.RTPTransceiver, error) { 134 | return pc.wrapped.AddTransceiverFromKind(kind, params) 135 | } 136 | 137 | func (pc *PeerConnection) IsTerminated() bool { 138 | switch pc.wrapped.ConnectionState() { 139 | case webrtc.PeerConnectionStateClosed: 140 | case webrtc.PeerConnectionStateDisconnected: 141 | case webrtc.PeerConnectionStateFailed: 142 | return true 143 | } 144 | 145 | return false 146 | } 147 | 148 | func (pc *PeerConnection) AddICECandidate(candidate webrtc.ICECandidateInit) error { 149 | return pc.wrapped.AddICECandidate(candidate) 150 | } 151 | 152 | func (pc *PeerConnection) CreateAnswer(options *webrtc.AnswerOptions) (webrtc.SessionDescription, error) { 153 | return pc.wrapped.CreateAnswer(options) 154 | } 155 | 156 | func (pc *PeerConnection) CreateOffer(options *webrtc.OfferOptions) (webrtc.SessionDescription, error) { 157 | return pc.wrapped.CreateOffer(options) 158 | } 159 | 160 | func (pc *PeerConnection) SetRemoteDescription(description webrtc.SessionDescription) error { 161 | return pc.wrapped.SetRemoteDescription(description) 162 | } 163 | 164 | func (pc *PeerConnection) SetLocalDescription(description webrtc.SessionDescription) error { 165 | return pc.wrapped.SetLocalDescription(description) 166 | } 167 | 168 | func (pc *PeerConnection) AddTrack(track webrtc.TrackLocal) (*webrtc.RTPSender, error) { 169 | return pc.wrapped.AddTrack(track) 170 | } 171 | 172 | func (pc *PeerConnection) RemoveTrack(sender *webrtc.RTPSender) error { 173 | return pc.wrapped.RemoveTrack(sender) 174 | } 175 | -------------------------------------------------------------------------------- /sfu/client_rtsp.go: -------------------------------------------------------------------------------- 1 | package sfu 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "atomirex.com/umbrella/razor" 8 | "github.com/bluenviron/gortsplib/v4" 9 | "github.com/bluenviron/gortsplib/v4/pkg/base" 10 | "github.com/bluenviron/gortsplib/v4/pkg/description" 11 | "github.com/bluenviron/gortsplib/v4/pkg/format" 12 | "github.com/google/uuid" 13 | "github.com/pion/rtcp" 14 | "github.com/pion/rtp" 15 | "github.com/pion/webrtc/v4" 16 | ) 17 | 18 | // Very experimental rtsp video only ingest 19 | // Serving as basis of client refactor and understanding more about rtsp 20 | 21 | type rtspClientCommand int 22 | 23 | const ( 24 | rtspClientStop rtspClientCommand = iota 25 | rtspClientDial 26 | ) 27 | 28 | type rtspClientCommandMessage struct{} 29 | 30 | type RtspClient struct { 31 | BaseClient 32 | 33 | url string 34 | handler *razor.MessageHandler[rtspClientCommand, rtspClientCommandMessage] 35 | 36 | rtsplibClient *gortsplib.Client 37 | rtspDescription *description.Session 38 | } 39 | 40 | func (r *RtspClient) stop() { 41 | r.handler.Send(rtspClientStop, nil) 42 | } 43 | 44 | func (r *RtspClient) run(s *Sfu) { 45 | r.handler = razor.NewMessageHandler(r.logger, r.label, 16, func(what rtspClientCommand, payload *rtspClientCommandMessage) bool { 46 | switch what { 47 | case rtspClientStop: 48 | if r.rtsplibClient != nil { 49 | r.rtsplibClient.Close() 50 | r.rtsplibClient = nil 51 | } 52 | 53 | r.handler.Abort() 54 | return true 55 | case rtspClientDial: 56 | if r.rtsplibClient != nil { 57 | return true 58 | } 59 | 60 | failedCheck := func(err error) bool { 61 | if err == nil { 62 | return false 63 | } 64 | 65 | log.Println(err) 66 | 67 | if r.rtsplibClient != nil { 68 | r.rtsplibClient.Close() 69 | r.rtsplibClient = nil 70 | } 71 | 72 | r.handler.Timeout(rtspClientDial, nil, 5000*time.Millisecond) 73 | return true 74 | } 75 | 76 | u, err := base.ParseURL(r.url) 77 | if err != nil { 78 | log.Println("Invalid url", err) 79 | return true 80 | } 81 | 82 | r.rtsplibClient = &gortsplib.Client{} 83 | err = r.rtsplibClient.Start(u.Scheme, u.Host) 84 | if failedCheck(err) { 85 | return true 86 | } 87 | 88 | r.rtspDescription, _, err = r.rtsplibClient.Describe(u) 89 | if failedCheck(err) { 90 | return true 91 | } 92 | 93 | // Video only for now with the eufy 94 | // Audio breaks things massively, and doesn't work at all 95 | // Might be best to work out how to use go2rtc libraries 96 | videointrack := &incomingTrack{ 97 | descriptor: &TrackDescriptor{ 98 | UmbrellaId: "UMB_ID" + uuid.NewString(), 99 | Kind: TrackKind_Video, 100 | }, 101 | } 102 | 103 | videorelay, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{ 104 | MimeType: webrtc.MimeTypeH264, 105 | ClockRate: 90000, 106 | Channels: 0, 107 | SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f", 108 | RTCPFeedback: nil, 109 | }, "UMB_RSTP_SRC"+uuid.New().String(), "rtsp-src-stream-id"+uuid.NewString()) 110 | 111 | videointrack.relay = videorelay 112 | 113 | if failedCheck(err) { 114 | return true 115 | } 116 | 117 | go func() { 118 | defer s.removeOutgoingTracksForIncomingTrack(videointrack) 119 | 120 | // setup all medias 121 | // this must be called before StartRecording(), since it overrides the control attribute. 122 | err = r.rtsplibClient.SetupAll(r.rtspDescription.BaseURL, r.rtspDescription.Medias) 123 | if failedCheck(err) { 124 | return 125 | } 126 | 127 | // read RTP packets from the reader and route them to the publisher 128 | r.rtsplibClient.OnPacketRTPAny(func(medi *description.Media, forma format.Format, pkt *rtp.Packet) { 129 | if medi.Type == description.MediaTypeVideo { 130 | p := pkt.Clone() 131 | p.Extension = false 132 | p.Extensions = nil 133 | 134 | if err := videointrack.relay.WriteRTP(p); err != nil { 135 | r.logger.Error(r.label, "Error writing rtp from "+videointrack.String()+" to relay "+err.Error()) 136 | return 137 | } 138 | } 139 | }) 140 | 141 | rtcpStop := make(chan struct{}) 142 | 143 | go func() { 144 | // Otherwise it basically never does this! 145 | keyframehack := time.NewTicker(12 * time.Second) 146 | defer keyframehack.Stop() 147 | 148 | // Ugh 149 | defer func() { 150 | if r := recover(); r != nil { 151 | log.Println("RTCP panicked", r) 152 | } 153 | }() 154 | 155 | for { 156 | select { 157 | case _, ok := <-rtcpStop: 158 | if !ok { 159 | return 160 | } 161 | case <-keyframehack.C: 162 | err := r.rtsplibClient.WritePacketRTCP(r.rtspDescription.Medias[0], &rtcp.PictureLossIndication{}) 163 | if err != nil { 164 | return 165 | } 166 | } 167 | } 168 | }() 169 | 170 | // start playing 171 | _, err = r.rtsplibClient.Play(nil) 172 | failedCheck(err) 173 | 174 | failedCheck(r.rtsplibClient.Wait()) 175 | 176 | // Properly stop the other goroutines 177 | // Dislike this, especially needing to check for panics 178 | close(rtcpStop) 179 | }() 180 | 181 | s.handler.Send(sfuAddOutgoingTracksForIncomingTrack, &sfuCommandMessage{intrack: videointrack}) 182 | 183 | return true 184 | } 185 | return true 186 | }) 187 | 188 | r.handler.Send(rtspClientDial, nil) 189 | 190 | r.handler.Loop(func() { 191 | }) 192 | } 193 | 194 | func (r *RtspClient) AddOutgoingTracksForIncomingTrack(intrack *incomingTrack) { 195 | // Do nothing 196 | } 197 | 198 | func (r *RtspClient) RemoveOutgoingTracksForIncomingTrack(intrack *incomingTrack) { 199 | // Do nothing 200 | } 201 | 202 | func (r *RtspClient) RequestEvalState() { 203 | 204 | } 205 | 206 | func (r *RtspClient) getStatus() *SFUStatusClient { 207 | return nil 208 | } 209 | -------------------------------------------------------------------------------- /razor/messagehandler.go: -------------------------------------------------------------------------------- 1 | package razor 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // Returns if the payload should be destroyed, which in golang doesn't do anything 9 | type MessageHandlerCallback[MessageQueueWhat comparable, PayloadType any] func(what MessageQueueWhat, payload *PayloadType) bool 10 | 11 | type MessageHandler[MessageQueueWhat comparable, PayloadType any] struct { 12 | label string 13 | 14 | queue *MessageQueue[MessageQueueWhat, PayloadType] 15 | callback MessageHandlerCallback[MessageQueueWhat, PayloadType] 16 | lock *LockAndNotify 17 | 18 | aborted bool 19 | 20 | logger *Logger 21 | } 22 | 23 | func NewMessageHandler[MessageQueueWhat comparable, PayloadType any](logger *Logger, label string, queueLength uint16, callback MessageHandlerCallback[MessageQueueWhat, PayloadType]) *MessageHandler[MessageQueueWhat, PayloadType] { 24 | return &MessageHandler[MessageQueueWhat, PayloadType]{ 25 | label: label, 26 | queue: NewMessageQueue[MessageQueueWhat, PayloadType](queueLength), 27 | callback: callback, 28 | lock: NewLockAndNotify(), 29 | aborted: false, 30 | logger: logger, 31 | } 32 | } 33 | 34 | func (mh *MessageHandler[MessageQueueWhat, PayloadType]) Size() uint16 { 35 | return mh.queue.CurrentSize() 36 | } 37 | 38 | func (mh *MessageHandler[MessageQueueWhat, PayloadType]) Capacity() uint16 { 39 | return mh.queue.Capacity() 40 | } 41 | 42 | func (mh *MessageHandler[MessageQueueWhat, PayloadType]) Send(what MessageQueueWhat, payload *PayloadType) bool { 43 | mh.logger.Trace(mh.label, fmt.Sprintf("Send: %v", what)) 44 | mh.lock.Enter() 45 | 46 | if mh.aborted { 47 | mh.logger.Info(mh.label, "Send called on aborted messagehandler") 48 | mh.lock.Leave() 49 | 50 | return false 51 | } else { 52 | result := mh.queue.TryPush(time.Now(), what, payload) 53 | 54 | mh.lock.Leave() 55 | 56 | mh.logger.Assert(mh.label, "Handler queue too short in send", !result) 57 | 58 | mh.lock.Notify() 59 | 60 | return result 61 | } 62 | } 63 | 64 | func (mh *MessageHandler[MessageQueueWhat, PayloadType]) Timeout(what MessageQueueWhat, payload *PayloadType, delay time.Duration) bool { 65 | mh.logger.Trace(mh.label, fmt.Sprintf("Timeout: %v %s", what, delay)) 66 | mh.lock.Enter() 67 | 68 | if mh.aborted { 69 | mh.logger.Info(mh.label, "Timeout called on aborted messagehandler") 70 | mh.lock.Leave() 71 | return false 72 | } else { 73 | result := mh.queue.TryPush(time.Now().Add(delay), what, payload) 74 | mh.lock.Leave() 75 | 76 | mh.logger.Assert(mh.label, "Handler queue too short in timout", !result) 77 | 78 | mh.lock.Notify() 79 | 80 | return result 81 | } 82 | } 83 | 84 | func (mh *MessageHandler[MessageQueueWhat, PayloadType]) Cancel(what MessageQueueWhat) { 85 | mh.logger.Trace(mh.label, fmt.Sprintf("Cancel: %v", what)) 86 | mh.lock.Enter() 87 | 88 | if mh.aborted { 89 | mh.logger.Info(mh.label, "Cancel called on aborted messagehandler") 90 | mh.lock.Leave() 91 | } else { 92 | mh.queue.RemoveMessagesOfWhat(what) 93 | 94 | mh.lock.Leave() 95 | mh.lock.Notify() 96 | } 97 | } 98 | 99 | func (mh *MessageHandler[MessageQueueWhat, PayloadType]) CancelAll() { 100 | mh.logger.Trace(mh.label, "CancelAll") 101 | mh.lock.Enter() 102 | 103 | if mh.aborted { 104 | mh.logger.Info(mh.label, "CancelAll called on aborted messagehandler (will happen once every time)") 105 | mh.lock.Leave() 106 | } else { 107 | mh.queue.Clear() 108 | 109 | mh.lock.Leave() 110 | mh.lock.Notify() 111 | } 112 | } 113 | 114 | func (mh *MessageHandler[MessageQueueWhat, PayloadType]) DoWork(now time.Time) { 115 | max_messages_per_update := 10 116 | 117 | for max_messages_per_update > 0 { 118 | // See if we have any work to do, and if so to do it 119 | mh.lock.Enter() 120 | itemOptional := mh.queue.TryPopAtTime(now) 121 | mh.lock.Leave() 122 | 123 | if itemOptional.IsSet { 124 | // Do the work 125 | item := itemOptional.Value 126 | 127 | mh.logger.Trace(mh.label, fmt.Sprintf("WorkEvent: %v", item.what)) 128 | // completed := make(chan bool, 0) 129 | 130 | // go func() { 131 | destroyBuffer := mh.callback(item.what, item.payload) 132 | if destroyBuffer { 133 | // Not really needed in golang 134 | } 135 | // completed <- true 136 | // }() 137 | 138 | // select { 139 | // case <-time.After(200 * time.Millisecond): 140 | // pstr := fmt.Sprintf("Handler %s took too long processing work item %v", mh.label, item.what) 141 | // panic(pstr) 142 | // case <-completed: 143 | // } 144 | mh.logger.Trace(mh.label, fmt.Sprintf("WorkEventCompleted: %v", item.what)) 145 | 146 | max_messages_per_update-- 147 | } else { 148 | return 149 | } 150 | } 151 | } 152 | 153 | func (mh *MessageHandler[MessageQueueWhat, PayloadType]) NextWorkAt() time.Time { 154 | mh.lock.Enter() 155 | itemOptional := mh.queue.TryPeek() 156 | mh.lock.Leave() 157 | 158 | if itemOptional.IsSet { 159 | return itemOptional.Value.at 160 | } else { 161 | // Sleep until some work arrives 162 | return EndOfTime 163 | } 164 | } 165 | 166 | func (mh *MessageHandler[MessageQueueWhat, PayloadType]) Loop(oncleanup func()) { 167 | go func() { 168 | work := func() { 169 | for { 170 | if mh.aborted { 171 | return 172 | } 173 | 174 | now := time.Now() 175 | waitUntil := now.Add(5 * time.Second) 176 | 177 | mh.DoWork(now) 178 | 179 | nextWork := mh.NextWorkAt() 180 | 181 | if nextWork.Before(waitUntil) { 182 | waitUntil = nextWork 183 | } 184 | 185 | if now.Before(waitUntil) { 186 | mh.lock.Wait(waitUntil.Sub(now)) 187 | } else { 188 | // This is because we must call wait regularly to stop the waker in lock from blocking! 189 | mh.lock.Wait(5 * time.Millisecond) 190 | } 191 | } 192 | } 193 | 194 | work() 195 | if oncleanup != nil { 196 | oncleanup() 197 | } 198 | }() 199 | } 200 | 201 | // Clears all work and stops any loops 202 | // Does not block waiting though 203 | func (mh *MessageHandler[MessageQueueWhat, PayloadType]) Abort() { 204 | mh.logger.Trace(mh.label, "Abort") 205 | mh.lock.Enter() 206 | mh.aborted = true 207 | mh.lock.Leave() 208 | 209 | mh.CancelAll() 210 | mh.logger.Trace(mh.label, "Abort returned") 211 | } 212 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/atomirex/mdns v0.0.13-0.20241125201824-943da8fa1231 h1:91mEHouufaTTLYjiqHY6C2vBEYW7xPdWmS/FmthixJ0= 2 | github.com/atomirex/mdns v0.0.13-0.20241125201824-943da8fa1231/go.mod h1:nhf3AaaXAZYM8Za49tsvZW5sEB+UO2OHKydJKAkbcyU= 3 | github.com/bluenviron/gortsplib/v4 v4.11.2 h1:V9WjA9sY99X0OiQyz/JgLOMeHaXyOcE3XsOIU+yQS4U= 4 | github.com/bluenviron/gortsplib/v4 v4.11.2/go.mod h1:H6bdvXU0+poDcR0etOvdcwsNKC/1xzAMVuBNO4hxeL4= 5 | github.com/bluenviron/mediacommon v1.13.1 h1:agxDtkooknxSxOO/oOpB+tEW48OLMqty1PDMC3x2n4E= 6 | github.com/bluenviron/mediacommon v1.13.1/go.mod h1:HDyW2CzjvhYJXtdxstdFPio3G0qSocPhqkhUt/qffec= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 11 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 12 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 13 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 14 | github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= 15 | github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 16 | github.com/pion/datachannel v1.5.9 h1:LpIWAOYPyDrXtU+BW7X0Yt/vGtYxtXQ8ql7dFfYUVZA= 17 | github.com/pion/datachannel v1.5.9/go.mod h1:kDUuk4CU4Uxp82NH4LQZbISULkX/HtzKa4P7ldf9izE= 18 | github.com/pion/dtls/v3 v3.0.3 h1:j5ajZbQwff7Z8k3pE3S+rQ4STvKvXUdKsi/07ka+OWM= 19 | github.com/pion/dtls/v3 v3.0.3/go.mod h1:weOTUyIV4z0bQaVzKe8kpaP17+us3yAuiQsEAG1STMU= 20 | github.com/pion/ice/v4 v4.0.2 h1:1JhBRX8iQLi0+TfcavTjPjI6GO41MFn4CeTBX+Y9h5s= 21 | github.com/pion/ice/v4 v4.0.2/go.mod h1:DCdqyzgtsDNYN6/3U8044j3U7qsJ9KFJC92VnOWHvXg= 22 | github.com/pion/interceptor v0.1.37 h1:aRA8Zpab/wE7/c0O3fh1PqY0AJI3fCSEM5lRWJVorwI= 23 | github.com/pion/interceptor v0.1.37/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y= 24 | github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= 25 | github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= 26 | github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= 27 | github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= 28 | github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= 29 | github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= 30 | github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= 31 | github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= 32 | github.com/pion/rtp v1.8.9 h1:E2HX740TZKaqdcPmf4pw6ZZuG8u5RlMMt+l3dxeu6Wk= 33 | github.com/pion/rtp v1.8.9/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= 34 | github.com/pion/sctp v1.8.33 h1:dSE4wX6uTJBcNm8+YlMg7lw1wqyKHggsP5uKbdj+NZw= 35 | github.com/pion/sctp v1.8.33/go.mod h1:beTnqSzewI53KWoG3nqB282oDMGrhNxBdb+JZnkCwRM= 36 | github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= 37 | github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= 38 | github.com/pion/srtp/v3 v3.0.4 h1:2Z6vDVxzrX3UHEgrUyIGM4rRouoC7v+NiF1IHtp9B5M= 39 | github.com/pion/srtp/v3 v3.0.4/go.mod h1:1Jx3FwDoxpRaTh1oRV8A/6G1BnFL+QI82eK4ms8EEJQ= 40 | github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw= 41 | github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU= 42 | github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= 43 | github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= 44 | github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM= 45 | github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA= 46 | github.com/pion/webrtc/v4 v4.0.1 h1:6Unwc6JzoTsjxetcAIoWH81RUM4K5dBc1BbJGcF9WVE= 47 | github.com/pion/webrtc/v4 v4.0.1/go.mod h1:SfNn8CcFxR6OUVjLXVslAQ3a3994JhyE3Hw1jAuqEto= 48 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 49 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 50 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 51 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 52 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 53 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 54 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 55 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 56 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 57 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 58 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 59 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 60 | github.com/wlynxg/anet v0.0.3 h1:PvR53psxFXstc12jelG6f1Lv4MWqE0tI76/hHGjh9rg= 61 | github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= 62 | golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= 63 | golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= 64 | golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= 65 | golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= 66 | golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= 67 | golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 68 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 69 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 70 | google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= 71 | google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 72 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 73 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 74 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 75 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Umbrella 2 | 3 | Demo video: https://x.com/atomirex/status/1863956802984984728 4 | 5 | This is a hacked up proof of concept WebRTC SFU that can run on many things, such as a local OpenWrt AP or in the cloud with Docker Compose/Traefik, including having a group session that spans both at once by backhauling from the AP to the cloud, and ingesting video from security cameras. Today it serves to help explore the problem space more than being any solution to anything. 6 | 7 | It's not production ready, mainly having almost no security! It also lacks quite a lot of potential optimizations. If you need a production video conf system today try https://livekit.io or https://www.daily.co (unaffiliated with either). It's also not very selective, just forwarding everything to everyone. 8 | 9 | This began as a fork of the pion example sfu-ws to see if it could run on OpenWrt. https://github.com/pion/example-webrtc-applications/tree/master/sfu-ws 10 | 11 | ## What is it? 12 | 13 | A Selective-Forwarding-Unit is a server that relays incoming traffic from one client by copying it to any number of other clients, system resources permitting. In the context of WebRTC it is decrypting the packets that it receives, and then re-encrypting them before forwarding them on, but it is not actually decoding and reencoding the media streams. A TURN ( https://en.wikipedia.org/wiki/Traversal_Using_Relays_around_NAT ) relay does not decode or decrypt a stream, but does not fan out the incoming traffic, only acting point-to-point, and so serves different purposes. 14 | 15 | Reasons you might want a SFU: 16 | * Isolating clients so they only see the IP of the SFU and not of each other - in a normal WebRTC session clients can see the IPs of other clients, which may present a security problem. 17 | * A central location for recording streams - since we are decrypting and not decoding the streams you can cheaply dump the decrypted but compressed media packets to disk to replay later. (Pion includes other examples showing this). 18 | * Reducing the amount of bandwidth needed at each node in contexts such as large group calls - in a normal P2P WebRTC session each additional participant must connect directly to every other participant causing an explosion in the number of connections it has to maintain, each of which is carrying a copy of the media. A central SFU provides a single point for all clients to connect to and absorbs the load of distributing the data, hopefully leveraging better connectivity. 19 | 20 | ![Diagram of relative at node cost of P2P vs SFU configurations](docs/sfu-diagram-1.svg?raw=true "The relative cost of P2P vs SFU configurations for edge nodes") 21 | 22 | There are also reasons you might not want an SFU, primarily if you do not trust the machine the SFUs are operated on. 23 | 24 | The semi-novel bit here is the backhaul from the assumed AP host to the cloud. The idea is this reduces load on the link between them, enabling different clients at each end to share the transported media streams. 25 | 26 | ![Diagram of dual SFU with backhaul between them](docs/sfu-diagram-2.svg?raw=true "Dual SFUs with backhaul") 27 | 28 | There are possible extensions to this like making the cloud server purely signalling and connecting two APs at separate sites directly together, and even merging the network connection negotiation information so devices can dynamically switch which SFU to use based on what is optimal for their situation. 29 | 30 | ## How do I try it out? 31 | It's fairly easy to build/run on a generic desktop, assuming you meet the security prerequisites and have the dependencies. The OpenWrt and Docker/traefik sections detail specific aspects of those environments. 32 | 33 | It would be unwise to leave this running on a public network as is for very long, and I cannot be responsible for anything that would result! Try it at your own risk, ideally somewhere unlikely to be found. 34 | 35 | ### Security prerequisites 36 | Using WebRTC features in a browser requires the browser to believe it is in a "secure context", which in practice for this means serving https. For the cloud side of things this is fairly easy since you can use something like letsencrypt in the normal way. 37 | 38 | Running the SFU on a local subnet, such as that run by an AP, requires creating a key and certificate for the server so devices can keep track of the identity of the server. This is detailed in [docs/KEYS_SETUP.md](docs/KEYS_SETUP.md) . 39 | 40 | ### Building 41 | The main SFU is a golang project using the pion library for webrtc functionality. Protobufs are used for communication, and there is a small React/Typescript front end which gets bundled into the binary for ease of deployment. The main entry points for building are the scripts build_PROCESSOR_OS.sh scripts which use the golang cross compilation support to build for the different target architectures. The frontend project is built with esbuild and triggered via the npm commands in build_common.sh, along with the protoc compilation step. 42 | 43 | To build you need: 44 | * golang 45 | * npm 46 | * protoc, including the golang support 47 | 48 | Then run one of the shell scripts, and it should produce an executable at ./umbrella . 49 | 50 | #### OpenWrt 51 | One of the main targets for this is access points running OpenWrt. More details are in [docs/openwrt.md](docs/openwrt.md) . 52 | 53 | #### Docker/traefik 54 | For running on a public webserver something like docker compose with traefik will take away a lot of headaches. More details are in [docs/docker.md](docs/docker.md) . 55 | 56 | ### Running it 57 | 58 | #### Locally 59 | Running locally should be a question of running the umbrella executable, in a directory with the service.crt and service.key files created earlier (see [docs/KEYS_SETUP.md](docs/KEYS_SETUP.md) ). Port 8081 will need to be available and accessible. Then you can visit: https://HOSTNAME:8081/sfu 60 | 61 | #### Cloud 62 | The easiest way to run it in the cloud is dockerized behind something like a traefik reverse proxy. This way traefik can act as the https endpoint, taking care of the certs, leaving the only real problem being ensuring you use host networking. 63 | 64 | The docker-compose-example.yml in deployment shows how you can run it as a sub page of an existing domain. 65 | 66 | When it's running you should be able to access it at https://DOMAIN/umbrella/sfu . 67 | 68 | #### Connecting the two 69 | The real party trick here is a joint session across SFUs. To do this access the server to push to the other (most likely local to cloud, so local), and visit https://HOSTNAME:8081/servers . In the text field put the websocket address for the other server and press the button. The websocket address is "wsb" instead of "sfu", and the protocol is "wss" instead of "https". For example: wss://DOMAIN/umbrella/wsb To stop trunking remove the server connection. 70 | 71 | What should happen is all client data for each SFU is relayed to every client of both SFUs, as the result of the data going over the connection between the relays. 72 | 73 | This is a hacked up UI, but it does work. 74 | 75 | #### Using RTSP for cameras 76 | Assuming you can access the rtsp feed of a camera (verifiable using VLC) you can ingest from the camera. Different camera brands are more/less reliable for this, and the whole feature is highly experimental, creating a whole load of new problems. 77 | 78 | A RTSP camera acts as a server, so to use it you add a server on the same page as connecting between SFUs, i.e. https://HOSTNAME:8081/servers The format is something like rtsp://yourcameraname:yourcamerapassword@CAMERAIP/stream1 but it will vary based on your camera brand. 79 | 80 | This only brings in video for now. 81 | 82 | ## How do I develop against it? 83 | This is a real proof of concept mess. Any focused PRs or issues are welcome, as are forks. Assume zero stability at this stage. 84 | 85 | The immediate priority is to get this out there to see what happens, and then work on adding "role" type functionality, primarily so signalling can be separated from the SFU so that two remote sites can directly bridge between them. Other things likely to happen soon are auth and adding an mqtt broker for IoT integrations. 86 | 87 | If you're not into the whole multi-site aspect of it you're almost certainly better off with livekit or daily as mentioned at the top! 88 | 89 | ## What does it not do? 90 | * Authentication - there isn't any at all. 91 | * Simulcast - a feature where clients upload streams at multiple different bitrates, and the SFU forwards on the most appropriate bitrate available for each client depending on the available bandwidth. 92 | * "Pull" optimizations - right now media is forwarded to endpoints whether it is consumed there or not. For example, if you have backhaul to the cloud active all AP client media is forwarded to the cloud even if no clients are connected to the cloud instance. 93 | * Cycles in backhaul will explode. It can deal with star topologies but because each node simply relays everything right now a cycle will go very wrong. -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "embed" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "io/fs" 10 | "log" 11 | "net" 12 | "net/http" 13 | "os" 14 | "runtime" 15 | "runtime/debug" 16 | "strconv" 17 | "strings" 18 | "text/template" 19 | "time" 20 | 21 | "atomirex.com/umbrella/razor" 22 | "atomirex.com/umbrella/sfu" 23 | "github.com/atomirex/mdns" 24 | "golang.org/x/net/ipv4" 25 | "golang.org/x/net/ipv6" 26 | "google.golang.org/protobuf/proto" 27 | ) 28 | 29 | // nolint 30 | var () 31 | 32 | //go:embed frontend/dist/static/* 33 | var staticFiles embed.FS 34 | 35 | //go:embed frontend/dist/templates/* 36 | var templateFiles embed.FS 37 | 38 | type responseRecorder struct { 39 | http.ResponseWriter 40 | statusCode int 41 | } 42 | 43 | func (rec *responseRecorder) WriteHeader(code int) { 44 | rec.statusCode = code 45 | rec.ResponseWriter.WriteHeader(code) 46 | } 47 | 48 | // Implement http.Hijacker for websocket 49 | func (rr *responseRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { 50 | hijacker, ok := rr.ResponseWriter.(http.Hijacker) 51 | if !ok { 52 | return nil, nil, fmt.Errorf("underlying ResponseWriter does not support hijacking") 53 | } 54 | return hijacker.Hijack() 55 | } 56 | 57 | type umbrellaInjectedParameters struct { 58 | HttpPrefix string 59 | } 60 | 61 | func main() { 62 | cloudEnv := os.Getenv("UMBRELLA_CLOUD") 63 | isCloud := cloudEnv == "1" 64 | 65 | // Path from the domain base, to the umbrella sfu base, including any leading slash 66 | // For example "/umbrella" serves at https://domain:port/umbrella/sfu etc. 67 | // This is the same form traefik uses for PathPrefix, so you can use a common env var 68 | httpPrefixEnv := os.Getenv("UMBRELLA_HTTP_PREFIX") 69 | httpPrefix := "" 70 | 71 | if httpPrefixEnv != "" { 72 | httpPrefix = httpPrefixEnv 73 | } 74 | 75 | publicIpEnv := os.Getenv("UMBRELLA_PUBLIC_IP") 76 | publicIp := net.ParseIP(publicIpEnv) 77 | 78 | publicHostEnv := os.Getenv("UMBRELLA_PUBLIC_HOST") 79 | 80 | publicMinPortEnv := os.Getenv("UMBRELLA_MIN_PORT") 81 | publicMaxPortEnv := os.Getenv("UMBRELLA_MAX_PORT") 82 | 83 | minPort := uint16(40000) 84 | maxPort := uint16(60000) 85 | 86 | if publicMinPortEnv != "" { 87 | p, err := strconv.Atoi(publicMinPortEnv) 88 | if err == nil { 89 | minPort = uint16(p) 90 | } 91 | } 92 | 93 | if publicMaxPortEnv != "" { 94 | p, err := strconv.Atoi(publicMaxPortEnv) 95 | if err == nil { 96 | maxPort = uint16(p) 97 | } 98 | } 99 | 100 | httpServeAddrEnv := os.Getenv("UMBRELLA_HTTP_SERVE_ADDR") 101 | 102 | // Default is to serve on 8081 103 | httpServeAddr := ":8081" 104 | if httpServeAddrEnv != "" { 105 | httpServeAddr = httpServeAddrEnv 106 | } 107 | 108 | log.Println("Hello there", runtime.GOOS, runtime.GOARCH) 109 | if isCloud { 110 | log.Println("Running in cloud configuration") 111 | 112 | if publicIp == nil { 113 | log.Println("WARNING: public IP must be set when running as cloud") 114 | } else { 115 | log.Println("Public IP is", publicIp.String()) 116 | } 117 | } else { 118 | log.Println("Running in edge configuration") 119 | } 120 | 121 | debug.SetMemoryLimit(256 * 1024 * 1024) 122 | 123 | host, err := os.Hostname() 124 | if err != nil { 125 | log.Fatal(err) 126 | return 127 | } 128 | 129 | if !strings.HasSuffix(host, ".local") { 130 | host = host + ".local" 131 | } 132 | 133 | if isCloud && publicHostEnv != "" { 134 | host = publicHostEnv 135 | } else { 136 | overriddenhost, overriddenport, err := net.SplitHostPort(httpServeAddr) 137 | if err != nil { 138 | host += ":8081" 139 | } else { 140 | usehost := host 141 | useport := "8081" 142 | 143 | if overriddenhost != "" { 144 | usehost = overriddenhost 145 | } 146 | 147 | if overriddenport != "" { 148 | useport = overriddenport 149 | } 150 | 151 | host = net.JoinHostPort(usehost, useport) 152 | } 153 | } 154 | 155 | log.Println("Starting server at", fmt.Sprintf("https://%v%s/", strings.ToLower(host), httpPrefix)) 156 | 157 | templates, err := template.ParseFS(templateFiles, "frontend/dist/templates/*.html") 158 | if err != nil { 159 | log.Fatal(err) 160 | return 161 | } 162 | 163 | staticFilesSub, err := fs.Sub(staticFiles, "frontend/dist/static") 164 | if err != nil { 165 | log.Fatalf("failed to create file system: %v", err) 166 | } 167 | 168 | var ipStr *string 169 | if publicIp != nil { 170 | s := publicIp.String() 171 | ipStr = &s 172 | } 173 | 174 | logger := razor.NewLogger(razor.LogLevelError, false) 175 | s := sfu.NewSfu(logger, minPort, maxPort, ipStr) 176 | 177 | mux := http.NewServeMux() 178 | 179 | addHandler := func(pattern string, handler http.Handler) { 180 | if httpPrefix == "" { 181 | mux.Handle(pattern, handler) 182 | } else { 183 | mux.Handle(httpPrefix+pattern, http.StripPrefix(httpPrefix, handler)) 184 | } 185 | } 186 | 187 | addHandler("/wsb", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 188 | s.WebsocketHandler(w, r) 189 | })) 190 | 191 | addHandler("/static/", http.StripPrefix("/static/", http.FileServer(http.FS(staticFilesSub)))) 192 | 193 | generic := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 194 | if r.URL.Path == "/status" { 195 | contentType := r.Header.Get("Content-Type") 196 | if contentType == "application/x-protobuf" { 197 | status := s.GetStatus() 198 | 199 | log.Println("SFU STATUS", status) 200 | 201 | data, err := proto.Marshal(status) 202 | if err != nil { 203 | http.Error(w, "Failed to serialize Protobuf", http.StatusInternalServerError) 204 | return 205 | } 206 | 207 | w.Header().Set("Content-Type", contentType) 208 | w.WriteHeader(http.StatusOK) 209 | 210 | w.Write(data) 211 | return 212 | } 213 | } 214 | 215 | if r.URL.Path == "/servers" { 216 | contentType := r.Header.Get("Content-Type") 217 | if contentType == "application/x-protobuf" { 218 | switch r.Method { 219 | case http.MethodGet: 220 | data, err := proto.Marshal(s.GetCurrentServers()) 221 | if err != nil { 222 | http.Error(w, "Failed to serialize Protobuf", http.StatusInternalServerError) 223 | return 224 | } 225 | 226 | w.Header().Set("Content-Type", contentType) 227 | w.WriteHeader(http.StatusOK) 228 | 229 | w.Write(data) 230 | return 231 | case http.MethodPost: 232 | body, err := io.ReadAll(r.Body) 233 | if err != nil { 234 | http.Error(w, "Failed to read payload", http.StatusInternalServerError) 235 | return 236 | } 237 | 238 | var update sfu.CurrentServers 239 | err = proto.Unmarshal(body, &update) 240 | if err != nil { 241 | http.Error(w, "Failed to deserialize payload", http.StatusInternalServerError) 242 | return 243 | } 244 | 245 | data, err := proto.Marshal(s.SetCurrentServers(&update)) 246 | if err != nil { 247 | http.Error(w, "Failed to serialize Protobuf", http.StatusInternalServerError) 248 | return 249 | } 250 | 251 | w.Header().Set("Content-Type", contentType) 252 | w.WriteHeader(http.StatusOK) 253 | 254 | w.Write(data) 255 | return 256 | } 257 | } 258 | } 259 | 260 | injectedBytes, err := json.Marshal(&umbrellaInjectedParameters{ 261 | HttpPrefix: httpPrefix, 262 | }) 263 | 264 | if err != nil { 265 | http.Error(w, "Error preparing template", http.StatusInternalServerError) 266 | return 267 | } 268 | 269 | data := map[string]string{ 270 | "Title": "SFU test page", 271 | "HttpPrefix": httpPrefix, 272 | "Injected": string(injectedBytes), 273 | } 274 | 275 | if err := templates.ExecuteTemplate(w, "generic.html", data); err != nil { 276 | http.Error(w, "Error rendering template", http.StatusInternalServerError) 277 | log.Println("Error executing template:", err) 278 | } 279 | }) 280 | 281 | addHandler("/sfu", generic) 282 | addHandler("/servers", generic) 283 | addHandler("/status", generic) 284 | 285 | wrapped := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 286 | defer func() { 287 | log.Println("Request handling complete for", r.URL.Path) 288 | }() 289 | 290 | log.Println("Req ", r.URL.Path) 291 | 292 | rec := &responseRecorder{ResponseWriter: w, statusCode: http.StatusOK} 293 | mux.ServeHTTP(rec, r) 294 | 295 | if rec.statusCode == http.StatusNotFound { 296 | log.Println("Requesting unfound url", r.URL.Path) 297 | } 298 | }) 299 | 300 | if isCloud { 301 | go func() { 302 | err = http.ListenAndServe(httpServeAddr, wrapped) 303 | if err != nil { 304 | log.Fatal(err) 305 | } 306 | }() 307 | } else { 308 | go func() { 309 | err = http.ListenAndServeTLS(httpServeAddr, "service.crt", "service.key", wrapped) 310 | if err != nil { 311 | log.Fatal(err) 312 | } 313 | }() 314 | 315 | // Use atomirex fork of the pion mdns which works with android clients 316 | go func() { 317 | addr4, err := net.ResolveUDPAddr("udp4", mdns.DefaultAddressIPv4) 318 | if err != nil { 319 | panic(err) 320 | } 321 | 322 | addr6, err := net.ResolveUDPAddr("udp6", mdns.DefaultAddressIPv6) 323 | if err != nil { 324 | panic(err) 325 | } 326 | 327 | l4, err := net.ListenUDP("udp4", addr4) 328 | if err != nil { 329 | panic(err) 330 | } 331 | 332 | l6, err := net.ListenUDP("udp6", addr6) 333 | if err != nil { 334 | panic(err) 335 | } 336 | 337 | hostname, _ := os.Hostname() 338 | hostname = strings.TrimSuffix(strings.ToLower(hostname), ".") 339 | hostname = strings.TrimSuffix(hostname, ".local") 340 | hostname = hostname + ".local" 341 | 342 | log.Println("Broadcasting hostname via mdns", hostname) 343 | 344 | mdnsConn, err := mdns.Server(ipv4.NewPacketConn(l4), ipv6.NewPacketConn(l6), &mdns.Config{ 345 | LocalNames: []mdns.RegisteredHost{{Name: hostname}}, 346 | }) 347 | if err != nil { 348 | panic(err) 349 | } 350 | 351 | s.SetMdnsConn(mdnsConn) 352 | select {} 353 | }() 354 | } 355 | 356 | go func() { 357 | for { 358 | <-time.After(5 * time.Second) 359 | runtime.GC() 360 | } 361 | }() 362 | 363 | razor.WaitForOsInterruptSignal() 364 | 365 | } 366 | -------------------------------------------------------------------------------- /sfu/sfu.go: -------------------------------------------------------------------------------- 1 | package sfu 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "time" 8 | 9 | "atomirex.com/umbrella/razor" 10 | "github.com/atomirex/mdns" 11 | "github.com/gorilla/websocket" 12 | "github.com/pion/interceptor" 13 | "github.com/pion/logging" 14 | "github.com/pion/webrtc/v4" 15 | ) 16 | 17 | var ( 18 | upgrader = websocket.Upgrader{ 19 | CheckOrigin: func(r *http.Request) bool { return true }, 20 | } 21 | ) 22 | 23 | type sfuCommand int 24 | 25 | const ( 26 | sfuAddOutgoingTracksForIncomingTrack sfuCommand = iota 27 | sfuRemoveAllOutgoingTracksForIncomingTrack 28 | sfuSignalClients 29 | sfuAddClient 30 | sfuRemoveClient 31 | 32 | sfuGetCurrentServers 33 | sfuSetCurrentServers 34 | 35 | sfuGetStatus 36 | ) 37 | 38 | type sfuCommandMessage struct { 39 | intrack *incomingTrack 40 | client *client 41 | SetCurrentServers *CurrentServers 42 | 43 | result *sfuCommandResult 44 | } 45 | 46 | type sfuCommandResult struct { 47 | servers chan *CurrentServers 48 | status chan *SFUStatus 49 | } 50 | 51 | type Sfu struct { 52 | clients []RemoteClient 53 | sfuCommands chan sfuCommandMessage 54 | 55 | peerConnectionFactory PeerConnectionFactory 56 | remoteClientFactory RemoteClientFactory 57 | 58 | // UmbrellaID -> track 59 | localTracks map[string]*incomingTrack // Set of all incoming tracks which are being relayed 60 | 61 | handler *razor.MessageHandler[sfuCommand, sfuCommandMessage] 62 | 63 | intendedServers map[string]bool // This works because strings completely define the spec of the server right now 64 | servers map[string]RemoteClient 65 | 66 | logger *razor.Logger 67 | 68 | loggerPion logging.LeveledLogger 69 | 70 | mdnsConn *mdns.Conn 71 | } 72 | 73 | func (s *Sfu) GetStatus() *SFUStatus { 74 | msg := sfuCommandMessage{result: &sfuCommandResult{ 75 | status: make(chan *SFUStatus, 1), 76 | }} 77 | 78 | s.handler.Send(sfuGetStatus, &msg) 79 | 80 | return <-msg.result.status 81 | } 82 | 83 | func (s *Sfu) GetCurrentServers() *CurrentServers { 84 | msg := sfuCommandMessage{result: &sfuCommandResult{ 85 | servers: make(chan *CurrentServers, 1), 86 | }} 87 | 88 | s.handler.Send(sfuGetCurrentServers, &msg) 89 | 90 | return <-msg.result.servers 91 | } 92 | 93 | func (s *Sfu) SetCurrentServers(update *CurrentServers) *CurrentServers { 94 | msg := sfuCommandMessage{ 95 | SetCurrentServers: update, 96 | result: &sfuCommandResult{ 97 | servers: make(chan *CurrentServers, 1), 98 | }, 99 | } 100 | 101 | s.handler.Send(sfuSetCurrentServers, &msg) 102 | 103 | return <-msg.result.servers 104 | } 105 | 106 | func NewSfu(logger *razor.Logger, minPort uint16, maxPort uint16, ip *string) *Sfu { 107 | loggerPion := logging.NewDefaultLoggerFactory().NewLogger("sfu-ws") 108 | loggerPion.(*logging.DefaultLeveledLogger).SetLevel(logging.LogLevelError) 109 | 110 | settingEngine := webrtc.SettingEngine{} 111 | 112 | settingEngine.SetEphemeralUDPPortRange(minPort, maxPort) 113 | 114 | if ip != nil { 115 | settingEngine.SetNAT1To1IPs([]string{*ip}, webrtc.ICECandidateTypeHost) 116 | } 117 | 118 | m := &webrtc.MediaEngine{} 119 | if err := m.RegisterDefaultCodecs(); err != nil { 120 | panic("Error setting default codecs") 121 | } 122 | 123 | // Now we know what this is and why . . . . facepalm 124 | // This is the "default" nack, sr, rr etc. handling for rtcp 125 | // We can come back to it when it's a problem 126 | interceptorRegistry := &interceptor.Registry{} 127 | if err := webrtc.RegisterDefaultInterceptors(m, interceptorRegistry); err != nil { 128 | panic("Panic setting interceptors") 129 | } 130 | 131 | webrtcApi := webrtc.NewAPI(webrtc.WithSettingEngine(settingEngine), webrtc.WithMediaEngine(m)) 132 | 133 | s := &Sfu{ 134 | sfuCommands: make(chan sfuCommandMessage, 256), 135 | remoteClientFactory: &DefaultRemoteClientFactory{}, 136 | peerConnectionFactory: &PionPeerConnectionFactory{ 137 | pcConfig: &webrtc.Configuration{ 138 | ICEServers: []webrtc.ICEServer{ 139 | { 140 | URLs: []string{"stun:stun.l.google.com:19302"}, 141 | }, 142 | }, 143 | }, 144 | webrtcApi: webrtcApi, 145 | logger: logger, 146 | }, 147 | localTracks: make(map[string]*incomingTrack), 148 | intendedServers: make(map[string]bool), 149 | servers: make(map[string]RemoteClient), 150 | logger: logger, 151 | loggerPion: loggerPion, 152 | } 153 | 154 | s.handler = razor.NewMessageHandler(logger, "sfu", 1024, func(what sfuCommand, payload *sfuCommandMessage) bool { 155 | shouldSignalClients := false 156 | 157 | switch what { 158 | case sfuAddClient: 159 | s.clients = append(s.clients, payload.client) 160 | 161 | // Add all existing tracks 162 | for _, t := range s.localTracks { 163 | payload.client.handler.Send(clientAddOutgoingTrackForIncomingTrack, &clientCommandMessage{incomingTrack: t}) 164 | } 165 | 166 | shouldSignalClients = true 167 | case sfuRemoveClient: 168 | index := -1 169 | for i, c := range s.clients { 170 | if c == payload.client { 171 | index = i 172 | } 173 | } 174 | 175 | if index >= 0 { 176 | s.clients = append(s.clients[:index], s.clients[index+1:]...) 177 | } 178 | 179 | shouldSignalClients = true 180 | case sfuAddOutgoingTracksForIncomingTrack: 181 | logger.Info("sfu", "adding track: "+payload.intrack.String()) 182 | intrack := payload.intrack 183 | s.localTracks[intrack.UmbrellaID()] = intrack 184 | 185 | for _, c := range s.clients { 186 | c.AddOutgoingTracksForIncomingTrack(intrack) 187 | } 188 | 189 | shouldSignalClients = true 190 | case sfuRemoveAllOutgoingTracksForIncomingTrack: 191 | logger.Info("sfu", "removing all outgoing tracks for track: "+payload.intrack.String()) 192 | delete(s.localTracks, payload.intrack.UmbrellaID()) 193 | 194 | for _, c := range s.clients { 195 | c.RemoveOutgoingTracksForIncomingTrack(payload.intrack) 196 | } 197 | 198 | shouldSignalClients = true 199 | case sfuSignalClients: 200 | shouldSignalClients = true 201 | case sfuGetStatus: 202 | // Terrible version of this which blocks while polling everything 203 | // Should really pass around a thing that gathers the info 204 | // but in reality I should move to a pub/sub thing that pushes the data out on changes anyway 205 | // this is to debug the mess as is :P 206 | logger.Info("sfu", "SFU getting status servers") 207 | servers := make([]string, 0) 208 | for t := range s.servers { 209 | servers = append(servers, t) 210 | } 211 | logger.Info("sfu", "SFU getting status relaying") 212 | relaying := make([]*TrackDescriptor, 0) 213 | for _, t := range s.localTracks { 214 | relaying = append(relaying, t.descriptor) 215 | } 216 | logger.Info("sfu", "SFU getting status clients") 217 | clients := make([]*SFUStatusClient, 0) 218 | for _, c := range s.clients { 219 | logger.Info("sfu", "SFU getting status for client "+c.Label()) 220 | clients = append(clients, c.getStatus()) 221 | logger.Info("sfu", "SFU received status for client "+c.Label()) 222 | } 223 | 224 | logger.Info("sfu", "SFU getting status returning") 225 | status := &SFUStatus{ 226 | RelayingTracks: relaying, 227 | Servers: servers, 228 | Clients: clients, 229 | } 230 | 231 | payload.result.status <- status 232 | case sfuGetCurrentServers: 233 | result := &CurrentServers{Servers: make([]string, 0)} 234 | 235 | for t := range s.servers { 236 | result.Servers = append(result.Servers, t) 237 | } 238 | 239 | payload.result.servers <- result 240 | case sfuSetCurrentServers: 241 | // "intended" servers model, then regularly evaluate, so setup/teardown repeatedly is ok 242 | mentioned := make(map[string]bool) 243 | for _, t := range payload.SetCurrentServers.Servers { 244 | mentioned[t] = true 245 | } 246 | 247 | // Just overwrite it with what the client sent! (For now) 248 | s.intendedServers = mentioned 249 | 250 | s.evaluateServers() 251 | 252 | shouldSignalClients = true 253 | 254 | result := &CurrentServers{Servers: make([]string, 0)} 255 | 256 | for t := range s.servers { 257 | result.Servers = append(result.Servers, t) 258 | } 259 | 260 | payload.result.servers <- result 261 | } 262 | 263 | if shouldSignalClients { 264 | logger.Verbose("sfu", "SFU should signaling clients") 265 | s.handler.Cancel(sfuSignalClients) 266 | 267 | for _, c := range s.clients { 268 | logger.Verbose("sfu", "SFU signaling client "+c.Label()) 269 | c.RequestEvalState() 270 | } 271 | 272 | logger.Verbose("sfu", "SFU finished signaling clients") 273 | } 274 | 275 | return true 276 | }) 277 | 278 | s.handler.Loop(func() { 279 | panic("SFU unexpectedly terminated") 280 | }) 281 | 282 | return s 283 | } 284 | 285 | func (s *Sfu) evaluateServers() { 286 | // Ensure any running servers that should be stopped are stopping or stopped 287 | for t := range s.servers { 288 | mention := s.intendedServers[t] 289 | if !mention { 290 | s.servers[t].stop() 291 | 292 | // TODO only actually delete the server when it's really stopped (need the get servers endpoint to be pub/sub based first or it'll be useless) 293 | delete(s.servers, t) 294 | } 295 | } 296 | 297 | // Create and start any servers that should exist but do not (i.e. if they are still stopping leave them until stopped and removed) 298 | for t, _ := range s.intendedServers { 299 | _, exists := s.servers[t] 300 | if !exists { 301 | server := s.remoteClientFactory.NewClient(&RemoteClientParameters{ 302 | logger: s.logger, 303 | trunkurl: t, 304 | s: s, 305 | }) 306 | s.servers[t] = server 307 | } 308 | } 309 | } 310 | 311 | func (s *Sfu) addTrack(intrack *incomingTrack) { 312 | s.handler.Send(sfuAddOutgoingTracksForIncomingTrack, &sfuCommandMessage{intrack: intrack}) 313 | } 314 | 315 | func (s *Sfu) removeOutgoingTracksForIncomingTrack(t *incomingTrack) { 316 | s.handler.Send(sfuRemoveAllOutgoingTracksForIncomingTrack, &sfuCommandMessage{intrack: t}) 317 | } 318 | 319 | func (s *Sfu) WebsocketHandler(w http.ResponseWriter, r *http.Request) { 320 | ws, err := upgrader.Upgrade(w, r, nil) 321 | if err != nil { 322 | s.logger.Error("sfu", "Failed to upgrade HTTP to Websocket: "+err.Error()) 323 | return 324 | } 325 | 326 | s.remoteClientFactory.NewClient(&RemoteClientParameters{logger: s.logger, ws: ws, s: s}) 327 | } 328 | 329 | func (s *Sfu) SetMdnsConn(mdnsConn *mdns.Conn) { 330 | s.mdnsConn = mdnsConn 331 | } 332 | 333 | // Use our own mdns for resolving .local addresses to work around compatibility problems 334 | // If we are in cloud mode this will return an error 335 | // If the host is not found this will return an error 336 | // If the host is found the string returned will contain the IP as a string . . . 337 | func (s *Sfu) mdnsLookup(hostname string) (string, error) { 338 | if s.mdnsConn == nil { 339 | return "", fmt.Errorf("attempting to resolve local subnet host while mdns not in use") 340 | } 341 | 342 | ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) 343 | _, addr, err := s.mdnsConn.QueryAddr(ctx, hostname) 344 | if err != nil { 345 | return "", err 346 | } else { 347 | return addr.String(), nil 348 | } 349 | } 350 | -------------------------------------------------------------------------------- /docs/sfu-diagram-2.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 158 | -------------------------------------------------------------------------------- /docs/sfu-diagram-1.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 219 | -------------------------------------------------------------------------------- /frontend/src/umbrella/SfuApp.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect } from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import { useRef, useState } from 'react'; 4 | import { SetUpstreamTracks, RemoteNodeMessage, TrackDescriptor, TrackKind, CurrentServers, MidToUmbrellaIDMapping, SFUStatus, SFUStatusClient, SFUStatusPeerConnection } from '../generated/sfu' 5 | 6 | function trackKindFromString(k: string) : TrackKind { 7 | switch(k) { 8 | case "audio": 9 | return TrackKind.Audio; 10 | case "video": 11 | return TrackKind.Video; 12 | default: 13 | return TrackKind.Unknown; 14 | } 15 | } 16 | 17 | function trackKindToString(k: TrackKind) : string { 18 | switch(k) { 19 | case TrackKind.Audio: 20 | return "audio"; 21 | case TrackKind.Video: 22 | return "video"; 23 | default: 24 | return "unknown"; 25 | } 26 | } 27 | 28 | abstract class track { 29 | private mediaStream: MediaStream; 30 | private mediaStreamTrack: MediaStreamTrack; 31 | umbrellaId: string; 32 | 33 | constructor(mediaStream: MediaStream, mediaStreamTrack: MediaStreamTrack, umbrellaId: string) { 34 | this.mediaStream = mediaStream; 35 | this.mediaStreamTrack = mediaStreamTrack; 36 | this.umbrellaId = umbrellaId; 37 | } 38 | 39 | kind() : TrackKind { 40 | return trackKindFromString(this.mediaStreamTrack.kind); 41 | } 42 | 43 | getDescriptor() : TrackDescriptor { 44 | return ({ 45 | id: this.mediaStreamTrack.id, 46 | kind: trackKindFromString(this.mediaStreamTrack.kind), 47 | streamId: this.mediaStream.id, 48 | umbrellaId: this.umbrellaId, 49 | }); 50 | } 51 | 52 | getTrack() : MediaStreamTrack { 53 | return this.mediaStreamTrack; 54 | } 55 | 56 | getStream() : MediaStream { 57 | return this.mediaStream; 58 | } 59 | } 60 | 61 | // This is the data associated with an incoming track before we can conclusively move to it being a remote track 62 | class stagedIncomingTrack { 63 | mediaStream: MediaStream; 64 | mediaStreamTrack: MediaStreamTrack; 65 | completed: boolean = false; 66 | 67 | constructor(mediaStream: MediaStream, mediaStreamTrack: MediaStreamTrack) { 68 | this.mediaStream = mediaStream; 69 | this.mediaStreamTrack = mediaStreamTrack; 70 | } 71 | } 72 | 73 | class remoteTrack extends track { 74 | } 75 | 76 | class localTrack extends track { 77 | published: boolean; 78 | private transceiver: RTCRtpTransceiver | null = null; 79 | 80 | constructor(mediaStream: MediaStream, mediaStreamTrack: MediaStreamTrack) { 81 | super(mediaStream, mediaStreamTrack, "UMB_TRACK_"+crypto.randomUUID()); 82 | this.published = false; 83 | } 84 | 85 | setTransceiver(transceiver : RTCRtpTransceiver | null) { 86 | this.transceiver = transceiver; 87 | } 88 | 89 | getTransceiver() : RTCRtpTransceiver | null { 90 | return this.transceiver; 91 | } 92 | } 93 | 94 | interface LocalVideoProps { 95 | stream: MediaStream | null; 96 | } 97 | 98 | const LocalVideo: React.FC = ({ stream }) => { 99 | const videoRef = useRef(null); 100 | 101 | useEffect(() => { 102 | if (videoRef.current && stream) { 103 | videoRef.current.srcObject = stream; 104 | 105 | const vt = stream.getVideoTracks()[0]; 106 | if(vt) { 107 | const facingMode = vt.getSettings().facingMode; 108 | if(facingMode == undefined || facingMode == "user") { 109 | videoRef.current.style.transform = "scaleX(-1)"; 110 | } else { 111 | videoRef.current.style.transform = "scaleX(1)"; 112 | } 113 | } 114 | } 115 | }, [stream]); 116 | 117 | return

; 118 | }; 119 | 120 | interface RemoteVideosProps { 121 | tracks: remoteTrack[]; 122 | } 123 | 124 | const RemoteVideos: React.FC = ({ tracks }) => { 125 | return ( 126 | <> 127 | {tracks.map((track, index) => ( 128 | 129 | ))} 130 | 131 | ); 132 | }; 133 | 134 | const RemoteVideo: React.FC<{ track: remoteTrack }> = ({ track }) => { 135 | const videoRef = useRef(null); 136 | 137 | useEffect(() => { 138 | if (videoRef.current) { 139 | videoRef.current.srcObject = track.getStream(); 140 | } 141 | }, [track]); 142 | 143 | return

; 144 | }; 145 | 146 | interface SfuAppJoinedProps { 147 | requestLocalMediaFirst: boolean; 148 | } 149 | 150 | const SfuAppJoined: React.FC = ({requestLocalMediaFirst}) => { 151 | const [localStream, setLocalStream] = useState(null); 152 | const [remoteTracks, setRemoteTracks] = useState([]); 153 | 154 | const websocketRef = useRef(null); 155 | const offerNeededTimerRef = useRef(-1); 156 | 157 | useEffect(() => { 158 | const pageUrl = window.location.href; 159 | const wsUrl = "wss" + pageUrl.substring(pageUrl.indexOf(":"), pageUrl.lastIndexOf("/")) + "/wsb"; 160 | console.log("Websocket url set to: " + wsUrl); 161 | 162 | function log(msg: string) { 163 | console.log("SFULOG: "+msg); 164 | } 165 | 166 | // Always have to ask permission for the streams for webrtc connections to actually work 167 | // We can safely ignore the streams afterwards 168 | // 169 | // The view only one we set because some devices without webcams still benefit from being 170 | // able to prompt the user (like firefox) due to implementing RFC 8828 with any 171 | // successful getUserMedia being interpreted as consent for ICE candidate discovery. 172 | // Without that you will get cryptic errors. 173 | // 174 | // You will also get Linux Firefox ICE errors if someone is sending h264 and you don't 175 | // have the OpenH264 plugin installed and enabled. 176 | // https://support.mozilla.org/en-US/kb/open-h264-plugin-firefox 177 | // 178 | // For now we cap at 720P 179 | navigator.mediaDevices.getUserMedia({ video: requestLocalMediaFirst ? { width:{max:1280}, height: {max: 720}} : false, audio: true }) 180 | .then(streamInit => { 181 | const stream = requestLocalMediaFirst ? streamInit : null; 182 | 183 | const pcConfig: RTCConfiguration = { 184 | iceServers: [ 185 | {urls: ["stun:stun.l.google.com:19302","stun:stun2.1.google.com:19302"]} 186 | ] 187 | }; 188 | 189 | const incoming = new RTCPeerConnection(pcConfig); 190 | const outgoing = new RTCPeerConnection(pcConfig); 191 | 192 | // Gives the offers something to be about 193 | const dataOut = outgoing.createDataChannel("data-out", {ordered: false}); 194 | const dataIn = incoming.createDataChannel("data-in", {ordered: false}); 195 | 196 | const midToUmbrellaIDMapping = new Map(); 197 | let stagedIncomingTracks : stagedIncomingTrack[] = []; 198 | 199 | // Should be called whenever the mapping changes or we receive a new track in incoming.ontrack 200 | // Then if we can fuse the data together the new remotetrack is created 201 | function evaluateIncomingTracks() { 202 | const newRemoteTracks : remoteTrack[] = []; 203 | 204 | stagedIncomingTracks.forEach(sit => { 205 | const transceiversForIncomingTrack = incoming.getTransceivers().filter((trx) => trx.receiver.track == sit.mediaStreamTrack); 206 | if(transceiversForIncomingTrack.length > 0) { 207 | console.log("FOUND THESE TRANSCEIVERS: "+JSON.stringify(transceiversForIncomingTrack)+" mid first "+transceiversForIncomingTrack[0].mid); 208 | const mid = transceiversForIncomingTrack[0].mid; 209 | if(mid != null) { 210 | const umbrellaId = midToUmbrellaIDMapping.get(mid); 211 | console.log("Umbrella ID with transceiver MID is "+umbrellaId); 212 | if(umbrellaId) { 213 | newRemoteTracks.push(new remoteTrack(sit.mediaStream, sit.mediaStreamTrack, umbrellaId)); 214 | sit.completed = true; 215 | 216 | sit.mediaStreamTrack.onmute = function (event) { 217 | // Hmm 218 | }; 219 | 220 | sit.mediaStream.onremovetrack = ({ track }) => { 221 | console.log("Incoming video track removed: "+umbrellaId); 222 | setRemoteTracks((prev) => prev.filter(t => t.umbrellaId !== umbrellaId)); 223 | }; 224 | } 225 | } 226 | } 227 | }); 228 | 229 | stagedIncomingTracks = stagedIncomingTracks.filter(sit => !sit.completed); 230 | 231 | setRemoteTracks((prev) => [...prev, ...newRemoteTracks]); 232 | } 233 | 234 | incoming.ontrack = (event) => { 235 | if (event.track.kind === 'audio') { 236 | console.log("ONTRACK Don't need to worry about audio"); 237 | return; 238 | } 239 | 240 | if(event.streams.length == 0 || event.streams[0] == null) { 241 | console.log("ONTRACK provided empty streams"); 242 | return; 243 | } 244 | 245 | if (stream != null && stream.id == event.streams[0].id) { 246 | console.log("ONTRACK Rejecting due to loopback suspicions "+stream.id+" "+event.streams); 247 | // Loopback detected - this doesn't seem right 248 | return; 249 | } 250 | 251 | if(stream != null) { 252 | const localVideoTrack = stream.getVideoTracks()[0]; 253 | console.log("Local video track id "+localVideoTrack.id+" local stream id "+stream.id+" incoming stream id "+event.streams[0].id+" incoming track id "+event.track.id); 254 | } 255 | 256 | const incomingStream = event.streams[0]; 257 | const incomingTrack = incomingStream.getVideoTracks()[0]; 258 | 259 | stagedIncomingTracks.push(new stagedIncomingTrack(incomingStream, incomingTrack)); 260 | 261 | evaluateIncomingTracks(); 262 | }; 263 | 264 | incoming.onconnectionstatechange = (() => { 265 | switch(incoming.connectionState) { 266 | case "disconnected": 267 | case "failed": 268 | case "closed": 269 | console.log("Incoming connection terminated, state: "+incoming.connectionState); 270 | setRemoteTracks([]); 271 | } 272 | }); 273 | 274 | let ws = new WebSocket(wsUrl); 275 | websocketRef.current = ws; 276 | 277 | ws.binaryType = "arraybuffer"; 278 | 279 | if(stream != null) { 280 | setLocalStream(stream); 281 | } 282 | 283 | const localTracks = new Map(); 284 | if(stream != null) { 285 | stream.getTracks().forEach(track => { 286 | localTracks.set(track.id, new localTrack(stream, track)); 287 | }); 288 | } 289 | 290 | ws.onopen = e => { 291 | ws.send(RemoteNodeMessage.toBinary({ 292 | upstreamTracks: { 293 | tracks: Array.from(localTracks.values()).map(track => track.getDescriptor()), 294 | }, 295 | })); 296 | 297 | offerNeeded(); 298 | }; 299 | 300 | incoming.onicecandidate = e => { 301 | if (!e.candidate) { 302 | return; 303 | } 304 | 305 | ws.send(RemoteNodeMessage.toBinary({ 306 | candidate: { 307 | candidate: JSON.stringify(e.candidate), 308 | incoming: true, 309 | }, 310 | })); 311 | }; 312 | 313 | outgoing.onicecandidate = e => { 314 | if (!e.candidate) { 315 | return; 316 | } 317 | 318 | ws.send(RemoteNodeMessage.toBinary({ 319 | candidate: { 320 | candidate: JSON.stringify(e.candidate), 321 | incoming: false, 322 | }, 323 | })); 324 | }; 325 | 326 | ws.onclose = function (evt) { 327 | log("ws.onclose"); 328 | 329 | websocketRef.current = null; 330 | 331 | window.alert("Websocket has closed") 332 | } 333 | 334 | function offerNeeded() { 335 | if(offerNeededTimerRef.current >= 0) { 336 | clearTimeout(offerNeededTimerRef.current); 337 | } 338 | 339 | offerNeededTimerRef.current = setTimeout(async () => { 340 | const offer = await outgoing.createOffer(); 341 | await outgoing.setLocalDescription(offer); 342 | 343 | if(ws != null) { 344 | ws.send(RemoteNodeMessage.toBinary({ 345 | offer: { 346 | offer: JSON.stringify(offer), 347 | }, 348 | })); 349 | } 350 | }, 100); 351 | } 352 | 353 | outgoing.onsignalingstatechange = () => { 354 | log("OUTGOING SIGNAL STATE CHANGE "+outgoing.signalingState); 355 | 356 | if(outgoing.signalingState === "stable") { 357 | // Review transceivers for local tracks and notify remote server of mappings 358 | const mappings : MidToUmbrellaIDMapping[] = []; 359 | 360 | const trx = outgoing.getTransceivers(); 361 | localTracks.forEach((lt, id) => { 362 | const mst = lt.getTrack(); 363 | 364 | trx.forEach((transceiver) => { 365 | if(transceiver.sender.track === mst && transceiver.mid != null) { 366 | console.log("FOUND TRANSCEIVER FOR TRACK! mid: "+transceiver.mid); 367 | 368 | mappings.push({ 369 | mid: transceiver.mid, 370 | umbrellaId: lt.umbrellaId, 371 | }) 372 | } 373 | }); 374 | }); 375 | 376 | ws.send(RemoteNodeMessage.toBinary({midMappings: {mapping: mappings}})); 377 | } 378 | }; 379 | 380 | ws.onmessage = async function (event) { 381 | if (!event.data) { 382 | return; 383 | } 384 | 385 | let msg = RemoteNodeMessage.fromBinary(new Uint8Array(event.data), { readUnknownField: true }); 386 | 387 | log("ws.onmessage: " + JSON.stringify(msg)); 388 | 389 | if (msg.offer) { 390 | let offer = JSON.parse(msg.offer.offer); 391 | 392 | incoming.setRemoteDescription(offer); 393 | const answer = await incoming.createAnswer(); 394 | await incoming.setLocalDescription(answer); 395 | 396 | ws.send(RemoteNodeMessage.toBinary({ 397 | answer: { 398 | answer: JSON.stringify(answer), 399 | }, 400 | })); 401 | } 402 | 403 | if(msg.answer) { 404 | // Outgoing stuff 405 | await outgoing.setRemoteDescription(JSON.parse(msg.answer.answer)); 406 | } 407 | 408 | if (msg.candidate) { 409 | let candidate = JSON.parse(msg.candidate.candidate); 410 | // Incoming from pov of the sender! 411 | (msg.candidate.incoming ? outgoing : incoming).addIceCandidate(candidate); 412 | } 413 | 414 | if (msg.upstreamTracks) { 415 | log("Upstream tracks recevied "+JSON.stringify(msg.upstreamTracks)); 416 | 417 | // Just echoing it for now, unlike pion we don't need to get ready 418 | ws.send(RemoteNodeMessage.toBinary({acceptTracks: {tracks: msg.upstreamTracks.tracks}})); 419 | } 420 | 421 | if (msg.midMappings) { 422 | log("MID <-> Umbrella mappings received "+JSON.stringify(msg.midMappings.mapping)); 423 | 424 | msg.midMappings.mapping.forEach(m => { 425 | midToUmbrellaIDMapping.set(m.mid, m.umbrellaId); 426 | }); 427 | 428 | evaluateIncomingTracks(); 429 | } 430 | 431 | if (msg.acceptTracks) { 432 | let changed = false; 433 | 434 | msg.acceptTracks.tracks.forEach(descriptor => { 435 | const t = localTracks.get(descriptor.id); 436 | 437 | if(t) { 438 | log("Confirmed track "+JSON.stringify(t.getDescriptor())); 439 | if(!t.published) { 440 | log("Publishing track "+t.getTrack().id); 441 | outgoing.addTrack(t.getTrack(), t.getStream()); 442 | 443 | t.published = true; 444 | changed = true; 445 | } 446 | } else { 447 | log("Remote track incoming: "+JSON.stringify(descriptor)); 448 | } 449 | }); 450 | 451 | if(changed) { 452 | offerNeeded(); 453 | } 454 | } 455 | } 456 | 457 | ws.onerror = function (evt) { 458 | log("ws.onerror"); 459 | } 460 | }).catch(console.log) 461 | 462 | return () => { 463 | if(offerNeededTimerRef.current >= 0) { 464 | clearTimeout(offerNeededTimerRef.current); 465 | offerNeededTimerRef.current = 0; 466 | } 467 | }; 468 | }, []); 469 | 470 | const sendWsMessage = (data: Uint8Array) => { 471 | if(websocketRef.current !== null) { 472 | websocketRef.current.send(data); 473 | } 474 | }; 475 | 476 | return ( 477 |

478 |

479 | { requestLocalMediaFirst && } 480 | 481 |

482 |

483 | ); 484 | }; 485 | 486 | export const SfuApp = () => { 487 | const [joined, setJoined] = useState(false); 488 | 489 | const requestLocalMediaFirstRef = useRef(false); 490 | 491 | const joinWithLocalMedia = () => { 492 | requestLocalMediaFirstRef.current = true; 493 | setJoined(true); 494 | }; 495 | 496 | const joinAsViewer = () => { 497 | requestLocalMediaFirstRef.current = false; 498 | setJoined(true); 499 | }; 500 | 501 | return ( 502 | <> 503 | { joined ? ( 504 | 505 | ) : ( 506 |

507 |
508 | 509 |

510 | ) } 511 | 512 | ) 513 | }; 514 | 515 | export const ServersApp = () => { 516 | const [servers, setServers] = useState([]); 517 | const addServerInputRef = useRef(null); 518 | 519 | useEffect(() => { 520 | fetch(window.location.pathname, {method: 'GET', headers:{'Content-Type': "application/x-protobuf"}}) 521 | .then((response) => response.arrayBuffer()) 522 | .then((buffer) => { 523 | setServers(CurrentServers.fromBinary(new Uint8Array(buffer)).servers); 524 | addServerInputRef.current?.focus(); 525 | }).catch(console.log); 526 | 527 | return () => {}; 528 | }, []); 529 | 530 | const addServerClick = () => { 531 | const updateVal = addServerInputRef.current!.value!; 532 | const serversUpdate = updateVal === "" ? [] : [updateVal]; 533 | 534 | fetch(window.location.pathname, 535 | { 536 | method: 'POST', 537 | headers:{'Content-Type': "application/x-protobuf"}, 538 | body: CurrentServers.toBinary({servers:serversUpdate}) 539 | } 540 | ).then((response) => response.arrayBuffer()) 541 | .then((buffer) => { 542 | setServers(CurrentServers.fromBinary(new Uint8Array(buffer)).servers); 543 | addServerInputRef.current?.focus(); 544 | }).catch(console.log); 545 | 546 | addServerInputRef.current!.value = ""; 547 | }; 548 | 549 | const removeServerClick = (server: string) => { 550 | fetch(window.location.pathname, 551 | { 552 | method: 'POST', 553 | headers:{'Content-Type': "application/x-protobuf"}, 554 | body: CurrentServers.toBinary({servers: servers.filter((s) => s !== server)}) 555 | } 556 | ).then((response) => response.arrayBuffer()) 557 | .then((buffer) => { 558 | setServers(CurrentServers.fromBinary(new Uint8Array(buffer)).servers); 559 | }).catch(console.log); 560 | }; 561 | 562 | return ( 563 | <> 564 |

565 |

Servers

566 |

567 |

No servers
{ server }

574 |

575 | 576 |

577 | 578 | ) 579 | }; 580 | 581 | const TrackDescriptorStatusListElement: React.FC<{ descriptor: TrackDescriptor }> = ({descriptor}) => { 582 | return ( 583 |

{ descriptor.umbrellaId }

Kind { trackKindToString(descriptor.kind) }
Track ID { descriptor.id }
Stream ID { descriptor.streamId }

588 | ); 589 | }; 590 | 591 | const PeerConnectionStatusListElement: React.FC<{ label: string, pc: SFUStatusPeerConnection | undefined }> = ({label, pc}) => { 592 | return ( 593 |

{ label }

596 | ); 597 | }; 598 | 599 | const ClientStatusListElement: React.FC<{ client: SFUStatusClient }> = ({client}) => { 600 | return ( 601 |

{ client.label }

Trunk url: { client.trunkUrl }

604 | 605 |

Incoming tracks

)} 607 |

608 |

Outgoing tracks

)} 610 |

611 |

Senders

{s.umbrellaId} { s.hasTrack ? ("Has a track of ID " + s.trackIdIfSet) : "Has no track ID"}

614 |

MID to Umbrella ID mappings

{m.mid} : {m.umbrellaId}

617 |

Staged incoming tracks

{sit.streamId} {sit.trackId} {sit.mid}

620 |

621 | ); 622 | }; 623 | 624 | export const StatusApp = () => { 625 | const [status, setStatus] = useState(null); 626 | 627 | useEffect(() => { 628 | fetch(window.location.pathname, {method: 'GET', headers:{'Content-Type': "application/x-protobuf"}}) 629 | .then((response) => response.arrayBuffer()) 630 | .then((buffer) => { 631 | setStatus(SFUStatus.fromBinary(new Uint8Array(buffer))); 632 | }).catch(console.log); 633 | 634 | return () => {}; 635 | }, []); 636 | 637 | return ( 638 | <> 639 |

640 |

Status

641 | { (status == null) ? ( 642 |

Status is null

643 | ) : ( 644 | <> 645 |

Relaying tracks

646 |

649 | ))} 650 |

651 |

Clients

652 | {status.clients.map(c => ( 653 | 654 | ))} 655 |

servers

656 |

{ t }

661 | 662 | )} 663 |

664 | 665 | ) 666 | }; -------------------------------------------------------------------------------- /sfu/client_webrtc.go: -------------------------------------------------------------------------------- 1 | package sfu 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "encoding/json" 7 | "fmt" 8 | "net" 9 | "strings" 10 | "time" 11 | 12 | "atomirex.com/umbrella/razor" 13 | "github.com/google/uuid" 14 | "github.com/gorilla/websocket" 15 | "github.com/pion/rtcp" 16 | "github.com/pion/rtp" 17 | "github.com/pion/webrtc/v4" 18 | "google.golang.org/protobuf/proto" 19 | ) 20 | 21 | type clientCommand int 22 | 23 | const ( 24 | clientSendProto clientCommand = iota 25 | clientHandleWSMessage 26 | clientStop 27 | clientAddOutgoingTrackForIncomingTrack 28 | clientRemoveOutgoingTracksForIncomingTrack 29 | clientKeyframeTick 30 | clientKeyframeGateUnlock 31 | clientEvalState 32 | clientDialWs 33 | clientIncomingTrackAdded 34 | clientGetStatus 35 | ) 36 | 37 | type rawIncomingTrack struct { 38 | track *webrtc.TrackRemote 39 | receiver *webrtc.RTPReceiver 40 | } 41 | 42 | type clientCommandMessage struct { 43 | message *RemoteNodeMessage 44 | incomingTrack *incomingTrack 45 | newincomingTrack *rawIncomingTrack 46 | result *clientCommandResult 47 | } 48 | 49 | type clientCommandResult struct { 50 | status chan *SFUStatusClient 51 | } 52 | 53 | type client struct { 54 | BaseClient 55 | 56 | trunkurl string // Empty means this is a normal web client - should replace with proper roles or similar 57 | incoming *PeerConnection 58 | outgoing *PeerConnection 59 | websocket *websocket.Conn 60 | 61 | // The umbrellaId -> incomingTrack 62 | incomingTracks map[string]*incomingTrackWithClientState 63 | 64 | // The umbrellaId -> outgoingTrack 65 | outgoingTracks map[string]*outgoingTrackWithClientState 66 | 67 | // The umbrellaId -> RTPSender 68 | senders map[string]*webrtc.RTPSender 69 | 70 | handler *razor.MessageHandler[clientCommand, clientCommandMessage] 71 | 72 | // mid <-> umbrella track ID mappings - only set when known to be valid! 73 | incomingMidToUmbrellaTrackID map[string]string 74 | 75 | // Incoming tracks that have yet to be attached to MIDs 76 | stagedIncomingTracks []*rawIncomingTrack 77 | } 78 | 79 | func (c *client) getStatus() *SFUStatusClient { 80 | msg := clientCommandMessage{result: &clientCommandResult{ 81 | status: make(chan *SFUStatusClient, 1), 82 | }} 83 | 84 | c.handler.Send(clientGetStatus, &msg) 85 | 86 | return <-msg.result.status 87 | } 88 | 89 | func (c *client) run(ws *websocket.Conn, s *Sfu) { 90 | c.incomingTracks = make(map[string]*incomingTrackWithClientState) 91 | c.outgoingTracks = make(map[string]*outgoingTrackWithClientState) 92 | 93 | c.incomingMidToUmbrellaTrackID = make(map[string]string) 94 | 95 | c.stagedIncomingTracks = make([]*rawIncomingTrack, 0) 96 | 97 | c.senders = make(map[string]*webrtc.RTPSender) 98 | 99 | incoming, err := s.peerConnectionFactory.NewPeerConnection(fmt.Sprintf("incoming for %s", c.label)) 100 | if c.logger.NilErrCheck(c.label, "Failed to create an incoming peer connection", err) { 101 | return 102 | } 103 | 104 | outgoing, err := s.peerConnectionFactory.NewPeerConnection(fmt.Sprintf("outgoing for %s", c.label)) 105 | if c.logger.NilErrCheck(c.label, "Failed to create an outgoing peer connection", err) { 106 | return 107 | } 108 | 109 | c.incoming = incoming 110 | c.outgoing = outgoing 111 | 112 | falseptr := false 113 | 114 | _, err = incoming.CreateDataChannel("data-out", &webrtc.DataChannelInit{Ordered: &falseptr}) 115 | c.logger.NilErrCheck(c.label, "Failed to create an incoming data channel", err) 116 | 117 | _, err = outgoing.CreateDataChannel("data-in", &webrtc.DataChannelInit{Ordered: &falseptr}) 118 | c.logger.NilErrCheck(c.label, "Failed to create an outgoing data channel", err) 119 | 120 | _, err = c.outgoing.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio, webrtc.RTPTransceiverInit{ 121 | Direction: webrtc.RTPTransceiverDirectionSendonly, 122 | }) 123 | c.logger.NilErrCheck(c.label, "Error adding fake transceiver to outgoing pc", err) 124 | 125 | // keyframe throttling 126 | sendKeyFrameGate := false 127 | 128 | c.handler = razor.NewMessageHandler(c.logger, c.label, 1024, func(what clientCommand, payload *clientCommandMessage) bool { 129 | shouldSendKeyframe := false 130 | shouldEvalState := false 131 | 132 | stop := func() { 133 | c.logger.Info(c.label, "Stopping") 134 | 135 | c.handler.Abort() 136 | 137 | c.logger.Info(c.label, "Stopped") 138 | } 139 | 140 | switch what { 141 | case clientKeyframeTick: 142 | shouldSendKeyframe = true 143 | case clientKeyframeGateUnlock: 144 | sendKeyFrameGate = false 145 | case clientSendProto: 146 | if c.websocket == nil { 147 | c.logger.Error(c.label, "Attempting send when not connected to websocket") 148 | return true 149 | } 150 | 151 | data, err := proto.Marshal(payload.message) 152 | 153 | if err != nil { 154 | c.logger.Error(c.label, "Failed to marshal proto for client") 155 | stop() 156 | return true 157 | } 158 | 159 | c.logger.Verbose(c.label, "ws proto sending "+payload.message.String()) 160 | 161 | err = c.websocket.WriteMessage(websocket.BinaryMessage, data) 162 | 163 | if c.logger.NilErrCheck(c.label, "Failed to write proto to ws", err) { 164 | stop() 165 | return true 166 | } 167 | case clientHandleWSMessage: 168 | c.handleWsMessage(payload.message, s) 169 | case clientStop: 170 | stop() 171 | return true 172 | case clientAddOutgoingTrackForIncomingTrack: 173 | // Add it to our outgoing if it's not on incoming 174 | if _, incomingExists := c.incomingTracks[payload.incomingTrack.UmbrellaID()]; !incomingExists { 175 | c.outgoingTracks[payload.incomingTrack.UmbrellaID()] = &outgoingTrackWithClientState{ 176 | track: &outgoingTrack{descriptor: payload.incomingTrack.descriptor}, 177 | source: payload.incomingTrack, 178 | } 179 | 180 | c.handler.Cancel(clientEvalState) 181 | c.handler.Timeout(clientEvalState, nil, 500*time.Millisecond) 182 | } 183 | case clientRemoveOutgoingTracksForIncomingTrack: 184 | if _, exists := c.outgoingTracks[payload.incomingTrack.UmbrellaID()]; exists { 185 | delete(c.outgoingTracks, payload.incomingTrack.UmbrellaID()) 186 | 187 | c.handler.Cancel(clientEvalState) 188 | c.handler.Timeout(clientEvalState, nil, 500*time.Millisecond) 189 | } 190 | case clientEvalState: 191 | if c.pcTerminated() { 192 | stop() 193 | return true 194 | } 195 | 196 | shouldEvalState = true 197 | case clientGetStatus: 198 | // Like the SFU this is horribly blocking 199 | // Also could cause problems if the client goes AWOL while the SFU is wanting to get status (i.e. would need a timeout) 200 | 201 | intd := make([]*TrackDescriptor, 0) 202 | for _, t := range c.incomingTracks { 203 | intd = append(intd, t.track.descriptor) 204 | } 205 | outtd := make([]*TrackDescriptor, 0) 206 | for _, t := range c.outgoingTracks { 207 | outtd = append(outtd, t.track.descriptor) 208 | } 209 | senderStatus := make([]*SFUStatusSender, 0) 210 | for umbrellaId, s := range c.senders { 211 | t := s.Track() 212 | id := "" 213 | if t != nil { 214 | id = t.ID() 215 | } 216 | senderStatus = append(senderStatus, &SFUStatusSender{HasTrack: t != nil, TrackIdIfSet: id, UmbrellaId: umbrellaId}) 217 | } 218 | midMapping := make([]*MidToUmbrellaIDMapping, 0) 219 | for mid, umbrellaId := range c.incomingMidToUmbrellaTrackID { 220 | midMapping = append(midMapping, &MidToUmbrellaIDMapping{Mid: mid, UmbrellaId: umbrellaId}) 221 | } 222 | stagedIncoming := make([]*SFUStatusStagedIncomingTrack, 0) 223 | for _, s := range c.stagedIncomingTracks { 224 | stagedIncoming = append(stagedIncoming, &SFUStatusStagedIncomingTrack{ 225 | StreamId: s.track.StreamID(), 226 | TrackId: s.track.ID(), 227 | Mid: s.receiver.RTPTransceiver().Mid(), 228 | }) 229 | } 230 | 231 | status := &SFUStatusClient{ 232 | Label: c.label, 233 | TrunkUrl: c.trunkurl, 234 | IncomingPC: c.incoming.GetStatus(), 235 | OutgoingPC: c.outgoing.GetStatus(), 236 | IncomingTracks: intd, 237 | OutgoingTracks: outtd, 238 | Senders: senderStatus, 239 | MidMapping: midMapping, 240 | StagedIncomingTracks: stagedIncoming, 241 | } 242 | 243 | payload.result.status <- status 244 | case clientDialWs: 245 | if c.websocket != nil { 246 | return true 247 | } 248 | 249 | dialer := websocket.Dialer{ 250 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // TODO remove this bad ignoring of certs, but useful for local testing 251 | 252 | NetDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { 253 | host, port, err := net.SplitHostPort(addr) 254 | if err != nil { 255 | port = "443" // Default wss port 256 | } else { 257 | // Use our own mdns to discover it, if possible 258 | // This is to enable an OpenWrt AP to discover Apple servers on the same subnet 259 | if strings.HasSuffix(host, ".local") { 260 | found, err := s.mdnsLookup(host) 261 | if err == nil { 262 | host = found 263 | } 264 | } 265 | } 266 | 267 | dialer := &net.Dialer{} 268 | return dialer.DialContext(ctx, network, net.JoinHostPort(host, port)) 269 | }, 270 | } 271 | 272 | conn, _, err := dialer.Dial(c.trunkurl, nil) 273 | if c.logger.NilErrCheck(c.label, "Error dialling ws "+c.trunkurl, err) { 274 | c.handler.Timeout(clientDialWs, nil, time.Second*10) 275 | return true 276 | } 277 | 278 | c.websocket = conn 279 | go func() { 280 | c.continueWebsocket(s) 281 | }() 282 | case clientIncomingTrackAdded: 283 | t := payload.newincomingTrack.track 284 | tsc := payload.newincomingTrack.receiver.RTPTransceiver() 285 | 286 | c.logger.Info(c.label, "Incoming track to add stream id "+t.StreamID()+" "+t.ID()+" MID "+tsc.Mid()) 287 | 288 | c.stagedIncomingTracks = append(c.stagedIncomingTracks, payload.newincomingTrack) 289 | 290 | c.evalIncomingState(s) 291 | } 292 | 293 | if shouldEvalState { 294 | c.handler.Cancel(clientEvalState) 295 | 296 | shouldSendKeyframe = true 297 | c.evalState(s) 298 | } 299 | 300 | if shouldSendKeyframe { 301 | // Throttled 302 | if !sendKeyFrameGate { 303 | c.handler.Cancel(clientKeyframeTick) 304 | 305 | sendKeyFrameGate = true 306 | c.handler.Timeout(clientKeyframeGateUnlock, nil, time.Millisecond*500) 307 | 308 | for _, receiver := range c.incoming.GetReceivers() { 309 | if receiver.Track() != nil { 310 | _ = c.incoming.WriteRTCP([]rtcp.Packet{ 311 | &rtcp.PictureLossIndication{ 312 | MediaSSRC: uint32(receiver.Track().SSRC()), 313 | }, 314 | }) 315 | } 316 | } 317 | 318 | c.handler.Timeout(clientKeyframeTick, nil, time.Second*3) 319 | } 320 | } 321 | 322 | return true 323 | }) 324 | 325 | if ws == nil { 326 | c.handler.Send(clientDialWs, nil) 327 | } 328 | 329 | c.handler.Loop(func() { 330 | for _, it := range c.incomingTracks { 331 | s.removeOutgoingTracksForIncomingTrack(it.track) 332 | } 333 | 334 | if ws != nil { 335 | ws.Close() 336 | ws = nil 337 | } 338 | 339 | if outgoing != nil { 340 | outgoing.wrapped.GracefulClose() 341 | } 342 | outgoing = nil 343 | 344 | if incoming != nil { 345 | incoming.wrapped.GracefulClose() 346 | } 347 | incoming = nil 348 | 349 | c.logger.Info(c.label, "Post clean up finished") 350 | }) 351 | } 352 | 353 | func (c *client) stop() { 354 | c.handler.CancelAll() 355 | c.handler.Send(clientStop, nil) 356 | } 357 | 358 | func (c *client) pcTerminated() bool { 359 | return c.incoming.IsTerminated() || c.outgoing.IsTerminated() 360 | } 361 | 362 | func (c *client) handleWsMessage(message *RemoteNodeMessage, s *Sfu) { 363 | if message.Candidate != nil { 364 | c.logger.Info(c.label, "WS PROTO RECEIVED ice candidate") 365 | candidate := webrtc.ICECandidateInit{} 366 | if err := json.Unmarshal([]byte(message.Candidate.Candidate), &candidate); err != nil { 367 | c.logger.Error(c.label, "Failed to unmarshal json to candidate: "+err.Error()) 368 | return 369 | } 370 | 371 | c.logger.Info(c.label, "Got candidate: "+candidate.Candidate) 372 | 373 | pc := c.incoming 374 | // Incoming from pov of the sender! 375 | if message.Candidate.Incoming { 376 | pc = c.outgoing 377 | } 378 | 379 | if err := pc.AddICECandidate(candidate); err != nil { 380 | c.logger.Error(c.label, "Failed to add ICE candidate: "+err.Error()) 381 | return 382 | } 383 | } 384 | 385 | if message.Answer != nil { 386 | c.logger.Info(c.label, "WS PROTO RECEIVED answer") 387 | answer := webrtc.SessionDescription{} 388 | if err := json.Unmarshal([]byte(message.Answer.Answer), &answer); err != nil { 389 | c.logger.Error(c.label, "Failed to umarshal JSON to answer: "+err.Error()) 390 | return 391 | } 392 | 393 | c.logger.Info(c.label, "Got answer: "+answer.SDP) 394 | 395 | if err := c.outgoing.SetRemoteDescription(answer); err != nil { 396 | c.logger.Error(c.label, "Failed to set remote description on outgoing from answer: "+err.Error()) 397 | return 398 | } 399 | } 400 | 401 | if message.Offer != nil { 402 | c.logger.Info(c.label, "WS PROTO RECEIVED offer") 403 | offer := webrtc.SessionDescription{} 404 | if err := json.Unmarshal([]byte(message.Offer.Offer), &offer); err != nil { 405 | c.logger.Error(c.label, "Failed to umarshal JSON to offer: "+err.Error()) 406 | return 407 | } 408 | 409 | c.logger.Info(c.label, "Got offer: "+offer.SDP) 410 | 411 | if err := c.incoming.SetRemoteDescription(offer); err != nil { 412 | c.logger.Error(c.label, "Failed to set remote description on incoming from offer: "+err.Error()) 413 | return 414 | } 415 | 416 | answer, err := c.incoming.CreateAnswer(&webrtc.AnswerOptions{}) 417 | if err != nil { 418 | c.logger.Error(c.label, "Failed to create answer in response to offer: "+err.Error()) 419 | return 420 | } 421 | 422 | c.incoming.SetLocalDescription(answer) 423 | 424 | answerString, err := json.Marshal(answer) 425 | if err != nil { 426 | c.logger.Error(c.label, "Failed to marshal answer to json: "+err.Error()) 427 | return 428 | } 429 | 430 | c.writeProto(&RemoteNodeMessage{Answer: &AnswerMessage{Answer: string(answerString)}}) 431 | } 432 | 433 | if message.AcceptTracks != nil { 434 | c.logger.Info(c.label, "WS PROTO RECEIVED accept tracks "+message.AcceptTracks.String()) 435 | 436 | // Mark the accepted tracks and schedule an evaluation 437 | for _, td := range message.AcceptTracks.Tracks { 438 | ot := c.outgoingTracks[td.UmbrellaId] 439 | if ot != nil { 440 | ot.remoteNotified = true 441 | ot.remoteAccepted = true 442 | } 443 | } 444 | 445 | c.handler.Send(clientEvalState, nil) 446 | } 447 | 448 | if message.UpstreamTracks != nil { 449 | c.logger.Info(c.label, "WS PROTO RECEIVED upstream tracks "+message.UpstreamTracks.String()) 450 | 451 | // Any previously unknown upstream tracks need to have transceivers created for them 452 | // Then we acknowledge with the complete set of expected upstream tracks 453 | for _, td := range message.UpstreamTracks.Tracks { 454 | _, exists := c.incomingTracks[td.UmbrellaId] 455 | if !exists { 456 | if td.Kind != TrackKind_Unknown { 457 | c.logger.Info(c.label, "Adding transceiver "+td.Id+" "+td.Kind.String()) 458 | 459 | _, err := c.incoming.AddTransceiverFromKind(trackKindToWebrtcKind(td.Kind), webrtc.RTPTransceiverInit{ 460 | Direction: webrtc.RTPTransceiverDirectionRecvonly, 461 | }) 462 | 463 | if err != nil { 464 | c.logger.Error(c.label, "Failed to add transceiver "+err.Error()) 465 | } else { 466 | intrack := &incomingTrackWithClientState{ 467 | track: &incomingTrack{descriptor: td}, 468 | } 469 | 470 | c.incomingTracks[intrack.UmbrellaID()] = intrack 471 | } 472 | } 473 | } 474 | } 475 | 476 | acceptedTracks := make([]*TrackDescriptor, 0) 477 | for _, intrack := range c.incomingTracks { 478 | acceptedTracks = append(acceptedTracks, intrack.track.descriptor) 479 | } 480 | 481 | c.writeProto(&RemoteNodeMessage{AcceptTracks: &AcceptUpstreamTracks{Tracks: acceptedTracks}}) 482 | 483 | s.handler.Send(sfuSignalClients, nil) 484 | } 485 | 486 | if message.MidMappings != nil { 487 | c.logger.Info(c.label, "WS PROTO RECEIVED mid <-> umbrella mapping "+message.MidMappings.String()) 488 | // Review all incoming tracks to assign MIDs, and if newly so then fan out appropriately 489 | 490 | // Update all the mappings in the client 491 | for _, mapping := range message.MidMappings.Mapping { 492 | c.incomingMidToUmbrellaTrackID[mapping.Mid] = mapping.UmbrellaId 493 | } 494 | 495 | c.evalIncomingState(s) 496 | } 497 | } 498 | 499 | func (c *client) evalIncomingState(s *Sfu) { 500 | c.logger.Info(c.label, "Eval incoming state") 501 | 502 | // Review staged incoming tracks to see if any can be updated as a result 503 | for i := 0; i < len(c.stagedIncomingTracks); i++ { 504 | sit := c.stagedIncomingTracks[i] 505 | 506 | mid := sit.receiver.RTPTransceiver().Mid() 507 | if mid == "" { 508 | continue 509 | } 510 | 511 | umbrellaId, midKnown := c.incomingMidToUmbrellaTrackID[mid] 512 | if midKnown { 513 | intrack, trackExists := c.incomingTracks[umbrellaId] 514 | if trackExists { 515 | if intrack.track.remote == nil { 516 | c.logger.Info(c.label, "Ready to fan out track: "+umbrellaId) 517 | 518 | // Assign it, remove from staged, and launch fan out 519 | intrack.track.remote = sit.track 520 | 521 | intrack.track.descriptor.Id = sit.track.ID() 522 | intrack.track.descriptor.StreamId = sit.track.StreamID() 523 | intrack.track.receiver = sit.receiver 524 | intrack.transceiverMid = mid 525 | 526 | // Remove from staged 527 | c.stagedIncomingTracks = append(c.stagedIncomingTracks[:i], c.stagedIncomingTracks[i+1:]...) 528 | i-- 529 | 530 | relay, err := webrtc.NewTrackLocalStaticRTP(intrack.track.remote.Codec().RTPCodecCapability, "UMB_RELAY"+uuid.New().String(), intrack.track.remote.StreamID()) 531 | if err != nil { 532 | panic(err) 533 | } 534 | 535 | intrack.track.relay = relay 536 | 537 | go c.fanoutIncoming(intrack.track, s) 538 | 539 | s.handler.Send(sfuAddOutgoingTracksForIncomingTrack, &sfuCommandMessage{intrack: intrack.track}) 540 | c.logger.Info(c.label, "New incoming track sent to SFU: "+umbrellaId) 541 | } 542 | } 543 | } 544 | } 545 | } 546 | 547 | func (c *client) fanoutIncoming(intrack *incomingTrack, s *Sfu) { 548 | defer s.removeOutgoingTracksForIncomingTrack(intrack) 549 | 550 | bufSize := 32768 551 | 552 | if intrack.remote.Kind() == webrtc.RTPCodecTypeVideo { 553 | bufSize = bufSize * 8 554 | } 555 | 556 | buf := make([]byte, bufSize) 557 | 558 | rtpPkt := &rtp.Packet{} 559 | for { 560 | i, _, err := intrack.remote.Read(buf) 561 | if c.logger.NilErrCheck(c.label, "Fan out error reading rtp on track "+intrack.String(), err) { 562 | return 563 | } 564 | 565 | if err = rtpPkt.Unmarshal(buf[:i]); err != nil { 566 | c.logger.Error(c.label, "Error unmarshaling rtp on track "+intrack.String()+" "+err.Error()) 567 | return 568 | } 569 | 570 | rtpPkt.Extension = false 571 | rtpPkt.Extensions = nil 572 | 573 | if err := intrack.relay.WriteRTP(rtpPkt); err != nil { 574 | c.logger.Error(c.label, "Error writing rtp from "+intrack.String()+" to relay "+err.Error()) 575 | return 576 | } 577 | } 578 | } 579 | 580 | func (c *client) evalState(s *Sfu) { 581 | // Do we have any non attached tracks we should be uploading which have not yet been sent for confirmation? 582 | needsNotification := false 583 | for _, ot := range c.outgoingTracks { 584 | needsNotification = needsNotification || !ot.remoteNotified 585 | } 586 | 587 | // If so, send them up, return and wait before evalling again 588 | if needsNotification { 589 | notifications := make([]*TrackDescriptor, 0) 590 | for _, ot := range c.outgoingTracks { 591 | notifications = append(notifications, ot.track.descriptor) 592 | ot.remoteNotified = true 593 | } 594 | 595 | c.writeProto(&RemoteNodeMessage{UpstreamTracks: &SetUpstreamTracks{Tracks: notifications}}) 596 | 597 | c.handler.Cancel(clientEvalState) 598 | c.handler.Timeout(clientEvalState, nil, 300*time.Millisecond) // Should probably do this a better way, like in the event listener 599 | return 600 | } 601 | 602 | // If we have any we're waiting on confirmation of we should return and wait before evalling again 603 | needsConfirmation := false 604 | for _, ot := range c.outgoingTracks { 605 | needsConfirmation = needsConfirmation || !ot.remoteAccepted 606 | } 607 | 608 | if needsConfirmation { 609 | // Wait for the accept tracks event 610 | 611 | c.handler.Cancel(clientEvalState) 612 | c.handler.Timeout(clientEvalState, nil, 300*time.Millisecond) // Should probably do this a better way, like in the event listener 613 | return 614 | } 615 | 616 | ss := c.outgoing.wrapped.SignalingState() 617 | if ss != webrtc.SignalingStateStable { 618 | c.handler.Cancel(clientEvalState) 619 | c.handler.Timeout(clientEvalState, nil, 300*time.Millisecond) // Should probably do this a better way, like in the event listener 620 | return 621 | } 622 | 623 | senderRemovalFailed := false 624 | // Find any senders that don't have a track, and remove them 625 | for umbrellaId, sender := range c.senders { 626 | _, exists := c.outgoingTracks[umbrellaId] 627 | if !exists { 628 | c.logger.Debug(c.label, "eval state removing sender for track with umb id "+umbrellaId) 629 | 630 | if err := c.outgoing.RemoveTrack(sender); err != nil { 631 | c.logger.Error(c.label, "Error removing sender for track with umb id "+umbrellaId+" "+err.Error()) 632 | senderRemovalFailed = true 633 | } else { 634 | delete(c.senders, umbrellaId) 635 | } 636 | } 637 | } 638 | 639 | if senderRemovalFailed { 640 | // Retry again later 641 | c.handler.Cancel(clientEvalState) 642 | c.handler.Timeout(clientEvalState, nil, 300*time.Millisecond) 643 | return 644 | } 645 | 646 | addingTrackFailed := false 647 | // Find any tracks which don't have a sender, and add them 648 | for umbrellaId, ot := range c.outgoingTracks { 649 | sender, exists := c.senders[umbrellaId] 650 | if !exists || sender.Track() == nil { 651 | c.logger.Debug(c.label, "eval state creating sender for track with umb id "+umbrellaId) 652 | if sender, err := c.outgoing.AddTrack(ot.source.relay); err != nil { 653 | c.logger.Error(c.label, "Error creating sender for track with umb id "+umbrellaId+" "+err.Error()) 654 | addingTrackFailed = true 655 | } else { 656 | c.senders[umbrellaId] = sender 657 | } 658 | } 659 | } 660 | 661 | if addingTrackFailed { 662 | // Retry again later 663 | c.handler.Cancel(clientEvalState) 664 | c.handler.Timeout(clientEvalState, nil, 300*time.Millisecond) 665 | return 666 | } 667 | 668 | c.logger.Info(c.label, "eval state creating offer") 669 | offer, err := c.outgoing.CreateOffer(nil) 670 | if c.logger.NilErrCheck(c.label, "eval state creating offer error", err) { 671 | // Retry again later 672 | c.handler.Cancel(clientEvalState) 673 | c.handler.Timeout(clientEvalState, nil, 300*time.Millisecond) 674 | c.logger.Info(c.label, "eval state creating offer erred so retry scheduled") 675 | return 676 | } 677 | 678 | c.logger.Info(c.label, "eval state setting local description") 679 | if err = c.outgoing.SetLocalDescription(offer); err != nil { 680 | c.logger.Error(c.label, "eval state setting local description error"+err.Error()) 681 | // Retry again later 682 | c.handler.Cancel(clientEvalState) 683 | c.handler.Timeout(clientEvalState, nil, 300*time.Millisecond) 684 | c.logger.Info(c.label, "eval state setting local description erred so retry scheduled") 685 | return 686 | } 687 | 688 | offerString, err := json.Marshal(offer) 689 | if c.logger.NilErrCheck(c.label, "Eval state offer marshalling failed", err) { 690 | return 691 | } 692 | 693 | c.logger.Info(c.label, "Send offer to client: "+offer.SDP) 694 | 695 | c.writeProto(&RemoteNodeMessage{ 696 | Offer: &OfferMessage{ 697 | Offer: string(offerString), 698 | }, 699 | }) 700 | } 701 | 702 | func (c *client) writeProto(m *RemoteNodeMessage) { 703 | c.handler.Send(clientSendProto, &clientCommandMessage{message: m}) 704 | } 705 | 706 | func (c *client) continueWebsocket(s *Sfu) { 707 | // This takes care of the various cleanup 708 | defer func() { 709 | c.stop() 710 | }() 711 | 712 | ws := c.websocket 713 | incoming := c.incoming 714 | outgoing := c.outgoing 715 | 716 | s.handler.Send(sfuAddClient, &sfuCommandMessage{client: c}) 717 | 718 | defer s.handler.Send(sfuRemoveClient, &sfuCommandMessage{client: c}) 719 | 720 | icecandidate := func(i *webrtc.ICECandidate, incoming bool) { 721 | if i == nil { 722 | return 723 | } 724 | 725 | // If you are serializing a candidate make sure to use ToJSON 726 | // Using Marshal will result in errors around `sdpMid` (thanks pion commenter!) 727 | candidateString, err := json.Marshal(i.ToJSON()) 728 | 729 | if c.logger.NilErrCheck(c.label, "Failed to marshal candidate to json", err) { 730 | return 731 | } 732 | 733 | c.writeProto(&RemoteNodeMessage{ 734 | Candidate: &CandidateMessage{ 735 | Candidate: string(candidateString), 736 | Incoming: incoming, 737 | }, 738 | }) 739 | } 740 | 741 | incoming.OnICECandidate = func(i *webrtc.ICECandidate) { 742 | icecandidate(i, true) 743 | } 744 | 745 | outgoing.OnICECandidate = func(i *webrtc.ICECandidate) { 746 | icecandidate(i, false) 747 | } 748 | 749 | incoming.OnConnectionStateChange = func(p webrtc.PeerConnectionState) { 750 | switch p { 751 | case webrtc.PeerConnectionStateFailed: 752 | case webrtc.PeerConnectionStateDisconnected: 753 | case webrtc.PeerConnectionStateClosed: 754 | c.stop() 755 | default: 756 | } 757 | 758 | s.handler.Send(sfuSignalClients, nil) 759 | } 760 | 761 | outgoing.OnConnectionStateChange = func(p webrtc.PeerConnectionState) { 762 | switch p { 763 | case webrtc.PeerConnectionStateFailed: 764 | case webrtc.PeerConnectionStateDisconnected: 765 | case webrtc.PeerConnectionStateClosed: 766 | c.stop() 767 | default: 768 | } 769 | 770 | s.handler.Send(sfuSignalClients, nil) 771 | } 772 | 773 | outgoing.OnSignalingStateChange = func(ss webrtc.SignalingState) { 774 | if ss == webrtc.SignalingStateStable { 775 | mappings := make([]*MidToUmbrellaIDMapping, 0) 776 | 777 | for umbrellaId := range c.outgoingTracks { 778 | sender, exists := c.senders[umbrellaId] 779 | if exists && sender.Track() != nil { 780 | for _, trx := range outgoing.GetTransceivers() { 781 | if trx.Sender() == sender { 782 | mappings = append(mappings, &MidToUmbrellaIDMapping{ 783 | Mid: trx.Mid(), 784 | UmbrellaId: umbrellaId, 785 | }) 786 | } 787 | } 788 | } 789 | } 790 | 791 | c.writeProto(&RemoteNodeMessage{MidMappings: &MidToUmbrellaIDMappings{Mapping: mappings}}) 792 | } 793 | } 794 | 795 | incoming.OnTrack = func(t *webrtc.TrackRemote, r *webrtc.RTPReceiver) { 796 | c.logger.Info(c.label, "OnTrack "+t.ID()+" "+r.RTPTransceiver().Mid()) 797 | c.handler.Send(clientIncomingTrackAdded, &clientCommandMessage{newincomingTrack: &rawIncomingTrack{track: t, receiver: r}}) 798 | } 799 | 800 | incoming.OnNegotiationNeeded = func() { 801 | c.handler.Send(clientEvalState, nil) 802 | } 803 | 804 | outgoing.OnNegotiationNeeded = func() { 805 | c.handler.Send(clientEvalState, nil) 806 | } 807 | 808 | // Signal for the new PeerConnection 809 | s.handler.Send(sfuSignalClients, nil) 810 | 811 | for { 812 | var message RemoteNodeMessage 813 | _, raw, err := ws.ReadMessage() 814 | if c.logger.NilErrCheck(c.label, "Failed to read message", err) { 815 | return 816 | } 817 | 818 | if err := proto.Unmarshal(raw, &message); err != nil { 819 | c.logger.Error(c.label, "Failed to unmarshal proto to message: "+err.Error()) 820 | return 821 | } 822 | 823 | c.handler.Send(clientHandleWSMessage, &clientCommandMessage{message: &message}) 824 | } 825 | } 826 | 827 | func (c *client) AddOutgoingTracksForIncomingTrack(intrack *incomingTrack) { 828 | c.handler.Send(clientAddOutgoingTrackForIncomingTrack, &clientCommandMessage{incomingTrack: intrack}) 829 | } 830 | 831 | func (c *client) RemoveOutgoingTracksForIncomingTrack(intrack *incomingTrack) { 832 | c.handler.Send(clientRemoveOutgoingTracksForIncomingTrack, &clientCommandMessage{incomingTrack: intrack}) 833 | } 834 | 835 | func (c *client) RequestEvalState() { 836 | c.handler.Send(clientEvalState, nil) 837 | } 838 | --------------------------------------------------------------------------------