├── .gitignore ├── .dockerignore ├── livekit_demo.png ├── screenshots └── simple.jpg ├── assets └── cursors │ └── normal.png ├── src ├── pixelstreaming │ ├── mod.rs │ ├── controller.rs │ ├── signaller │ │ ├── mod.rs │ │ ├── protocol.rs │ │ └── imp.rs │ ├── handler.rs │ ├── message.rs │ └── utils.rs ├── encoder.rs ├── settings.rs ├── helper.rs ├── capture │ ├── mod.rs │ └── driver.rs ├── gst_webrtc_encoder │ └── mod.rs ├── lib.rs └── livekit │ └── mod.rs ├── LICENSE ├── scripts ├── build-livekit-gstreamer-macos.sh └── generate-viewer-token.py ├── examples ├── simple │ ├── cursor.rs │ ├── main.rs │ └── camera_controller.rs └── livekit │ └── main.rs ├── docker └── Dockerfile ├── Cargo.toml └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.zed 3 | /.cargo 4 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .cargo 2 | docker/Dockerfile* 3 | target 4 | -------------------------------------------------------------------------------- /livekit_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rlamarche/bevy_streaming/HEAD/livekit_demo.png -------------------------------------------------------------------------------- /screenshots/simple.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rlamarche/bevy_streaming/HEAD/screenshots/simple.jpg -------------------------------------------------------------------------------- /assets/cursors/normal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rlamarche/bevy_streaming/HEAD/assets/cursors/normal.png -------------------------------------------------------------------------------- /src/pixelstreaming/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod controller; 2 | pub mod handler; 3 | pub mod message; 4 | pub mod signaller; 5 | pub mod utils; 6 | -------------------------------------------------------------------------------- /src/encoder.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::sync::Arc; 3 | 4 | pub trait StreamEncoder: Send + Sync { 5 | fn push_frame(&self, frame_data: &[u8]) -> Result<()>; 6 | fn start(&self) -> Result<()>; 7 | } 8 | 9 | pub type EncoderHandle = Arc; -------------------------------------------------------------------------------- /src/pixelstreaming/controller.rs: -------------------------------------------------------------------------------- 1 | use bevy_platform::collections::HashMap; 2 | use crossbeam_channel::Receiver; 3 | 4 | use super::handler::PSMessageHandler; 5 | 6 | pub struct PSControllerState { 7 | pub add_remove_handlers: Receiver<(String, Option)>, 8 | pub handlers: HashMap, 9 | } 10 | -------------------------------------------------------------------------------- /src/pixelstreaming/signaller/mod.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MPL-2.0 2 | #![allow(clippy::non_send_fields_in_send_ty, unused_doc_comments)] 3 | 4 | use gst::glib; 5 | use gstrswebrtc::signaller::Signallable; 6 | 7 | mod imp; 8 | mod protocol; 9 | 10 | glib::wrapper! { 11 | pub struct UePsSignaller(ObjectSubclass) @implements Signallable; 12 | } 13 | 14 | impl Default for UePsSignaller { 15 | fn default() -> Self { 16 | glib::Object::new() 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/settings.rs: -------------------------------------------------------------------------------- 1 | #[derive(Clone)] 2 | pub enum SignallingServer { 3 | GstWebRtc { 4 | uri: String, 5 | peer_id: Option, 6 | }, 7 | #[cfg(feature = "pixelstreaming")] 8 | PixelStreaming { 9 | uri: String, 10 | streamer_id: Option, 11 | }, 12 | } 13 | 14 | impl AsRef for SignallingServer { 15 | fn as_ref(&self) -> &Self { 16 | self 17 | } 18 | } 19 | 20 | #[derive(Clone, Default)] 21 | pub enum CongestionControl { 22 | #[default] 23 | Disabled, 24 | Homegrown, 25 | GoogleCongestionControl, 26 | } 27 | 28 | #[derive(Clone)] 29 | pub struct GstWebRtcSettings { 30 | pub signalling_server: SignallingServer, 31 | pub width: u32, 32 | pub height: u32, 33 | pub video_caps: Option, 34 | pub congestion_control: Option, 35 | /// Enables converting controller events to mouse/keyboard events 36 | pub enable_controller: bool, 37 | } 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Romain Lamarche 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /scripts/build-livekit-gstreamer-macos.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script to build gst-plugins-rs with LiveKit feature enabled on macOS 4 | 5 | set -e 6 | 7 | echo "=== Building gst-plugins-rs with LiveKit support ===" 8 | echo "Note: This installs to ~/.local/lib/gstreamer-1.0 to avoid Homebrew conflicts" 9 | echo "" 10 | 11 | BUILD_DIR="/tmp/gst-plugins-rs-build" 12 | mkdir -p "$BUILD_DIR" 13 | cd "$BUILD_DIR" 14 | 15 | if [ ! -d "gst-plugins-rs" ]; then 16 | echo "Cloning gst-plugins-rs repository..." 17 | git clone https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git 18 | cd gst-plugins-rs 19 | else 20 | cd gst-plugins-rs 21 | echo "Updating existing repository..." 22 | git fetch 23 | git pull 24 | fi 25 | 26 | echo "Building gst-plugin-webrtc with livekit feature..." 27 | cargo build --release --package gst-plugin-webrtc --features livekit 28 | 29 | BUILT_LIB="target/release/libgstrswebrtc.dylib" 30 | 31 | if [ ! -f "$BUILT_LIB" ]; then 32 | echo "Error: Built library not found at $BUILT_LIB" 33 | exit 1 34 | fi 35 | 36 | # Install to user directory instead of system directory 37 | # This follows Homebrew's recommendation to avoid plugin deletion on upgrades 38 | USER_PLUGIN_DIR="$HOME/.local/lib/gstreamer-1.0" 39 | mkdir -p "$USER_PLUGIN_DIR" 40 | 41 | echo "Installing to $USER_PLUGIN_DIR..." 42 | cp "$BUILT_LIB" "$USER_PLUGIN_DIR/" 43 | 44 | echo "=== Installation complete! ===" 45 | echo "" 46 | echo "To verify, run:" 47 | echo "GST_PLUGIN_PATH=$USER_PLUGIN_DIR gst-inspect-1.0 livekitwebrtcsink" 48 | echo "" 49 | echo "To use in your project, run:" 50 | echo "export GST_PLUGIN_PATH=$USER_PLUGIN_DIR" 51 | echo "cargo run" 52 | echo "" 53 | echo "Or add this to your ~/.zshrc or ~/.bash_profile for permanent setup:" 54 | echo "export GST_PLUGIN_PATH=\"\$HOME/.local/lib/gstreamer-1.0\"" 55 | -------------------------------------------------------------------------------- /scripts/generate-viewer-token.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env -S uv run --script 2 | # /// script 3 | # requires-python = ">=3.11" 4 | # dependencies = [ 5 | # "pyjwt", 6 | # "python-dotenv", 7 | # ] 8 | # /// 9 | 10 | import jwt 11 | import time 12 | import os 13 | import sys 14 | from dotenv import load_dotenv 15 | 16 | load_dotenv('.env.local', override=True) 17 | 18 | api_key = os.environ["LIVEKIT_API_KEY"] 19 | api_secret = os.environ["LIVEKIT_API_SECRET"] 20 | livekit_url = os.environ["LIVEKIT_URL"] 21 | room_name = os.environ["LIVEKIT_ROOM_NAME"] 22 | 23 | # Convert https:// to wss:// if needed 24 | if livekit_url.startswith("https://"): 25 | livekit_url = livekit_url.replace("https://", "wss://") 26 | elif livekit_url.startswith("http://"): 27 | livekit_url = livekit_url.replace("http://", "ws://") 28 | 29 | payload = { 30 | "exp": int(time.time()) + 86400, # 24 hours from now 31 | "iss": api_key, 32 | "nbf": int(time.time()) - 5, 33 | "sub": "viewer", 34 | "name": "Web Viewer", 35 | "video": { 36 | "room": room_name, 37 | "roomJoin": True, 38 | "canSubscribe": True, 39 | "canPublish": False, 40 | "canPublishData": False 41 | }, 42 | "iat": int(time.time()), 43 | "jti": "viewer-" + str(int(time.time())) 44 | } 45 | 46 | token = jwt.encode(payload, api_secret, algorithm="HS256") 47 | print("\n" + "="*60) 48 | print("LIVEKIT VIEWER TOKEN") 49 | print("="*60) 50 | print(f"\nRoom: {room_name}") 51 | print(f"LiveKit URL: {livekit_url}") 52 | print(f"\nToken:\n{token}") 53 | print("\n" + "="*60) 54 | print("HOW TO USE:") 55 | print("="*60) 56 | print("\n1. Go to: https://meet.livekit.io/") 57 | print("2. Click 'Custom' tab") 58 | print(f"3. Enter LiveKit URL: {livekit_url}") 59 | print("4. Paste the token above") 60 | print("5. Click 'Connect'") 61 | print("="*60) 62 | -------------------------------------------------------------------------------- /src/pixelstreaming/handler.rs: -------------------------------------------------------------------------------- 1 | use bevy_log::prelude::*; 2 | use crossbeam_channel::Receiver; 3 | use gst::glib::prelude::*; 4 | use gst_webrtc::WebRTCDataChannel; 5 | use gstrswebrtc::webrtcsink::BaseWebRTCSink; 6 | 7 | use super::message::PSMessage; 8 | 9 | #[allow(dead_code)] 10 | #[derive(Debug)] 11 | pub struct PSMessageHandler { 12 | signal_handler_id: glib::SignalHandlerId, 13 | data_channel: WebRTCDataChannel, 14 | pub message_receiver: Receiver, 15 | } 16 | 17 | impl PSMessageHandler { 18 | pub fn new(element: &BaseWebRTCSink, webrtcbin: &gst::Element, session_id: &str) -> Self { 19 | info!("Creating Pixel Streaming data channel"); 20 | let channel = webrtcbin.emit_by_name::( 21 | "create-data-channel", 22 | &[ 23 | &"input", 24 | &gst::Structure::builder("config") 25 | .field("priority", gst_webrtc::WebRTCPriorityType::High) 26 | .build(), 27 | ], 28 | ); 29 | 30 | let session_id = session_id.to_string(); 31 | 32 | let (sender, receiver) = crossbeam_channel::unbounded::(); 33 | 34 | #[allow(unused)] 35 | Self { 36 | signal_handler_id: channel.connect_closure("on-message-data", false, { 37 | let sender = sender.clone(); 38 | glib::closure!( 39 | #[watch] 40 | element, 41 | #[strong] 42 | session_id, 43 | move |_channel: &WebRTCDataChannel, data: &glib::Bytes| { 44 | match PSMessage::try_from(data.get(..).unwrap()) { 45 | Ok(message) => { 46 | sender.send(message).unwrap(); 47 | } 48 | Err(error) => { 49 | warn!("Unable to decode UE Message: {}", error); 50 | } 51 | } 52 | } 53 | ) 54 | }), 55 | data_channel: channel, 56 | message_receiver: receiver, 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /examples/simple/cursor.rs: -------------------------------------------------------------------------------- 1 | use bevy::prelude::*; 2 | use bevy::window::PrimaryWindow; 3 | use bevy_window::WindowEvent; 4 | 5 | pub(crate) struct CursorPlugin; 6 | 7 | #[derive(Component)] 8 | struct Cursor {} 9 | 10 | impl Plugin for CursorPlugin { 11 | fn build(&self, app: &mut App) { 12 | app.add_systems(Startup, setup) 13 | .add_systems(PreUpdate, update_cursor_camera) 14 | .add_systems(Update, update_cursor_position); 15 | } 16 | } 17 | 18 | fn setup( 19 | mut commands: Commands, 20 | asset_server: Res, 21 | q_window: Query<&Window, With>, 22 | ) { 23 | let cursor_image = ImageNode::new(asset_server.load("cursors/normal.png")); 24 | 25 | let mut spawnpos = (0.0, 0.0); 26 | 27 | if let Some(position) = q_window.single().unwrap().cursor_position() { 28 | spawnpos = (position.x, position.y); 29 | } 30 | 31 | commands.spawn(( 32 | cursor_image, 33 | Node { 34 | position_type: PositionType::Absolute, 35 | top: Val::Px(spawnpos.1), 36 | left: Val::Px(spawnpos.0), 37 | ..default() 38 | }, 39 | Cursor {}, 40 | )); 41 | } 42 | 43 | fn update_cursor_camera( 44 | mut commands: Commands, 45 | q_camera: Query>, 46 | q_cursor: Query, Without)>, 47 | ) { 48 | if let Some(cursor_entity) = q_cursor.iter().next() { 49 | if let Some(camera_entity) = q_camera.iter().next() { 50 | commands 51 | .entity(cursor_entity) 52 | .insert(UiTargetCamera(camera_entity)); 53 | } 54 | } 55 | } 56 | 57 | fn update_cursor_position( 58 | mut q_cursor: Query<&mut Node, With>, 59 | mut window_events: EventReader, 60 | ) { 61 | let mut cursor = q_cursor.single_mut(); 62 | 63 | if let Some(WindowEvent::CursorMoved(cursor_moved)) = window_events 64 | .read() 65 | .filter(|event| matches!(event, WindowEvent::CursorMoved(..))) 66 | .last() 67 | { 68 | let cursor = cursor.as_mut().unwrap(); 69 | cursor.top = Val::Px(cursor_moved.position.y); 70 | cursor.left = Val::Px(cursor_moved.position.x); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Stage 1: Build dependencies 2 | FROM ubuntu:24.04 AS builder 3 | 4 | # Update package lists and install necessary packages 5 | RUN apt-get update && apt-get install -y --no-install-recommends \ 6 | curl \ 7 | ca-certificates \ 8 | build-essential pkg-config \ 9 | libssl-dev \ 10 | libvulkan-dev \ 11 | gstreamer1.0-plugins-base \ 12 | gstreamer1.0-plugins-good \ 13 | gstreamer1.0-plugins-bad \ 14 | gstreamer1.0-plugins-ugly \ 15 | gstreamer1.0-libav \ 16 | gstreamer1.0-nice \ 17 | gstreamer1.0-tools \ 18 | libgstreamer1.0-dev \ 19 | libgstreamer-plugins-base1.0-dev \ 20 | libgstreamer-plugins-good1.0-dev \ 21 | libgstreamer-plugins-bad1.0-dev \ 22 | libasound2-dev \ 23 | && apt-get clean && rm -rf /var/lib/apt/lists/* 24 | 25 | # Install Rust 26 | RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y 27 | ENV PATH="/root/.cargo/bin:${PATH}" 28 | 29 | 30 | # Set working directory 31 | WORKDIR /app 32 | 33 | # Copy Cargo.toml and Cargo.lock 34 | COPY Cargo.toml Cargo.lock ./ 35 | # Empty lib 36 | RUN mkdir ./src && touch ./src/lib.rs 37 | # Empty example 38 | RUN mkdir -p ./examples/simple && echo 'fn main() {}' > ./examples/simple/main.rs 39 | 40 | # Install Rust dependencies 41 | RUN cargo build --locked --release --examples --target x86_64-unknown-linux-gnu 42 | 43 | # Copy source code 44 | COPY src src 45 | COPY examples examples 46 | # Need to touch files after erasing them to have them build again 47 | RUN touch ./examples/simple/main.rs && touch ./src/lib.rs 48 | 49 | # Build the application 50 | RUN cargo build --locked --release --examples --target x86_64-unknown-linux-gnu 51 | 52 | # Stage 2: Final image 53 | FROM ubuntu:24.04 54 | 55 | # Install necessary runtime dependencies (only what's needed for the app to run) 56 | RUN apt-get update && apt-get install -y --no-install-recommends \ 57 | libssl3 \ 58 | ca-certificates \ 59 | libvulkan1 \ 60 | gstreamer1.0-plugins-base \ 61 | gstreamer1.0-plugins-good \ 62 | gstreamer1.0-plugins-bad \ 63 | gstreamer1.0-plugins-ugly \ 64 | # vaapi encoders 65 | gstreamer1.0-vaapi \ 66 | gstreamer1.0-libav \ 67 | gstreamer1.0-nice \ 68 | gstreamer1.0-tools \ 69 | libasound2t64 \ 70 | # make vulkan backend available for mesa 71 | mesa-vulkan-drivers \ 72 | # intel va encoder driver 73 | intel-media-va-driver \ 74 | # useful to check vulkan drivers 75 | vulkan-tools \ 76 | && apt-get clean && rm -rf /var/lib/apt/lists/* 77 | 78 | # Set working directory 79 | WORKDIR /app 80 | 81 | # Copy only the necessary built application from the builder stage 82 | COPY --from=builder /app/target/x86_64-unknown-linux-gnu/release/examples/simple . 83 | COPY assets /app/assets 84 | 85 | # Define the entrypoint 86 | CMD ["./simple"] 87 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bevy_streaming" 3 | version = "0.1.0" 4 | edition = "2024" 5 | resolver = "2" 6 | 7 | [lib] 8 | name = "bevy_streaming" 9 | crate-type = ["lib"] 10 | path = "src/lib.rs" 11 | 12 | [[example]] 13 | name = "simple" 14 | required-features = ["pixelstreaming"] 15 | 16 | [[example]] 17 | name = "livekit" 18 | required-features = ["livekit"] 19 | 20 | [dependencies] 21 | bevy_app = { version = "0.16" } 22 | bevy_ecs = { version = "0.16" } 23 | bevy_render = { version = "0.16" } 24 | bevy_asset = { version = "0.16" } 25 | bevy_image = { version = "0.16" } 26 | bevy_log = { version = "0.16" } 27 | bevy_input = { version = "0.16" } 28 | bevy_picking = { version = "0.16" } 29 | bevy_math = { version = "0.16" } 30 | bevy_window = { version = "0.16", optional = true } 31 | bevy_utils = { version = "0.16" } 32 | bevy_derive = { version = "0.16" } 33 | bevy_platform = { version = "0.16" } 34 | crossbeam-channel = "0.5" 35 | 36 | ## GSTREAMER 37 | glib = { package = "glib", version = "0.20.0" } 38 | gst = { package = "gstreamer", version = "0.23" } 39 | gst-app = { package = "gstreamer-app", version = "0.23" } 40 | gst-base = { package = "gstreamer-base", version = "0.23" } 41 | gst-video = { package = "gstreamer-video", version = "0.23" } 42 | gst-sdp = { package = "gstreamer-sdp", version = "0.23" } 43 | gst-rtp = { package = "gstreamer-rtp", version = "0.23" } 44 | gst-webrtc = { package = "gstreamer-webrtc", version = "0.23" } 45 | gst-utils = { package = "gstreamer-utils", version = "0.23" } 46 | gst-plugin-webrtc = "0.13.3" 47 | gst-plugin-rtp = "0.13.3" 48 | anyhow = "1" 49 | derive_more = { version = "1", features = ["display", "error"] } 50 | 51 | tokio = { version = "1", features = [ 52 | "fs", 53 | "macros", 54 | "rt-multi-thread", 55 | "time", 56 | ], optional = true } 57 | tokio-native-tls = { version = "0.3.0", optional = true } 58 | tokio-stream = { version = "0.1.11", optional = true } 59 | serde = { version = "1", features = ["derive"], optional = true } 60 | serde_json = { version = "1", optional = true } 61 | futures = { version = "0.3", optional = true } 62 | async-tungstenite = { version = "0.29", optional = true, features = [ 63 | "tokio-runtime", 64 | "tokio-native-tls", 65 | "url", 66 | ] } 67 | url = { version = "2", optional = true } 68 | byteorder = { version = "1.5.0", optional = true } 69 | 70 | [dev-dependencies] 71 | bevy = { version = "0.16" } 72 | 73 | [features] 74 | default = ["pixelstreaming"] 75 | cuda = [] 76 | pixelstreaming = [ 77 | "dep:url", 78 | "dep:async-tungstenite", 79 | "dep:serde", 80 | "dep:futures", 81 | "dep:serde_json", 82 | "dep:tokio", 83 | "dep:tokio-native-tls", 84 | "dep:tokio-stream", 85 | "dep:byteorder", 86 | "dep:bevy_window", 87 | ] 88 | livekit = [] 89 | 90 | # Enable a small amount of optimization in the dev profile. 91 | [profile.dev] 92 | opt-level = 1 93 | 94 | # Enable a large amount of optimization in the dev profile for dependencies. 95 | [profile.dev.package."*"] 96 | opt-level = 3 97 | 98 | 99 | # Enable more optimization in the release profile at the cost of compile time. 100 | [profile.release] 101 | # Compile the entire crate as one unit. 102 | # Slows compile times, marginal improvements. 103 | codegen-units = 1 104 | # Do a second optimization pass over the entire program, including dependencies. 105 | # Slows compile times, marginal improvements. 106 | lto = "thin" 107 | 108 | # Optimize for size in the wasm-release profile to reduce load times and bandwidth usage on web. 109 | [profile.wasm-release] 110 | # Default to release profile values. 111 | inherits = "release" 112 | # Optimize with size in mind (also try "z", sometimes it is better). 113 | # Slightly slows compile times, great improvements to file size and runtime performance. 114 | opt-level = "s" 115 | # Strip all debugging information from the binary to slightly reduce file size. 116 | strip = "debuginfo" 117 | -------------------------------------------------------------------------------- /src/helper.rs: -------------------------------------------------------------------------------- 1 | use bevy_asset::prelude::*; 2 | use bevy_ecs::{prelude::*, system::SystemParam}; 3 | use bevy_image::prelude::*; 4 | use bevy_log::prelude::*; 5 | use bevy_render::{prelude::*, renderer::RenderDevice}; 6 | use gst::prelude::*; 7 | use gstrswebrtc::webrtcsink; 8 | use std::{marker::PhantomData, sync::Arc}; 9 | 10 | use crate::{ 11 | capture::setup_render_target, encoder::StreamEncoder, gst_webrtc_encoder::GstWebRtcEncoder, ControllerState, GstWebRtcSettings 12 | }; 13 | #[cfg(feature = "livekit")] 14 | use crate::livekit::{LiveKitSettings, LiveKitEncoder}; 15 | 16 | #[cfg(feature = "pixelstreaming")] 17 | use crate::pixelstreaming::{controller::PSControllerState, handler::PSMessageHandler}; 18 | 19 | #[derive(SystemParam)] 20 | pub struct StreamerHelper<'w, 's, E: StreamEncoder + 'static> { 21 | commands: Commands<'w, 's>, 22 | images: ResMut<'w, Assets>, 23 | render_device: Res<'w, RenderDevice>, 24 | _phantom_encoder: PhantomData 25 | } 26 | 27 | pub trait StreamerCameraBuilder { 28 | fn new_streamer_camera(&mut self, settings: S) -> impl Bundle; 29 | } 30 | 31 | impl<'w, 's> StreamerCameraBuilder 32 | for StreamerHelper<'w, 's, GstWebRtcEncoder> 33 | { 34 | fn new_streamer_camera(&mut self, settings: GstWebRtcSettings) -> impl Bundle { 35 | let encoder = GstWebRtcEncoder::with_settings(settings.clone()) 36 | .expect("Unable to create gst encoder"); 37 | encoder.start().expect("Unable to start pipeline"); 38 | 39 | let controller_state = if settings.enable_controller { 40 | match &settings.signalling_server { 41 | #[cfg(feature = "pixelstreaming")] 42 | crate::SignallingServer::PixelStreaming { .. } => { 43 | create_pixelstreaming_controller(&encoder) 44 | } 45 | _ => ControllerState::None, 46 | } 47 | } else { 48 | ControllerState::None 49 | }; 50 | 51 | let render_target = setup_render_target( 52 | &mut self.commands, 53 | &mut self.images, 54 | &self.render_device, 55 | settings.width, 56 | settings.height, 57 | Arc::new(encoder), 58 | ); 59 | 60 | let camera = Camera { 61 | target: render_target, 62 | ..Default::default() 63 | }; 64 | 65 | (camera, controller_state) 66 | } 67 | } 68 | 69 | #[cfg(feature = "livekit")] 70 | impl<'w, 's> StreamerCameraBuilder 71 | for StreamerHelper<'w, 's, LiveKitEncoder> 72 | { 73 | fn new_streamer_camera(&mut self, settings: LiveKitSettings) -> impl Bundle { 74 | let encoder = LiveKitEncoder::new(settings.clone()) 75 | .expect("Unable to create LiveKit encoder"); 76 | 77 | let render_target = setup_render_target( 78 | &mut self.commands, 79 | &mut self.images, 80 | &self.render_device, 81 | settings.width, 82 | settings.height, 83 | encoder, 84 | ); 85 | 86 | let camera = Camera { 87 | target: render_target, 88 | ..Default::default() 89 | }; 90 | 91 | (camera, ControllerState::None) 92 | } 93 | } 94 | 95 | #[cfg(feature = "pixelstreaming")] 96 | fn create_pixelstreaming_controller(encoder: &GstWebRtcEncoder) -> ControllerState { 97 | use bevy_platform::collections::HashMap; 98 | 99 | let (sender, receiver) = crossbeam_channel::unbounded::<(String, Option)>(); 100 | 101 | encoder 102 | .webrtcsink 103 | .connect_closure("consumer-added", false, { 104 | let sender = sender.clone(); 105 | glib::closure!(move |sink: &webrtcsink::BaseWebRTCSink, 106 | peer_id: &str, 107 | webrtcbin: &gst::Element| { 108 | info!("New consumer: {}", peer_id); 109 | 110 | let message_handler = PSMessageHandler::new(sink, webrtcbin, peer_id); 111 | 112 | sender 113 | .send((peer_id.to_string(), Some(message_handler))) 114 | .unwrap(); 115 | }) 116 | }); 117 | 118 | encoder 119 | .webrtcsink 120 | .connect_closure("consumer-removed", false, { 121 | let sender = sender.clone(); 122 | glib::closure!(move |_sink: &webrtcsink::BaseWebRTCSink, 123 | peer_id: &str, 124 | _webrtcbin: &gst::Element| { 125 | info!("Consumer removed: {}", peer_id); 126 | 127 | sender.send((peer_id.to_string(), None)).unwrap(); 128 | }) 129 | }); 130 | 131 | ControllerState::PSControllerState(PSControllerState { 132 | add_remove_handlers: receiver, 133 | handlers: HashMap::new(), 134 | }) 135 | } 136 | -------------------------------------------------------------------------------- /src/capture/mod.rs: -------------------------------------------------------------------------------- 1 | use bevy_asset::{RenderAssetUsages, prelude::*}; 2 | use bevy_derive::{Deref, DerefMut}; 3 | use bevy_ecs::prelude::*; 4 | use bevy_image::prelude::*; 5 | use bevy_log::prelude::*; 6 | use bevy_render::{ 7 | Extract, 8 | camera::RenderTarget, 9 | render_resource::{ 10 | Buffer, BufferDescriptor, BufferUsages, Extent3d, TextureDimension, TextureFormat, 11 | TextureUsages, 12 | }, 13 | renderer::RenderDevice, 14 | }; 15 | use crossbeam_channel::{Receiver, Sender, unbounded}; 16 | use std::sync::{ 17 | Arc, 18 | atomic::{AtomicBool, AtomicUsize, Ordering}, 19 | }; 20 | 21 | use crate::encoder::EncoderHandle; 22 | pub mod driver; 23 | 24 | /// `Captures` aggregator in `RenderWorld` 25 | #[derive(Clone, Default, Resource, Deref, DerefMut)] 26 | pub struct Captures(pub Vec); 27 | 28 | /// Extracting `Capture`s into render world, because `ImageCopyDriver` accesses them 29 | pub fn capture_extract(mut commands: Commands, captures: Extract>) { 30 | commands.insert_resource(Captures(captures.iter().cloned().collect::>())); 31 | } 32 | 33 | #[derive(Clone)] 34 | struct CaptureBuffer { 35 | buffer: Buffer, 36 | in_use: Arc, 37 | } 38 | 39 | /// Used by `CaptureDriver` for copying from render target to buffer 40 | #[derive(Clone, Component)] 41 | pub struct Capture { 42 | buffers: Vec, 43 | current: Arc, 44 | skip: Arc, 45 | 46 | enabled: Arc, 47 | src_image: Handle, 48 | encoder: EncoderHandle, 49 | } 50 | 51 | pub struct SendBufferJob { 52 | // slice: BufferSlice<'static>, 53 | buffer: Buffer, 54 | // len: usize, 55 | encoder: EncoderHandle, 56 | // in_use: Arc, 57 | capture_idx: usize, 58 | buffer_idx: usize, 59 | } 60 | 61 | #[derive(Resource, Clone)] 62 | pub struct WorkerSendBuffer { 63 | pub tx: Sender, 64 | } 65 | 66 | #[derive(Resource, Clone)] 67 | pub struct ReleaseBufferSignal { 68 | pub rx: Receiver, 69 | } 70 | 71 | pub struct ReleaseSignal { 72 | // index of the capture 73 | capture_idx: usize, 74 | // index of the buffer to release 75 | buffer_idx: usize, 76 | } 77 | 78 | impl Capture { 79 | pub fn new( 80 | src_image: Handle, 81 | size: Extent3d, 82 | render_device: &RenderDevice, 83 | encoder: EncoderHandle, 84 | ) -> Self { 85 | let padded_bytes_per_row = 86 | RenderDevice::align_copy_bytes_per_row((size.width) as usize) * 4; 87 | 88 | let buffers = (0..3) // triple buffering 89 | .map(|_| { 90 | let buffer = render_device.create_buffer(&BufferDescriptor { 91 | label: Some("Capture buffer"), 92 | size: padded_bytes_per_row as u64 * size.height as u64, 93 | usage: BufferUsages::COPY_DST | BufferUsages::MAP_READ, 94 | mapped_at_creation: false, 95 | }); 96 | CaptureBuffer { 97 | buffer, 98 | in_use: Arc::new(AtomicBool::new(false)), 99 | } 100 | }) 101 | .collect(); 102 | 103 | Self { 104 | buffers, 105 | current: Arc::new(AtomicUsize::new(0)), 106 | skip: Arc::new(AtomicBool::new(false)), 107 | enabled: Arc::new(AtomicBool::new(true)), 108 | src_image, 109 | encoder, 110 | } 111 | } 112 | 113 | pub fn enabled(&self) -> bool { 114 | self.enabled.load(Ordering::Relaxed) 115 | } 116 | } 117 | 118 | /// Setups render target and cpu image for saving, changes scene state into render mode 119 | pub fn setup_render_target( 120 | commands: &mut Commands, 121 | images: &mut ResMut>, 122 | render_device: &Res, 123 | // render_instance: &Res, 124 | width: u32, 125 | height: u32, 126 | encoder: EncoderHandle, 127 | ) -> RenderTarget { 128 | let size = Extent3d { 129 | width, 130 | height, 131 | ..Default::default() 132 | }; 133 | 134 | // This is the texture that will be rendered to. 135 | let mut render_target_image = Image::new_fill( 136 | size, 137 | TextureDimension::D2, 138 | &[0; 4], 139 | TextureFormat::bevy_default(), 140 | RenderAssetUsages::default(), 141 | ); 142 | render_target_image.texture_descriptor.usage |= 143 | TextureUsages::COPY_SRC | TextureUsages::RENDER_ATTACHMENT | TextureUsages::TEXTURE_BINDING; 144 | let render_target_image_handle = images.add(render_target_image); 145 | 146 | commands.spawn(Capture::new( 147 | render_target_image_handle.clone(), 148 | size, 149 | render_device, 150 | encoder, 151 | )); 152 | 153 | // commands.spawn(ImageToSave(cpu_image_handle)); 154 | 155 | RenderTarget::Image(render_target_image_handle.into()) 156 | } 157 | 158 | pub fn spawn_worker() -> (Sender, Receiver) { 159 | let (tx_job, rx_job) = unbounded::(); 160 | let (tx_release, rx_release) = unbounded::(); 161 | 162 | std::thread::spawn(move || { 163 | while let Ok(job) = rx_job.recv() { 164 | let slice = job.buffer.slice(..); 165 | let data = slice.get_mapped_range().to_vec(); 166 | 167 | 168 | let _ = job.encoder.push_frame(&data); 169 | 170 | if let Err(e) = tx_release.send(ReleaseSignal { 171 | capture_idx: job.capture_idx, 172 | buffer_idx: job.buffer_idx, 173 | }) { 174 | error!("Release channel closed: {:?}", e); 175 | } 176 | } 177 | }); 178 | 179 | (tx_job, rx_release) 180 | } 181 | -------------------------------------------------------------------------------- /examples/livekit/main.rs: -------------------------------------------------------------------------------- 1 | use bevy::{ 2 | app::ScheduleRunnerPlugin, 3 | prelude::*, 4 | render::RenderPlugin, 5 | winit::WinitPlugin, 6 | }; 7 | use bevy_streaming::{livekit::{LiveKitEncoder, LiveKitSettings}, StreamerCameraBuilder, StreamerHelper}; 8 | use std::time::Duration; 9 | 10 | fn main() { 11 | App::new() 12 | .add_plugins(( 13 | DefaultPlugins 14 | .build() 15 | .disable::() 16 | // Make sure pipelines are ready before rendering 17 | .set(RenderPlugin { 18 | synchronous_pipeline_compilation: true, 19 | ..default() 20 | }), 21 | ScheduleRunnerPlugin::run_loop(Duration::from_secs_f64(1.0 / 60.0)), 22 | )) 23 | .add_plugins(bevy_streaming::StreamerPlugin) 24 | .add_systems(Startup, setup) 25 | .add_systems(Update, (move_player, rotate_camera)) 26 | .run(); 27 | } 28 | 29 | #[derive(Component)] 30 | struct Player; 31 | 32 | #[derive(Component)] 33 | struct SpectatorCamera; 34 | 35 | fn setup( 36 | mut commands: Commands, 37 | mut meshes: ResMut>, 38 | mut materials: ResMut>, 39 | mut helper: StreamerHelper, 40 | ) { 41 | commands.spawn(( 42 | Mesh3d(meshes.add(Plane3d::default().mesh().size(10.0, 10.0))), 43 | MeshMaterial3d(materials.add(StandardMaterial { 44 | base_color: Color::srgb(0.3, 0.5, 0.3), 45 | ..default() 46 | })), 47 | Transform::from_xyz(0.0, 0.0, 0.0), 48 | )); 49 | 50 | commands.spawn(( 51 | Mesh3d(meshes.add(Cuboid::new(1.0, 1.0, 1.0))), 52 | MeshMaterial3d(materials.add(StandardMaterial { 53 | base_color: Color::srgb(0.8, 0.2, 0.2), 54 | ..default() 55 | })), 56 | Transform::from_xyz(-2.0, 0.5, 0.0), 57 | )); 58 | 59 | commands.spawn(( 60 | Mesh3d(meshes.add(Sphere::new(0.5))), 61 | MeshMaterial3d(materials.add(StandardMaterial { 62 | base_color: Color::srgb(0.2, 0.2, 0.8), 63 | ..default() 64 | })), 65 | Transform::from_xyz(0.0, 0.5, 0.0), 66 | Player, 67 | )); 68 | 69 | commands.spawn(( 70 | PointLight { 71 | intensity: 1500.0, 72 | shadows_enabled: true, 73 | ..default() 74 | }, 75 | Transform::from_xyz(4.0, 8.0, 4.0), 76 | )); 77 | 78 | // Player camera with LiveKit streaming 79 | let livekit_settings = LiveKitSettings { 80 | url: std::env::var("LIVEKIT_URL") 81 | .expect("LIVEKIT_URL must be set"), 82 | api_key: std::env::var("LIVEKIT_API_KEY") 83 | .expect("LIVEKIT_API_KEY must be set"), 84 | api_secret: std::env::var("LIVEKIT_API_SECRET") 85 | .expect("LIVEKIT_API_SECRET must be set"), 86 | room_name: std::env::var("LIVEKIT_ROOM_NAME") 87 | .unwrap_or_else(|_| "bevy_streaming_demo".to_string()), 88 | participant_identity: std::env::var("LIVEKIT_PARTICIPANT_IDENTITY") 89 | .unwrap_or_else(|_| "bevy_player_camera".to_string()), 90 | participant_name: std::env::var("LIVEKIT_PARTICIPANT_NAME") 91 | .unwrap_or_else(|_| "Player Camera".to_string()), 92 | width: 1280, 93 | height: 720, 94 | enable_controller: false, 95 | }; 96 | 97 | commands.spawn(( 98 | helper.new_streamer_camera(livekit_settings), 99 | Camera3d::default(), 100 | Transform::from_xyz(0.0, 2.5, 5.0).looking_at(Vec3::ZERO, Vec3::Y), 101 | )); 102 | 103 | // Spectator camera with LiveKit streaming (different participant) 104 | let spectator_settings = LiveKitSettings { 105 | url: std::env::var("LIVEKIT_URL") 106 | .expect("LIVEKIT_URL must be set"), 107 | api_key: std::env::var("LIVEKIT_API_KEY") 108 | .expect("LIVEKIT_API_KEY must be set"), 109 | api_secret: std::env::var("LIVEKIT_API_SECRET") 110 | .expect("LIVEKIT_API_SECRET must be set"), 111 | room_name: std::env::var("LIVEKIT_ROOM_NAME") 112 | .unwrap_or_else(|_| "bevy_streaming_demo".to_string()), 113 | participant_identity: "bevy_spectator_camera".to_string(), 114 | participant_name: "Spectator Camera".to_string(), 115 | width: 1280, 116 | height: 720, 117 | enable_controller: false, 118 | }; 119 | 120 | commands.spawn(( 121 | helper.new_streamer_camera(spectator_settings), 122 | Camera3d::default(), 123 | Transform::from_xyz(5.0, 5.0, 5.0).looking_at(Vec3::ZERO, Vec3::Y), 124 | SpectatorCamera, 125 | )); 126 | } 127 | 128 | fn move_player( 129 | time: Res