├── init_maturin.sh ├── examples ├── Cargo.toml ├── flute-sender │ ├── Cargo.toml │ └── src │ │ └── main.rs └── flute-receiver │ ├── Cargo.toml │ └── src │ ├── main.rs │ └── msocket.rs ├── Makefile ├── src ├── py │ ├── sender │ │ ├── mod.rs │ │ ├── config.rs │ │ ├── senderpy.rs │ │ └── oti.rs │ ├── mod.rs │ └── receiver │ │ ├── mod.rs │ │ ├── udpendpoint.rs │ │ ├── multireceiver.rs │ │ ├── objectwriterbuilder.rs │ │ ├── receiverpy.rs │ │ ├── lct.rs │ │ └── config.rs ├── common │ ├── pkt.rs │ ├── mod.rs │ ├── udpendpoint.rs │ ├── alccodec │ │ ├── mod.rs │ │ ├── alcrs28.rs │ │ ├── alcnocode.rs │ │ ├── alcraptor.rs │ │ ├── alcraptorq.rs │ │ ├── alcrs28underspecified.rs │ │ └── alcrs2m.rs │ └── partition.rs ├── receiver │ ├── mod.rs │ ├── uncompress.rs │ ├── tsifilter.rs │ ├── blockdecoder.rs │ ├── writer │ │ ├── objectwriterbuffer.rs │ │ ├── objectwriterfs.rs │ │ └── mod.rs │ ├── objectreceiverlogger.rs │ ├── blockwriter.rs │ └── fdtreceiver.rs ├── sender │ ├── mod.rs │ ├── observer.rs │ ├── objectsenderlogger.rs │ ├── toiallocator.rs │ ├── sendersession.rs │ ├── block.rs │ ├── compress.rs │ └── blockencoder.rs ├── tools │ ├── error.rs │ ├── mod.rs │ └── ringbuffer.rs └── fec │ ├── mod.rs │ ├── nocode.rs │ ├── raptor.rs │ ├── raptorq.rs │ └── rscodec.rs ├── assets └── xsd │ ├── FLUTE-FDT-3GPP-2008-Extensions.xsd │ ├── FLUTE-FDT-3GPP-2009-Extensions.xsd │ ├── schema-version.xsd │ ├── FLUTE-FDT-3GPP-2015-Extensions.xsd │ ├── FLUTE-FDT-3GPP-2007-Extensions.xsd │ ├── FLUTE-FDT-3GPP-2005-Extensions.xsd │ ├── FLUTE-FDT-3GPP-2012-Extensions.xsd │ └── FLUTE-FDT-3GPP-Main.xsd ├── release.sh ├── pyproject.toml ├── .github ├── workflows │ ├── rust.yml │ ├── python.yml │ ├── coverage.yml │ ├── crates-io.yml │ ├── release-drafter.yml │ ├── release.yml │ └── pypi.yml └── release-drafter.yml ├── LICENSE ├── .gitignore ├── Cargo.toml ├── README.tpl └── py_tests └── __init__.py /init_maturin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | python3 -m venv venv 3 | source venv/bin/activate 4 | pip install maturin 5 | -------------------------------------------------------------------------------- /examples/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = [ 4 | "flute-receiver", 5 | "flute-sender", 6 | ] 7 | 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | test_py: 3 | maturin develop --all-features 4 | python3 -m unittest discover 5 | 6 | publish_py: 7 | act -j linux --env ACTIONS_RUNTIME_TOKEN=foo --artifact-server-path ./artifact 8 | 9 | readme: 10 | cargo readme > README.md 11 | -------------------------------------------------------------------------------- /src/py/sender/mod.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | mod config; 4 | mod oti; 5 | mod senderpy; 6 | 7 | #[pymodule] 8 | pub fn sender(m: &Bound<'_, PyModule>) -> PyResult<()> { 9 | m.add_class::()?; 10 | m.add_class::()?; 11 | m.add_class::()?; 12 | Ok(()) 13 | } 14 | -------------------------------------------------------------------------------- /assets/xsd/FLUTE-FDT-3GPP-2008-Extensions.xsd: -------------------------------------------------------------------------------- 1 | 2 | 6 | 8 | 9 | -------------------------------------------------------------------------------- /assets/xsd/FLUTE-FDT-3GPP-2009-Extensions.xsd: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /src/py/mod.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | use pyo3::wrap_pymodule; 3 | 4 | mod receiver; 5 | mod sender; 6 | 7 | /// A Python module implemented in Rust. 8 | #[pymodule] 9 | fn flute(m: &Bound<'_, PyModule>) -> PyResult<()> { 10 | pyo3_log::init(); 11 | m.add_wrapped(wrap_pymodule!(sender::sender))?; 12 | m.add_wrapped(wrap_pymodule!(receiver::receiver))?; 13 | Ok(()) 14 | } 15 | -------------------------------------------------------------------------------- /assets/xsd/schema-version.xsd: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /examples/flute-sender/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "flute-sender" 3 | version = "1.0.0" 4 | edition = "2021" 5 | description = "FLUTE Sender example" 6 | keywords = ["multicast", "network", "broadcast", "5g", "satellite"] 7 | categories = ["network-programming", "encoding", "aerospace::space-protocols"] 8 | 9 | [dependencies] 10 | log = "0.4" 11 | flute = { path = "../.." } 12 | env_logger = "0.11" 13 | url = "2.5.0" 14 | -------------------------------------------------------------------------------- /src/common/pkt.rs: -------------------------------------------------------------------------------- 1 | use super::lct; 2 | 3 | #[derive(Debug)] 4 | pub struct Pkt { 5 | pub payload: Vec, 6 | pub transfer_length: u64, 7 | pub esi: u32, 8 | pub sbn: u32, 9 | pub toi: u128, 10 | pub fdt_id: Option, 11 | pub cenc: lct::Cenc, 12 | pub inband_cenc: bool, 13 | pub close_object: bool, 14 | pub source_block_length: u32, 15 | pub sender_current_time: bool, 16 | } 17 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z ${1} ]; then 4 | current_version=$(cat Cargo.toml | grep '^version =' | awk '{print $3}' | tr -d '"') 5 | IFS='.' read -r major minor patch <<< "$current_version" 6 | ((patch++)) 7 | version="$major.$minor.$patch" 8 | else 9 | version=$1 10 | fi 11 | 12 | cargo set-version $version || exit -1 13 | git commit Cargo.toml -m "v$version" 14 | git tag v$version 15 | git push 16 | git push --tags 17 | -------------------------------------------------------------------------------- /examples/flute-receiver/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "flute-receiver" 3 | version = "1.0.0" 4 | edition = "2021" 5 | description = "FLUTE Receiver example" 6 | keywords = ["multicast", "network", "broadcast", "5g", "satellite"] 7 | categories = ["network-programming", "encoding", "aerospace::space-protocols"] 8 | 9 | [dependencies] 10 | log = "0.4" 11 | flute = { path = "../.." } 12 | env_logger = "0.11" 13 | url = "2.5.0" 14 | libc = "0.2" 15 | pnet = "0.34" -------------------------------------------------------------------------------- /src/common/mod.rs: -------------------------------------------------------------------------------- 1 | mod alccodec; 2 | 3 | /// FLUTE Profile 4 | #[derive(Debug, Copy, Clone)] 5 | pub enum Profile { 6 | /// FLUTE Version 2 7 | /// 8 | RFC6726, 9 | /// FLUTE Version 1 10 | /// 11 | RFC3926, 12 | } 13 | 14 | pub mod alc; 15 | pub mod fdtinstance; 16 | pub mod lct; 17 | pub mod oti; 18 | pub mod partition; 19 | pub mod pkt; 20 | pub mod udpendpoint; 21 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["maturin>=1.0,<2.0"] 3 | build-backend = "maturin" 4 | 5 | [project] 6 | name = "flute-alc" 7 | requires-python = ">=3.7" 8 | dynamic = ["version"] 9 | classifiers = [ 10 | "Programming Language :: Python", 11 | "Programming Language :: Python :: 3", 12 | "Programming Language :: Python :: 3 :: Only", 13 | "Programming Language :: Rust" 14 | ] 15 | 16 | [tool.maturin] 17 | compatibility = "manylinux2014" 18 | features = ["python"] 19 | 20 | -------------------------------------------------------------------------------- /assets/xsd/FLUTE-FDT-3GPP-2015-Extensions.xsd: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/receiver/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! FLUTE Receivers to re-construct ALC/LCT packets to Objects (files) 3 | //! 4 | 5 | mod blockdecoder; 6 | mod blockwriter; 7 | mod fdtreceiver; 8 | mod multireceiver; 9 | mod objectreceiver; 10 | mod receiver; 11 | mod tsifilter; 12 | mod uncompress; 13 | 14 | #[cfg(feature = "opentelemetry")] 15 | mod objectreceiverlogger; 16 | 17 | pub mod writer; 18 | pub use multireceiver::MultiReceiver; 19 | pub use multireceiver::MultiReceiverListener; 20 | pub use multireceiver::ReceiverEndpoint; 21 | pub use receiver::Config; 22 | pub use receiver::Receiver; 23 | -------------------------------------------------------------------------------- /src/py/receiver/mod.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | mod config; 4 | mod objectwriterbuilder; 5 | mod multireceiver; 6 | mod receiverpy; 7 | mod udpendpoint; 8 | mod lct; 9 | 10 | #[pymodule] 11 | pub fn receiver(m: &Bound<'_, PyModule>) -> PyResult<()> { 12 | m.add_class::()?; 13 | m.add_class::()?; 14 | m.add_class::()?; 15 | m.add_class::()?; 16 | m.add_class::()?; 17 | m.add_class::()?; 18 | Ok(()) 19 | } 20 | -------------------------------------------------------------------------------- /assets/xsd/FLUTE-FDT-3GPP-2007-Extensions.xsd: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | rust: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | - name: Install latest stable 18 | uses: actions-rs/toolchain@v1 19 | with: 20 | toolchain: stable 21 | - name: Install xmllint 22 | run: | 23 | sudo apt update 24 | sudo apt-get install -y libxml2-utils 25 | - name: Build 26 | run: cargo build --verbose 27 | - name: Run tests 28 | run: cargo test --verbose 29 | -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name-template: 'v$RESOLVED_VERSION 🌈' 2 | tag-template: 'v$RESOLVED_VERSION' 3 | categories: 4 | - title: '🚀 Features' 5 | labels: 6 | - 'feature' 7 | - 'enhancement' 8 | - title: '🐛 Bug Fixes' 9 | labels: 10 | - 'fix' 11 | - 'bugfix' 12 | - 'bug' 13 | - title: '🧰 Maintenance' 14 | label: 'chore' 15 | change-template: '- $TITLE @$AUTHOR (#$NUMBER)' 16 | change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks. 17 | version-resolver: 18 | major: 19 | labels: 20 | - 'major' 21 | minor: 22 | labels: 23 | - 'minor' 24 | patch: 25 | labels: 26 | - 'patch' 27 | default: patch 28 | template: | 29 | ## Changes 30 | 31 | $CHANGES -------------------------------------------------------------------------------- /src/py/receiver/udpendpoint.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | #[pyclass(unsendable)] 4 | #[derive(Debug)] 5 | pub struct UDPEndpoint { 6 | pub inner: crate::core::UDPEndpoint, 7 | } 8 | 9 | #[pymethods] 10 | impl UDPEndpoint { 11 | #[new] 12 | #[pyo3(signature = (destination_group_address, port, source_address=None))] 13 | fn new( 14 | destination_group_address: &str, 15 | port: u16, 16 | source_address: Option<&str>, 17 | ) -> PyResult { 18 | Ok(Self { 19 | inner: crate::core::UDPEndpoint { 20 | source_address: source_address.map(|f| f.to_string()), 21 | destination_group_address: destination_group_address.to_string(), 22 | port, 23 | }, 24 | }) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /assets/xsd/FLUTE-FDT-3GPP-2005-Extensions.xsd: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /src/common/udpendpoint.rs: -------------------------------------------------------------------------------- 1 | use std::hash::Hash; 2 | 3 | use serde::{Deserialize, Serialize}; 4 | 5 | /// UDP Endpoint 6 | #[derive(Debug, PartialEq, Deserialize, Serialize, Clone, Eq, Hash)] 7 | #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] 8 | pub struct UDPEndpoint { 9 | /// Network source adress 10 | pub source_address: Option, 11 | /// Network destination group address (multicast ip) 12 | pub destination_group_address: String, 13 | /// port 14 | pub port: u16, 15 | } 16 | 17 | impl UDPEndpoint { 18 | /// Create a new UDP Endpoint 19 | pub fn new(src: Option, dest: String, port: u16) -> Self { 20 | Self { 21 | source_address: src, 22 | destination_group_address: dest, 23 | port, 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.github/workflows/python.yml: -------------------------------------------------------------------------------- 1 | name: Python 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | jobs: 10 | python: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | os: ["ubuntu-latest"] 15 | steps: 16 | - uses: actions/checkout@v3 17 | - name: Install packages 18 | run: | 19 | sudo apt update 20 | sudo apt install -y python3 python3-pip python3-dev python3-venv 21 | - uses: actions-rs/toolchain@v1 22 | with: 23 | toolchain: stable 24 | 25 | - name: Run Python tests 26 | run: | 27 | python3 -m venv venv 28 | source venv/bin/activate 29 | pip3 install --upgrade pip 30 | pip3 install maturin 31 | make test_py 32 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: coverage 2 | 3 | on: [push] 4 | 5 | jobs: 6 | coverage: 7 | name: coverage 8 | runs-on: ubuntu-latest 9 | environment: coverage 10 | container: 11 | image: xd009642/tarpaulin:develop-nightly 12 | options: --security-opt seccomp=unconfined 13 | steps: 14 | - name: Checkout repository 15 | uses: actions/checkout@v3 16 | - name: Install xmllint 17 | run: | 18 | apt-get update 19 | apt-get install -y libxml2-utils 20 | 21 | - name: Generate code coverage 22 | run: | 23 | cargo tarpaulin --verbose --workspace --timeout 120 --out xml 24 | 25 | - name: Upload to codecov.io 26 | uses: codecov/codecov-action@v2 27 | with: 28 | token: ${{secrets.CODECOV_TOKEN}} 29 | fail_ci_if_error: true 30 | -------------------------------------------------------------------------------- /src/py/receiver/multireceiver.rs: -------------------------------------------------------------------------------- 1 | use super::{config, objectwriterbuilder, udpendpoint}; 2 | use pyo3::{exceptions::PyTypeError, prelude::*}; 3 | use std::time::SystemTime; 4 | 5 | #[pyclass(unsendable)] 6 | #[derive(Debug)] 7 | pub struct MultiReceiver(crate::receiver::MultiReceiver); 8 | 9 | #[pymethods] 10 | impl MultiReceiver { 11 | #[new] 12 | fn new(writer: &objectwriterbuilder::ObjectWriterBuilder, config: &config::Config) -> Self { 13 | let c = config.0.clone(); 14 | Self { 15 | 0: crate::receiver::MultiReceiver::new(writer.inner.clone(), Some(c), false), 16 | } 17 | } 18 | 19 | fn push(&mut self, endpoint: &udpendpoint::UDPEndpoint, data: &[u8]) -> PyResult<()> { 20 | self.0 21 | .push(&endpoint.inner, data, SystemTime::now()) 22 | .map_err(|e| PyTypeError::new_err(e.0.to_string())) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/py/receiver/objectwriterbuilder.rs: -------------------------------------------------------------------------------- 1 | use pyo3::{exceptions::PyTypeError, prelude::*}; 2 | use std::rc::Rc; 3 | 4 | #[pyclass(unsendable)] 5 | #[derive(Debug)] 6 | pub struct ObjectWriterBuilder { 7 | pub inner: Rc, 8 | } 9 | 10 | #[pymethods] 11 | impl ObjectWriterBuilder { 12 | #[new] 13 | fn new(path: &str) -> PyResult { 14 | let writer = 15 | crate::receiver::writer::ObjectWriterFSBuilder::new(std::path::Path::new(path), true) 16 | .map_err(|e| PyTypeError::new_err(e.0.to_string()))?; 17 | Ok(Self { 18 | inner: Rc::new(writer), 19 | }) 20 | } 21 | 22 | #[staticmethod] 23 | fn new_buffer() -> Self { 24 | let writer = crate::receiver::writer::ObjectWriterBufferBuilder::new(true); 25 | Self { 26 | inner: Rc::new(writer), 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/py/receiver/receiverpy.rs: -------------------------------------------------------------------------------- 1 | use super::{config, objectwriterbuilder, udpendpoint}; 2 | use pyo3::{exceptions::PyTypeError, prelude::*}; 3 | use std::time::SystemTime; 4 | 5 | #[pyclass(unsendable)] 6 | #[derive(Debug)] 7 | pub struct Receiver(crate::receiver::Receiver); 8 | 9 | #[pymethods] 10 | impl Receiver { 11 | #[new] 12 | fn new( 13 | endpoint: &udpendpoint::UDPEndpoint, 14 | tsi: u64, 15 | writer: &objectwriterbuilder::ObjectWriterBuilder, 16 | config: &config::Config, 17 | ) -> Self { 18 | let c = config.0.clone(); 19 | Self { 20 | 0: crate::receiver::Receiver::new(&endpoint.inner, tsi, writer.inner.clone(), Some(c)), 21 | } 22 | } 23 | 24 | fn push(&mut self, data: &[u8]) -> PyResult<()> { 25 | self.0 26 | .push_data(data, SystemTime::now()) 27 | .map_err(|e| PyTypeError::new_err(e.0.to_string())) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/sender/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! FLUTE Sender to convert Objects (files) to ALC/LCT packets 3 | //! 4 | 5 | mod block; 6 | mod blockencoder; 7 | mod fdt; 8 | mod filedesc; 9 | mod objectdesc; 10 | mod observer; 11 | mod sender; 12 | mod sendersession; 13 | mod toiallocator; 14 | 15 | #[cfg(feature = "opentelemetry")] 16 | mod objectsenderlogger; 17 | 18 | pub mod compress; 19 | pub use crate::common::Profile; 20 | pub use objectdesc::CacheControl; 21 | pub use objectdesc::ObjectDesc; 22 | pub use objectdesc::ObjectDataSource; 23 | pub use objectdesc::ObjectDataStream; 24 | pub use objectdesc::ObjectDataStreamTrait; 25 | pub use objectdesc::TargetAcquisition; 26 | pub use objectdesc::CarouselRepeatMode; 27 | pub use observer::Event; 28 | pub use observer::FileInfo; 29 | pub use observer::Subscriber; 30 | pub use sender::Config; 31 | pub use sender::PriorityQueue; 32 | pub use sender::Sender; 33 | pub use sender::TOIMaxLength; 34 | pub use sender::FDTPublishMode; 35 | pub use toiallocator::Toi; 36 | 37 | -------------------------------------------------------------------------------- /.github/workflows/crates-io.yml: -------------------------------------------------------------------------------- 1 | name: Crates.io 2 | 3 | on: 4 | repository_dispatch: 5 | types: [release] 6 | 7 | env: 8 | CARGO_TERM_COLOR: always 9 | 10 | jobs: 11 | publish: 12 | runs-on: ubuntu-latest 13 | environment: production 14 | # if: "startsWith(github.ref, 'refs/tags/')" 15 | steps: 16 | - uses: actions/checkout@v3 17 | - name: Install latest stable 18 | uses: actions-rs/toolchain@v1 19 | with: 20 | toolchain: stable 21 | - name: Install xmllint 22 | run: | 23 | sudo apt update 24 | sudo apt-get install -y libxml2-utils 25 | - name: Login to crates.io 26 | uses: actions-rs/cargo@v1 27 | with: 28 | command: login 29 | args: ${{ secrets.CRATES_TOKEN }} 30 | - name: Cargo test 31 | uses: actions-rs/cargo@v1 32 | with: 33 | command: test 34 | - name: Publish to crates.io 35 | uses: actions-rs/cargo@v1 36 | with: 37 | command: publish 38 | -------------------------------------------------------------------------------- /src/tools/error.rs: -------------------------------------------------------------------------------- 1 | /// Generic FLUTE Error 2 | #[derive(Debug)] 3 | pub struct FluteError(pub std::io::Error); 4 | 5 | /// 6 | pub type Result = std::result::Result; 7 | 8 | impl FluteError { 9 | /// Return a new FLUTE Error with a message 10 | pub fn new(msg: E) -> Self 11 | where 12 | E: Into> + std::fmt::Debug, 13 | { 14 | log::error!("{:?}", msg); 15 | FluteError(std::io::Error::new(std::io::ErrorKind::Other, msg)) 16 | } 17 | 18 | /// Return a new FLUTE Error 19 | pub fn new_kind(kind: std::io::ErrorKind, msg: E) -> Self 20 | where 21 | E: Into> + std::fmt::Debug, 22 | { 23 | log::error!("{:?}", msg); 24 | FluteError(std::io::Error::new(kind, msg)) 25 | } 26 | } 27 | 28 | impl From for FluteError { 29 | fn from(err: std::io::Error) -> Self { 30 | log::error!("{:?}", err); 31 | FluteError(err) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /assets/xsd/FLUTE-FDT-3GPP-2012-Extensions.xsd: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 10 | 11 | 12 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Yannick Poirier 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/py/receiver/lct.rs: -------------------------------------------------------------------------------- 1 | use pyo3::{exceptions::PyTypeError, prelude::*}; 2 | 3 | use crate::common::alc; 4 | 5 | #[pyclass(unsendable)] 6 | #[derive(Debug)] 7 | pub struct LCTHeader { 8 | pub inner: crate::core::lct::LCTHeader, 9 | pub payload_id: Option, 10 | } 11 | 12 | #[pymethods] 13 | impl LCTHeader { 14 | #[new] 15 | fn new(data: &[u8]) -> PyResult { 16 | let alc = crate::core::alc::parse_alc_pkt(data) 17 | .map_err(|e| PyTypeError::new_err(e.0.to_string()))?; 18 | 19 | let payload_id = alc::get_fec_inline_payload_id(&alc).ok(); 20 | 21 | Ok(LCTHeader { 22 | inner: alc.lct, 23 | payload_id, 24 | }) 25 | } 26 | 27 | #[getter] 28 | fn cci(&self) -> PyResult { 29 | Ok(self.inner.cci) 30 | } 31 | 32 | #[getter] 33 | fn toi(&self) -> PyResult { 34 | Ok(self.inner.toi) 35 | } 36 | 37 | #[getter] 38 | fn tsi(&self) -> PyResult { 39 | Ok(self.inner.tsi) 40 | } 41 | 42 | #[getter] 43 | fn sbn(&self) -> PyResult> { 44 | Ok(self.payload_id.as_ref().map(|p| p.sbn)) 45 | } 46 | 47 | #[getter] 48 | fn esi(&self) -> PyResult> { 49 | Ok(self.payload_id.as_ref().map(|p| p.esi)) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/tools/mod.rs: -------------------------------------------------------------------------------- 1 | use self::error::FluteError; 2 | use self::error::Result; 3 | use std::time::SystemTime; 4 | 5 | /// Handle errors 6 | pub mod error; 7 | pub mod ringbuffer; 8 | 9 | /// Convert the `SystemTime`into NTP. 10 | pub fn system_time_to_ntp(time: SystemTime) -> Result { 11 | let duration = time 12 | .duration_since(std::time::UNIX_EPOCH) 13 | .map_err(|_| FluteError::new("Fail to get UNIX time"))?; 14 | let seconds_utc = duration.as_secs(); 15 | let submicro = duration.subsec_micros(); 16 | 17 | let seconds_ntp = seconds_utc + 2208988800u64; 18 | let fraction = (((submicro as u128) * (1u128 << 32)) / 1000000u128) as u32; 19 | Ok((seconds_ntp << 32) | (fraction as u64)) 20 | } 21 | 22 | /// Convert NTP to SystemTime 23 | pub fn ntp_to_system_time(ntp: u64) -> Result { 24 | let seconds_ntp = ntp >> 32; 25 | if seconds_ntp < 2208988800u64 { 26 | return Err(FluteError::new(format!( 27 | "Invalid NTP seconds {}", 28 | seconds_ntp 29 | ))); 30 | } 31 | let seconds_utc = seconds_ntp - 2208988800u64; 32 | let fraction = ntp & 0xFFFFFFFF; 33 | let submicro = ((fraction as u128 * 1000000u128) / (1u128 << 32)) as u64; 34 | let utc_micro = (seconds_utc * 1000000u64) + submicro; 35 | Ok(SystemTime::UNIX_EPOCH + std::time::Duration::from_micros(utc_micro)) 36 | } 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | /examples/target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | 13 | # Byte-compiled / optimized / DLL files 14 | __pycache__/ 15 | .pytest_cache/ 16 | *.py[cod] 17 | 18 | # C extensions 19 | *.so 20 | 21 | # Distribution / packaging 22 | .Python 23 | .venv/ 24 | env/ 25 | bin/ 26 | build/ 27 | develop-eggs/ 28 | dist/ 29 | eggs/ 30 | lib/ 31 | lib64/ 32 | parts/ 33 | sdist/ 34 | var/ 35 | include/ 36 | man/ 37 | venv/ 38 | *.egg-info/ 39 | .installed.cfg 40 | *.egg 41 | doc 42 | 43 | # Installer logs 44 | pip-log.txt 45 | pip-delete-this-directory.txt 46 | pip-selfcheck.json 47 | 48 | # Unit test / coverage reports 49 | htmlcov/ 50 | .tox/ 51 | .coverage 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | 56 | # Translations 57 | *.mo 58 | 59 | # Mr Developer 60 | .mr.developer.cfg 61 | .project 62 | .pydevproject 63 | 64 | # Rope 65 | .ropeproject 66 | 67 | # Django stuff: 68 | *.log 69 | *.pot 70 | 71 | .DS_Store 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyCharm 77 | .idea/ 78 | 79 | # VSCode 80 | .vscode/ 81 | 82 | # Pyenv 83 | .python-version 84 | 85 | # maturin 86 | .env 87 | 88 | # Act 89 | artifact/ -------------------------------------------------------------------------------- /src/fec/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod nocode; 2 | pub mod raptor; 3 | pub mod raptorq; 4 | pub mod rscodec; 5 | 6 | use crate::tools::error::Result; 7 | 8 | pub trait FecShard: Send + Sync + std::fmt::Debug { 9 | fn data(&self) -> &[u8]; 10 | fn esi(&self) -> u32; 11 | } 12 | 13 | #[derive(Debug)] 14 | pub struct DataFecShard { 15 | shard: Vec, 16 | index: u32, 17 | } 18 | 19 | impl FecShard for DataFecShard { 20 | fn data(&self) -> &[u8] { 21 | self.shard.as_ref() 22 | } 23 | 24 | fn esi(&self) -> u32 { 25 | self.index 26 | } 27 | } 28 | 29 | impl DataFecShard { 30 | pub fn new(shard: &[u8], index: u32) -> Self { 31 | DataFecShard { 32 | shard: shard.to_vec(), 33 | index, 34 | } 35 | } 36 | } 37 | 38 | pub trait FecEncoder { 39 | fn encode(&self, data: &[u8]) -> Result>>; 40 | } 41 | 42 | pub trait FecDecoder { 43 | fn push_symbol(&mut self, encoding_symbol: &[u8], esi: u32); 44 | fn can_decode(&self) -> bool; 45 | fn decode(&mut self) -> bool; 46 | fn source_block(&self) -> Result<&[u8]>; 47 | } 48 | 49 | impl std::fmt::Debug for dyn FecEncoder { 50 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 51 | write!(f, "FecEncoder {{ }}") 52 | } 53 | } 54 | 55 | impl std::fmt::Debug for dyn FecDecoder { 56 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 57 | write!(f, "FecDecoder {{ }}") 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | push: 5 | # branches to consider in the event; optional, defaults to all 6 | branches: 7 | - main 8 | 9 | # pull_request event is required only for autolabeler 10 | pull_request: 11 | # Only following types are handled by the action, but one can default to all as well 12 | types: [opened, reopened, synchronize] 13 | # pull_request_target event is required for autolabeler to support PRs from forks 14 | # pull_request_target: 15 | # types: [opened, reopened, synchronize] 16 | 17 | permissions: 18 | contents: read 19 | 20 | jobs: 21 | update_release_draft: 22 | permissions: 23 | # write permission is required to create a github release 24 | contents: write 25 | # write permission is required for autolabeler 26 | # otherwise, read permission is required at least 27 | pull-requests: write 28 | runs-on: ubuntu-latest 29 | steps: 30 | # (Optional) GitHub Enterprise requires GHE_HOST variable set 31 | #- name: Set GHE_HOST 32 | # run: | 33 | # echo "GHE_HOST=${GITHUB_SERVER_URL##https:\/\/}" >> $GITHUB_ENV 34 | 35 | # Drafts your next Release notes as Pull Requests are merged into "master" 36 | - uses: release-drafter/release-drafter@v6 37 | # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml 38 | # with: 39 | # config-name: my-config.yml 40 | # disable-autolabeler: true 41 | env: 42 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /src/py/receiver/config.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | #[pyclass] 4 | #[derive(Debug)] 5 | pub struct Config(pub crate::receiver::Config); 6 | 7 | #[pymethods] 8 | impl Config { 9 | #[new] 10 | fn new() -> Self { 11 | Self { 12 | 0: crate::receiver::Config { 13 | ..Default::default() 14 | }, 15 | } 16 | } 17 | 18 | #[getter] 19 | fn get_max_objects_error(&self) -> PyResult { 20 | Ok(self.0.max_objects_error) 21 | } 22 | 23 | #[setter] 24 | fn set_max_objects_error(&mut self, value: usize) -> PyResult<()> { 25 | self.0.max_objects_error = value; 26 | Ok(()) 27 | } 28 | 29 | #[getter] 30 | fn get_session_timeout_ms(&self) -> PyResult> { 31 | Ok(self 32 | .0 33 | .session_timeout 34 | .map(|timeout| timeout.as_millis() as u64)) 35 | } 36 | 37 | #[setter] 38 | fn set_session_timeout_ms(&mut self, value: Option) -> PyResult<()> { 39 | self.0.session_timeout = value.map(|timeout| std::time::Duration::from_millis(timeout)); 40 | Ok(()) 41 | } 42 | 43 | #[getter] 44 | fn get_object_timeout_ms(&self) -> PyResult> { 45 | Ok(self 46 | .0 47 | .object_timeout 48 | .map(|timeout| timeout.as_millis() as u64)) 49 | } 50 | 51 | #[setter] 52 | fn set_object_timeout_ms(&mut self, value: Option) -> PyResult<()> { 53 | self.0.object_timeout = value.map(|timeout| std::time::Duration::from_millis(timeout)); 54 | Ok(()) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "flute" 3 | version = "1.10.7" 4 | authors = ["Yannick Poirier "] 5 | edition = "2021" 6 | license = "MIT" 7 | description = "File Delivery over Unidirectional Transport (FLUTE)" 8 | homepage = "https://github.com/ypo/flute" 9 | repository = "https://github.com/ypo/flute" 10 | keywords = ["multicast", "network", "broadcast", "5g", "satellite"] 11 | categories = ["network-programming", "encoding", "aerospace::space-protocols"] 12 | rust-version = "1.66" 13 | 14 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 15 | [lib] 16 | name = "flute" 17 | crate-type = ["cdylib", "rlib"] 18 | path = "src/lib.rs" 19 | 20 | [dependencies] 21 | log = "0.4" 22 | chrono = "0.4" 23 | serde = { version = "1.0", features = ["derive"] } 24 | serde_json = "1.0" 25 | quick-xml = { version = "0.38", features = ["serialize"] } 26 | base64 = "0.22" 27 | url = "2.5" 28 | num-integer = "0.1" 29 | reed-solomon-erasure = "6.0" 30 | flate2 = "1.0" 31 | md5 = "0.8" 32 | pyo3 = { version = "0.25", features = ["extension-module"], optional = true } 33 | pyo3-log = { version = "0.12", optional = true } 34 | raptorq = "2.0" 35 | raptor-code = "1.0.9" 36 | opentelemetry = { version = "0.30", optional = true } 37 | opentelemetry-semantic-conventions = { version = "0.30" , optional = true } 38 | rand = "0.9" 39 | utoipa = { version = "5", optional = true } 40 | 41 | [dev-dependencies] 42 | env_logger = "0.11" 43 | tempfile = "3.10.1" 44 | 45 | [features] 46 | python = ["pyo3", "pyo3-log"] 47 | optel = ["opentelemetry", "opentelemetry-semantic-conventions"] 48 | openapi = ["utoipa"] 49 | -------------------------------------------------------------------------------- /src/sender/observer.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, RwLock}; 2 | 3 | /// File State Changed event 4 | #[derive(Debug, PartialEq, Eq, Hash, Clone)] 5 | pub struct FileInfo { 6 | /// Object TOI 7 | pub toi: u128, 8 | } 9 | 10 | /// Event 11 | #[derive(Debug, PartialEq, Eq, Hash, Clone)] 12 | pub enum Event { 13 | /// Transfer is started 14 | StartTransfer(FileInfo), 15 | /// Transfer has stopped 16 | StopTransfer(FileInfo), 17 | } 18 | 19 | /// Subscribe to events 20 | pub trait Subscriber: Send + Sync { 21 | /// Flute sender event 22 | fn on_sender_event(&self, evt: &Event, now: std::time::SystemTime); 23 | } 24 | 25 | #[derive(Clone)] 26 | pub struct ObserverList(Arc>>>); 27 | 28 | impl ObserverList { 29 | pub fn new() -> Self { 30 | ObserverList(Arc::new(RwLock::new(Vec::new()))) 31 | } 32 | 33 | pub fn subscribe(&mut self, s: Arc) { 34 | self.0.write().unwrap().push(s); 35 | } 36 | 37 | pub fn unsubscribe(&mut self, s: Arc) { 38 | self.0 39 | .write() 40 | .unwrap() 41 | .retain(|a| !std::ptr::eq(a.as_ref() as *const _, s.as_ref() as *const _)) 42 | } 43 | 44 | pub fn dispatch(&self, event: &Event, now: std::time::SystemTime) { 45 | let lock = self.0.read().unwrap(); 46 | 47 | for subscriber in lock.iter() { 48 | subscriber.on_sender_event(event, now); 49 | } 50 | } 51 | } 52 | 53 | impl std::fmt::Debug for ObserverList { 54 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 55 | write!(f, "ObserverList") 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/fec/nocode.rs: -------------------------------------------------------------------------------- 1 | use crate::error::FluteError; 2 | 3 | use super::FecDecoder; 4 | 5 | pub struct NoCodeDecoder { 6 | shards: Vec>>, 7 | nb_symbols: usize, 8 | data: Option>, 9 | } 10 | 11 | impl NoCodeDecoder { 12 | pub fn new(nb_source_symbols: usize) -> NoCodeDecoder { 13 | NoCodeDecoder { 14 | shards: vec![None; nb_source_symbols], 15 | nb_symbols: 0, 16 | data: None, 17 | } 18 | } 19 | } 20 | 21 | impl FecDecoder for NoCodeDecoder { 22 | fn push_symbol(&mut self, encoding_symbol: &[u8], esi: u32) { 23 | if self.shards.len() <= esi as usize { 24 | log::error!("ESI {} > {}", esi, self.shards.len()); 25 | return; 26 | } 27 | 28 | if self.shards[esi as usize].is_some() { 29 | return; 30 | } 31 | 32 | self.shards[esi as usize] = Some(encoding_symbol.to_vec()); 33 | self.nb_symbols += 1; 34 | } 35 | 36 | fn can_decode(&self) -> bool { 37 | self.nb_symbols == self.shards.len() 38 | } 39 | 40 | fn decode(&mut self) -> bool { 41 | if self.data.is_some() { 42 | return true; 43 | } 44 | 45 | if !self.can_decode() { 46 | return false; 47 | } 48 | 49 | let mut output = Vec::new(); 50 | for shard in &self.shards { 51 | output.extend(shard.as_ref().unwrap()); 52 | } 53 | 54 | self.data = Some(output); 55 | true 56 | } 57 | 58 | fn source_block(&self) -> crate::error::Result<&[u8]> { 59 | match self.data.as_ref() { 60 | Some(e) => Ok(e), 61 | None => Err(FluteError::new("Block not decoded")), 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/common/alccodec/mod.rs: -------------------------------------------------------------------------------- 1 | use super::{ 2 | alc::{AlcPkt, PayloadID}, 3 | lct, oti, pkt, 4 | }; 5 | use crate::tools::error::Result; 6 | 7 | mod alcnocode; 8 | mod alcraptor; 9 | mod alcraptorq; 10 | mod alcrs28; 11 | mod alcrs28underspecified; 12 | mod alcrs2m; 13 | 14 | pub trait AlcCodec { 15 | fn add_fti(&self, data: &mut Vec, oti: &oti::Oti, transfer_length: u64); 16 | fn get_fti(&self, data: &[u8], lct_header: &lct::LCTHeader) -> Result>; 17 | fn add_fec_payload_id(&self, data: &mut Vec, oti: &oti::Oti, pkt: &pkt::Pkt); 18 | fn get_fec_payload_id(&self, pkt: &AlcPkt, oti: &oti::Oti) -> Result; 19 | fn get_fec_inline_payload_id(&self, pkt: &AlcPkt) -> Result; 20 | fn fec_payload_id_block_length(&self) -> usize; 21 | } 22 | 23 | impl dyn AlcCodec { 24 | pub fn instance(fec: oti::FECEncodingID) -> &'static dyn AlcCodec { 25 | const NOCODE: alcnocode::AlcNoCode = alcnocode::AlcNoCode {}; 26 | const ALCRS28: alcrs28::AlcRS28 = alcrs28::AlcRS28 {}; 27 | const ALCRS2M: alcrs2m::AlcRS2m = alcrs2m::AlcRS2m {}; 28 | const ALCRS28UNDERSPECIFIED: alcrs28underspecified::AlcRS28UnderSpecified = 29 | alcrs28underspecified::AlcRS28UnderSpecified {}; 30 | const ALCRAPTORQ: alcraptorq::AlcRaptorQ = alcraptorq::AlcRaptorQ {}; 31 | const ALCRAPTOR: alcraptor::AlcRaptor = alcraptor::AlcRaptor {}; 32 | 33 | match fec { 34 | oti::FECEncodingID::NoCode => &NOCODE, 35 | oti::FECEncodingID::ReedSolomonGF2M => &ALCRS2M, 36 | oti::FECEncodingID::ReedSolomonGF28 => &ALCRS28, 37 | oti::FECEncodingID::ReedSolomonGF28UnderSpecified => &ALCRS28UNDERSPECIFIED, 38 | oti::FECEncodingID::RaptorQ => &ALCRAPTORQ, 39 | oti::FECEncodingID::Raptor => &ALCRAPTOR, 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /examples/flute-receiver/src/main.rs: -------------------------------------------------------------------------------- 1 | use flute::{ 2 | core::UDPEndpoint, 3 | receiver::{writer, MultiReceiver}, 4 | }; 5 | use std::rc::Rc; 6 | 7 | mod msocket; 8 | 9 | fn main() { 10 | env_logger::builder().try_init().ok(); 11 | 12 | let endpoint = UDPEndpoint::new(None, "224.0.0.1".to_string(), 3400); 13 | 14 | let args: Vec = std::env::args().collect(); 15 | if args.len() == 1 { 16 | println!( 17 | "Save FLUTE objects to a destination folder received from UDP/FLUTE {:?}", 18 | endpoint 19 | ); 20 | println!("Usage: {} path/to/destination_folder", args[0]); 21 | std::process::exit(0); 22 | } 23 | 24 | let dest_dir = std::path::Path::new(&args[1]); 25 | if !dest_dir.is_dir() { 26 | log::error!("{:?} is not a directory", dest_dir); 27 | std::process::exit(-1); 28 | } 29 | 30 | log::info!("Create FLUTE, write objects to {:?}", dest_dir); 31 | 32 | let mut config = flute::receiver::Config::default(); 33 | config.object_receive_once = true; // receive objects only once. If the same object is repeated, it will be ignored 34 | 35 | let enable_md5_check = true; 36 | let writer = Rc::new(writer::ObjectWriterFSBuilder::new(dest_dir, enable_md5_check).unwrap()); 37 | let mut receiver = MultiReceiver::new(writer, Some(config), false); 38 | 39 | // Receive from 224.0.0.1:3400 on 127.0.0.1 (lo) interface 40 | let socket = msocket::MSocket::new(&endpoint, Some("127.0.0.1"), false) 41 | .expect("Fail to create Multicast Socket"); 42 | 43 | let mut buf = [0; 2048]; 44 | loop { 45 | let (n, _src) = socket 46 | .sock 47 | .recv_from(&mut buf) 48 | .expect("Failed to receive data"); 49 | 50 | let now = std::time::SystemTime::now(); 51 | match receiver.push(&endpoint, &buf[..n], now) { 52 | Err(_) => log::error!("Wrong ALC/LCT packet"), 53 | _ => {} 54 | }; 55 | receiver.cleanup(now); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /README.tpl: -------------------------------------------------------------------------------- 1 | {{readme}} 2 | 3 | # Python bindings 4 | 5 | [![PyPI version](https://badge.fury.io/py/flute-alc.svg)](https://badge.fury.io/py/flute-alc) 6 | 7 | ## Installation 8 | 9 | ```bash 10 | pip install flute-alc 11 | ``` 12 | 13 | ## Example 14 | 15 | Flute Sender python example 16 | 17 | ```python 18 | from flute import sender 19 | 20 | # Flute Sender config parameters 21 | sender_config = sender.Config() 22 | 23 | # Object transmission parameters (no_code => no FEC) 24 | # encoding symbol size : 1400 bytes 25 | # Max source block length : 64 encoding symbols 26 | oti = sender.Oti.new_no_code(1400, 64) 27 | 28 | # Create FLUTE Sender 29 | flute_sender = sender.Sender(1, oti, sender_config) 30 | 31 | # Transfer a file 32 | flute_sender.add_file("/path/to/file", 0, "application/octet-stream", None, None) 33 | flute_sender.publish() 34 | 35 | while True: 36 | alc_pkt = flute_sender.read() 37 | if alc_pkt == None: 38 | break 39 | 40 | #TODO Send alc_pkt over UDP/IP 41 | ``` 42 | 43 | Flute Receiver python example 44 | ```python 45 | from flute import receiver 46 | 47 | # Write received objects to a destination folder 48 | receiver_writer = receiver.ObjectWriterBuilder("/path/to/dest") 49 | 50 | # FLUTE Receiver configuration parameters 51 | receiver_config = receiver.Config() 52 | 53 | tsi = 1 54 | 55 | # Create a FLUTE receiver with the specified endpoint, tsi, writer, and configuration 56 | udp_endpoint = receiver.UDPEndpoint("224.0.0.1", 1234) 57 | flute_receiver = receiver.Receiver(udp_endpoint, tsi, receiver_writer, receiver_config) 58 | 59 | while True: 60 | # Receive LCT/ALC packet from UDP/IP multicast (Implement your own receive_from_udp_socket() function) 61 | # Note: FLUTE does not handle the UDP/IP layer, you need to implement the socket reception mechanism yourself 62 | pkt = receive_from_udp_socket() 63 | 64 | # Push the received packet to the FLUTE receiver 65 | flute_receiver.push(bytes(pkt)) 66 | ``` 67 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | env: 7 | CARGO_TERM_COLOR: always 8 | 9 | permissions: 10 | contents: write 11 | packages: write 12 | 13 | jobs: 14 | release: 15 | name: Create a new Release 16 | runs-on: ubuntu-latest 17 | steps: 18 | 19 | - name: Checkout code 20 | uses: actions/checkout@v4 21 | 22 | - name: Set up environment variables 23 | run: | 24 | VERSION="$(gh release list --json isDraft,tagName --jq 'map(select(.isDraft == true)) | .[0].tagName')" 25 | VERSION="${VERSION#v}" 26 | echo "VERSION=$VERSION" >> $GITHUB_ENV 27 | env: 28 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 29 | 30 | - name: Set up Rust 31 | uses: actions-rs/toolchain@v1 32 | with: 33 | toolchain: stable 34 | 35 | - name: Install cargo-set-version 36 | run: cargo install cargo-set-version 37 | 38 | - name: Install xmllint 39 | run: | 40 | sudo apt update 41 | sudo apt-get install -y libxml2-utils 42 | 43 | - name: Run Tests 44 | uses: actions-rs/cargo@v1 45 | with: 46 | command: test 47 | 48 | - name: Bump to new version 49 | run: | 50 | cargo set-version ${VERSION} 51 | cargo generate-lockfile 52 | git config user.name "${{ github.actor }}" 53 | git config user.email "${{ github.actor }}@users.noreply.github.com" 54 | git commit Cargo.toml -m "Bump to version ${VERSION}" 55 | git tag -a v${VERSION} -m "Release version $VERSION" 56 | env: 57 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 58 | 59 | - name: Push changes and tag 60 | run: | 61 | git push origin main v${VERSION} 62 | env: 63 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 64 | 65 | - name: Publish release 66 | env: 67 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 68 | run: gh release edit v${{ env.VERSION }} --draft=false 69 | 70 | - name: Repository Dispatch 71 | uses: peter-evans/repository-dispatch@v3 72 | with: 73 | event-type: release 74 | -------------------------------------------------------------------------------- /examples/flute-sender/src/main.rs: -------------------------------------------------------------------------------- 1 | use flute::{ 2 | core::lct::Cenc, 3 | core::UDPEndpoint, 4 | sender::{ObjectDesc, Sender}, 5 | }; 6 | use std::{net::UdpSocket, time::SystemTime}; 7 | 8 | fn main() { 9 | std::env::set_var("RUST_LOG", "info"); 10 | env_logger::builder().try_init().ok(); 11 | let dest = "224.0.0.1:3400"; 12 | let endpoint = UDPEndpoint::new(None, "224.0.0.1".to_owned(), 3400); 13 | 14 | let args: Vec = std::env::args().collect(); 15 | if args.len() == 1 { 16 | println!("Send a list of files over UDP/FLUTE to {}", dest); 17 | println!("Usage: {} path/to/file1 path/to/file2 ...", args[0]); 18 | std::process::exit(0); 19 | } 20 | 21 | log::info!("Create UDP Socket"); 22 | 23 | let udp_socket = UdpSocket::bind("0.0.0.0:0").unwrap(); 24 | 25 | log::info!("Create FLUTE Sender"); 26 | let tsi = 1; 27 | let mut sender = Sender::new(endpoint, tsi, &Default::default(), &Default::default()); 28 | 29 | log::info!("Connect to {}", dest); 30 | udp_socket.connect(dest).expect("Connection failed"); 31 | 32 | for file in &args[1..] { 33 | let path = std::path::Path::new(file); 34 | 35 | if !path.is_file() { 36 | log::error!("{} is not a file", file); 37 | std::process::exit(-1); 38 | } 39 | 40 | log::info!("Insert file {} to FLUTE sender", file); 41 | let obj = ObjectDesc::create_from_file( 42 | path, 43 | None, 44 | "application/octet-stream", 45 | true, 46 | 1, 47 | None, 48 | None, 49 | None, 50 | None, 51 | Cenc::Null, 52 | true, 53 | None, 54 | true, 55 | ) 56 | .unwrap(); 57 | sender.add_object(0, obj).expect("Add object failed"); 58 | } 59 | 60 | log::info!("Publish FDT update"); 61 | sender.publish(SystemTime::now()).expect("Publish failed"); 62 | 63 | // Send a "close session" packet to notify the receiver that the 64 | // previous session should be terminated, a new one is about to start. 65 | let close_session_pkt = sender.read_close_session(SystemTime::now()); 66 | udp_socket.send(&close_session_pkt).expect("Send failed"); 67 | 68 | while let Some(pkt) = sender.read(SystemTime::now()) { 69 | udp_socket.send(&pkt).expect("Send failed"); 70 | std::thread::sleep(std::time::Duration::from_millis(1)); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/py/sender/config.rs: -------------------------------------------------------------------------------- 1 | use pyo3::{exceptions::PyTypeError, prelude::*}; 2 | 3 | #[pyclass] 4 | #[derive(Debug)] 5 | pub struct Config(pub crate::sender::Config); 6 | 7 | #[pymethods] 8 | impl Config { 9 | #[new] 10 | pub fn new() -> Self { 11 | Self { 12 | 0: crate::sender::Config { 13 | ..Default::default() 14 | }, 15 | } 16 | } 17 | 18 | #[getter] 19 | pub fn get_fdt_duration_ms(&self) -> PyResult { 20 | Ok(self.0.fdt_duration.as_millis() as u64) 21 | } 22 | 23 | #[setter] 24 | pub fn set_fdt_duration_ms(&mut self, value: u64) -> PyResult<()> { 25 | self.0.fdt_duration = std::time::Duration::from_millis(value); 26 | Ok(()) 27 | } 28 | 29 | #[getter] 30 | pub fn get_fdt_start_id(&self) -> PyResult { 31 | Ok(self.0.fdt_start_id) 32 | } 33 | 34 | #[setter] 35 | pub fn set_fdt_start_id(&mut self, value: u32) -> PyResult<()> { 36 | self.0.fdt_start_id = value; 37 | Ok(()) 38 | } 39 | 40 | #[getter] 41 | pub fn get_fdt_cenc(&self) -> PyResult { 42 | Ok(self.0.fdt_cenc as u8) 43 | } 44 | 45 | #[setter] 46 | pub fn set_fdt_cenc(&mut self, value: u8) -> PyResult<()> { 47 | let cenc = match crate::core::lct::Cenc::try_from(value) { 48 | Ok(res) => res, 49 | Err(_) => return Err(PyTypeError::new_err("Wrong CENC parameter")), 50 | }; 51 | 52 | self.0.fdt_cenc = cenc; 53 | Ok(()) 54 | } 55 | 56 | #[getter] 57 | pub fn get_fdt_inband_sct(&self) -> PyResult { 58 | Ok(self.0.fdt_inband_sct) 59 | } 60 | 61 | #[setter] 62 | pub fn set_fdt_inband_sct(&mut self, value: bool) -> PyResult<()> { 63 | self.0.fdt_inband_sct = value; 64 | Ok(()) 65 | } 66 | 67 | #[getter] 68 | pub fn get_multiplex_files(&self) -> PyResult { 69 | Ok(self.0.priority_queues.get(&0).unwrap().multiplex_files) 70 | } 71 | 72 | #[setter] 73 | pub fn set_multiplex_files(&mut self, value: u32) -> PyResult<()> { 74 | self.0.priority_queues.get_mut(&0).unwrap().multiplex_files = value; 75 | Ok(()) 76 | } 77 | 78 | #[getter] 79 | pub fn get_interleave_blocks(&self) -> PyResult { 80 | Ok(self.0.interleave_blocks) 81 | } 82 | 83 | #[setter] 84 | pub fn set_interleave_blocks(&mut self, value: u8) -> PyResult<()> { 85 | self.0.interleave_blocks = value; 86 | Ok(()) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: PyPi 2 | 3 | on: 4 | repository_dispatch: 5 | types: [release] 6 | 7 | 8 | jobs: 9 | linux: 10 | name: "Linux" 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | target: [x86_64] 15 | steps: 16 | - uses: actions/checkout@v3 17 | - uses: actions/setup-python@v4 18 | with: 19 | python-version: "3.11" 20 | architecture: x64 21 | - uses: actions-rs/toolchain@v1 22 | with: 23 | toolchain: stable 24 | - name: Build wheels 25 | uses: PyO3/maturin-action@v1 26 | with: 27 | target: ${{ matrix.target }} 28 | manylinux: auto 29 | args: --all-features --release --out dist -i 3.7 3.8 3.9 3.10 3.11 pypy3.9 30 | - name: Upload wheels 31 | uses: actions/upload-artifact@v4 32 | with: 33 | name: wheels 34 | path: dist 35 | 36 | #windows: 37 | # runs-on: windows-latest 38 | # name: windows (${{ matrix.platform.target }}) 39 | # strategy: 40 | # matrix: 41 | # platform: 42 | # - target: x64 43 | # interpreter: 3.7 3.8 3.9 3.10 3.11 pypy3.8 pypy3.9 44 | # steps: 45 | # - uses: actions/checkout@v3 46 | # - uses: actions/setup-python@v4 47 | # with: 48 | # python-version: '3.11' 49 | # architecture: ${{ matrix.platform.target }} 50 | # - uses: actions-rs/toolchain@v1 51 | # with: 52 | # toolchain: stable 53 | # - name: Build wheels 54 | # uses: PyO3/maturin-action@v1 55 | # with: 56 | # target: ${{ matrix.platform.target }} 57 | # args: --all-features --release --out dist -i ${{ matrix.platform.interpreter }} 58 | # - name: Upload wheels 59 | # uses: actions/upload-artifact@v3 60 | # with: 61 | # name: wheels 62 | # path: dist 63 | 64 | release: 65 | name: Release 66 | runs-on: ubuntu-latest 67 | environment: production 68 | # if: "startsWith(github.ref, 'refs/tags/')" 69 | needs: [ linux ] 70 | steps: 71 | - name: Download wheels 72 | uses: actions/download-artifact@v4 73 | with: 74 | name: wheels 75 | - uses: actions/setup-python@v4 76 | with: 77 | python-version: "3.11" 78 | - name: Publish to PyPI 79 | env: 80 | TWINE_USERNAME: __token__ 81 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 82 | run: | 83 | pip install --upgrade twine 84 | twine upload --verbose --skip-existing * 85 | -------------------------------------------------------------------------------- /src/fec/raptor.rs: -------------------------------------------------------------------------------- 1 | use super::{DataFecShard, FecDecoder, FecEncoder, FecShard}; 2 | use crate::error::{FluteError, Result}; 3 | 4 | pub struct RaptorEncoder { 5 | nb_parity_symbols: usize, 6 | nb_source_symbols: usize, 7 | } 8 | 9 | impl RaptorEncoder { 10 | pub fn new(nb_source_symbols: usize, nb_parity_symbols: usize) -> RaptorEncoder { 11 | RaptorEncoder { 12 | nb_parity_symbols, 13 | nb_source_symbols, 14 | } 15 | } 16 | } 17 | 18 | impl FecEncoder for RaptorEncoder { 19 | fn encode(&self, data: &[u8]) -> Result>> { 20 | let mut encoder = raptor_code::SourceBlockEncoder::new(data, self.nb_source_symbols) 21 | .map_err(|_| FluteError::new("Fail to create Raptor codec"))?; 22 | let nb_source_symbols = encoder.nb_source_symbols() as usize; 23 | let n = nb_source_symbols + self.nb_parity_symbols; 24 | 25 | let mut output: Vec> = Vec::new(); 26 | 27 | for esi in 0..n { 28 | let shard = DataFecShard { 29 | shard: encoder.fountain(esi as u32), 30 | index: esi as u32, 31 | }; 32 | log::info!("Encode shard {}", shard.shard.len()); 33 | output.push(Box::new(shard)); 34 | } 35 | 36 | Ok(output) 37 | } 38 | } 39 | 40 | pub struct RaptorDecoder { 41 | source_block_size: usize, 42 | decoder: raptor_code::SourceBlockDecoder, 43 | data: Option>, 44 | } 45 | 46 | impl RaptorDecoder { 47 | pub fn new(nb_source_symbols: usize, source_block_size: usize) -> RaptorDecoder { 48 | log::info!( 49 | "new RaptorDecoder nb_source_symbols={} source_block_size={}", 50 | nb_source_symbols, 51 | source_block_size 52 | ); 53 | RaptorDecoder { 54 | decoder: raptor_code::SourceBlockDecoder::new(nb_source_symbols), 55 | source_block_size, 56 | data: None, 57 | } 58 | } 59 | } 60 | 61 | impl FecDecoder for RaptorDecoder { 62 | fn push_symbol(&mut self, encoding_symbol: &[u8], esi: u32) { 63 | if self.data.is_some() { 64 | return; 65 | } 66 | 67 | log::info!( 68 | "encoding symbol length={} source_block_size={}", 69 | encoding_symbol.len(), 70 | self.source_block_size 71 | ); 72 | 73 | self.decoder.push_encoding_symbol(encoding_symbol, esi) 74 | } 75 | 76 | fn can_decode(&self) -> bool { 77 | self.decoder.fully_specified() 78 | } 79 | 80 | fn decode(&mut self) -> bool { 81 | log::debug!("Decode source block length {}", self.source_block_size); 82 | self.data = self.decoder.decode(self.source_block_size); 83 | self.data.is_some() 84 | } 85 | 86 | fn source_block(&self) -> Result<&[u8]> { 87 | match self.data.as_ref() { 88 | Some(e) => Ok(e), 89 | None => Err(FluteError::new("Block not decoded")), 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/sender/objectsenderlogger.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use opentelemetry::{ 4 | global::{self}, 5 | propagation::Extractor, 6 | trace::{Span, SpanKind, TraceContextExt, Tracer}, 7 | Context, KeyValue, 8 | }; 9 | 10 | use crate::common::udpendpoint::UDPEndpoint; 11 | 12 | pub struct ObjectSenderLogger { 13 | _cx: Context, 14 | } 15 | 16 | impl std::fmt::Debug for ObjectSenderLogger { 17 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 18 | f.debug_struct("ObjectSenderLogger").finish() 19 | } 20 | } 21 | 22 | struct HeaderExtractor<'a>(pub &'a HashMap); 23 | impl Extractor for HeaderExtractor<'_> { 24 | fn get(&self, key: &str) -> Option<&str> { 25 | self.0.get(key).map(|s| s.as_str()) 26 | } 27 | 28 | fn keys(&self) -> Vec<&str> { 29 | self.0.keys().map(|s| s.as_str()).collect() 30 | } 31 | } 32 | 33 | impl ObjectSenderLogger { 34 | fn extract_context_from_propagator(req: &HashMap) -> Context { 35 | global::get_text_map_propagator(|propagator| propagator.extract(&HeaderExtractor(req))) 36 | } 37 | 38 | pub fn new( 39 | endpoint: &UDPEndpoint, 40 | tsi: u64, 41 | toi: u128, 42 | propagator: Option<&HashMap>, 43 | ) -> Self { 44 | let tracer = global::tracer("FluteLogger"); 45 | let name = match toi { 46 | 0 => "FDT Transfer", 47 | _ => "Object Transfer", 48 | }; 49 | 50 | let mut span; 51 | if let Some(propagator) = propagator { 52 | let parent_cx = Self::extract_context_from_propagator(propagator); 53 | span = tracer 54 | .span_builder(name) 55 | .with_kind(SpanKind::Producer) 56 | .start_with_context(&tracer, &parent_cx) 57 | } else { 58 | span = tracer 59 | .span_builder(name) 60 | .with_kind(SpanKind::Producer) 61 | .start(&tracer); 62 | } 63 | 64 | span.set_attribute(KeyValue::new( 65 | opentelemetry_semantic_conventions::attribute::NETWORK_TRANSPORT, 66 | "flute", 67 | )); 68 | 69 | span.set_attribute(KeyValue::new( 70 | opentelemetry_semantic_conventions::attribute::NETWORK_PEER_ADDRESS, 71 | endpoint.destination_group_address.clone(), 72 | )); 73 | 74 | span.set_attribute(KeyValue::new( 75 | opentelemetry_semantic_conventions::attribute::NETWORK_PEER_PORT, 76 | endpoint.port as i64, 77 | )); 78 | 79 | if let Some(source_address) = endpoint.source_address.as_ref() { 80 | span.set_attribute(KeyValue::new( 81 | opentelemetry_semantic_conventions::attribute::NETWORK_LOCAL_ADDRESS, 82 | source_address.clone(), 83 | )); 84 | } 85 | 86 | span.set_attribute(KeyValue::new("flute.toi", toi.to_string())); 87 | span.set_attribute(KeyValue::new("flute.tsi", tsi.to_string())); 88 | 89 | let cx = Context::current_with_span(span); 90 | Self { _cx: cx } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/receiver/uncompress.rs: -------------------------------------------------------------------------------- 1 | use crate::tools::ringbuffer::RingBuffer; 2 | use flate2::read::{DeflateDecoder, GzDecoder, ZlibDecoder}; 3 | use std::io::Read; 4 | use std::io::Write; 5 | 6 | pub trait Decompress { 7 | fn write(&mut self, data: &[u8]) -> std::io::Result; 8 | fn read(&mut self, data: &mut [u8]) -> std::io::Result; 9 | fn finish(&mut self); 10 | } 11 | 12 | impl std::fmt::Debug for dyn Decompress { 13 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 14 | write!(f, "Decompress {{ }}") 15 | } 16 | } 17 | 18 | pub struct DecompressGzip { 19 | decoder: GzDecoder, 20 | } 21 | 22 | impl DecompressGzip { 23 | pub fn new(pkt: &[u8]) -> DecompressGzip { 24 | let mut ring = RingBuffer::new(pkt.len() * 2); 25 | let result = ring.write(pkt).unwrap(); 26 | debug_assert!(result == pkt.len()); 27 | DecompressGzip { 28 | decoder: GzDecoder::new(ring), 29 | } 30 | } 31 | } 32 | 33 | impl Decompress for DecompressGzip { 34 | fn write(&mut self, data: &[u8]) -> std::io::Result { 35 | self.decoder.get_mut().write(data) 36 | } 37 | 38 | fn read(&mut self, data: &mut [u8]) -> std::io::Result { 39 | self.decoder.read(data) 40 | } 41 | fn finish(&mut self) { 42 | self.decoder.get_mut().finish(); 43 | } 44 | } 45 | 46 | pub struct DecompressDeflate { 47 | decoder: DeflateDecoder, 48 | } 49 | 50 | impl DecompressDeflate { 51 | pub fn new(pkt: &[u8]) -> DecompressDeflate { 52 | let mut ring = RingBuffer::new(pkt.len() * 2); 53 | let result = ring.write(pkt).unwrap(); 54 | debug_assert!(result == pkt.len()); 55 | DecompressDeflate { 56 | decoder: DeflateDecoder::new(ring), 57 | } 58 | } 59 | } 60 | 61 | impl Decompress for DecompressDeflate { 62 | fn write(&mut self, data: &[u8]) -> std::io::Result { 63 | self.decoder.get_mut().write(data) 64 | } 65 | 66 | fn read(&mut self, data: &mut [u8]) -> std::io::Result { 67 | self.decoder.read(data) 68 | } 69 | fn finish(&mut self) { 70 | self.decoder.get_mut().finish(); 71 | } 72 | } 73 | 74 | pub struct DecompressZlib { 75 | decoder: ZlibDecoder, 76 | } 77 | 78 | impl DecompressZlib { 79 | pub fn new(pkt: &[u8]) -> DecompressZlib { 80 | let mut ring = RingBuffer::new(pkt.len() * 2); 81 | let result = ring.write(pkt).unwrap(); 82 | debug_assert!(result == pkt.len()); 83 | DecompressZlib { 84 | decoder: ZlibDecoder::new(ring), 85 | } 86 | } 87 | } 88 | 89 | impl Decompress for DecompressZlib { 90 | fn write(&mut self, data: &[u8]) -> std::io::Result { 91 | self.decoder.get_mut().write(data) 92 | } 93 | 94 | fn read(&mut self, data: &mut [u8]) -> std::io::Result { 95 | self.decoder.read(data) 96 | } 97 | fn finish(&mut self) { 98 | self.decoder.get_mut().finish(); 99 | } 100 | } 101 | 102 | #[cfg(test)] 103 | mod tests { 104 | #[test] 105 | pub fn test_gzip() {} 106 | } 107 | -------------------------------------------------------------------------------- /src/receiver/tsifilter.rs: -------------------------------------------------------------------------------- 1 | use crate::common::udpendpoint::UDPEndpoint; 2 | 3 | struct TSI { 4 | endpoints: std::collections::HashMap, 5 | } 6 | 7 | pub struct TSIFilter { 8 | tsi: std::collections::HashMap, 9 | endpoint_bypass: std::collections::HashMap, 10 | } 11 | 12 | impl std::fmt::Debug for TSIFilter { 13 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 14 | write!(f, "FecEncoder {{ }}") 15 | } 16 | } 17 | 18 | impl TSIFilter { 19 | pub fn new() -> Self { 20 | TSIFilter { 21 | tsi: std::collections::HashMap::new(), 22 | endpoint_bypass: std::collections::HashMap::new(), 23 | } 24 | } 25 | 26 | pub fn add_endpoint_bypass(&mut self, endpoint: UDPEndpoint) { 27 | match self.endpoint_bypass.get_mut(&endpoint) { 28 | Some(bypass) => *bypass += 1, 29 | None => { 30 | self.endpoint_bypass.insert(endpoint, 1); 31 | } 32 | } 33 | } 34 | 35 | pub fn remove_endpoint_bypass(&mut self, endpoint: &UDPEndpoint) { 36 | if let Some(bypass) = self.endpoint_bypass.get_mut(endpoint) { 37 | if *bypass > 1 { 38 | *bypass -= 1 39 | } else { 40 | self.endpoint_bypass.remove(endpoint); 41 | } 42 | } 43 | } 44 | 45 | pub fn add(&mut self, endpoint: UDPEndpoint, tsi: u64) { 46 | match self.tsi.get_mut(&tsi) { 47 | Some(tsi) => tsi.add(endpoint), 48 | None => { 49 | self.tsi.insert(tsi, TSI::new(endpoint)); 50 | } 51 | } 52 | } 53 | 54 | pub fn remove(&mut self, endpoint: &UDPEndpoint, tsi: u64) { 55 | if let Some(t) = self.tsi.get_mut(&tsi) { 56 | t.remove(endpoint); 57 | if t.is_empty() { 58 | self.tsi.remove(&tsi); 59 | } 60 | } 61 | } 62 | 63 | pub fn is_valid(&self, endpoint: &UDPEndpoint, tsi: u64) -> bool { 64 | if self.endpoint_bypass.contains_key(endpoint) { 65 | return true; 66 | } 67 | 68 | if let Some(t) = self.tsi.get(&tsi) { 69 | return t.is_valid(endpoint); 70 | } 71 | 72 | false 73 | } 74 | } 75 | 76 | impl TSI { 77 | fn new(endpoint: UDPEndpoint) -> Self { 78 | let mut output = Self { 79 | endpoints: std::collections::HashMap::new(), 80 | }; 81 | output.endpoints.insert(endpoint, 1); 82 | output 83 | } 84 | 85 | fn add(&mut self, endpoint: UDPEndpoint) { 86 | match self.endpoints.get_mut(&endpoint) { 87 | Some(a) => *a += 1, 88 | None => { 89 | self.endpoints.insert(endpoint.clone(), 1); 90 | } 91 | } 92 | } 93 | 94 | fn remove(&mut self, endpoint: &UDPEndpoint) { 95 | if let Some(v) = self.endpoints.get_mut(endpoint) { 96 | if *v > 1 { 97 | *v -= 1; 98 | } else { 99 | self.endpoints.remove(endpoint); 100 | } 101 | } 102 | } 103 | 104 | fn is_empty(&self) -> bool { 105 | self.endpoints.is_empty() 106 | } 107 | 108 | fn is_valid(&self, endpoint: &UDPEndpoint) -> bool { 109 | if self.endpoints.contains_key(endpoint) { 110 | return true; 111 | } 112 | 113 | let mut endpoint_no_src = endpoint.clone(); 114 | endpoint_no_src.source_address = None; 115 | self.endpoints.contains_key(&endpoint_no_src) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/common/partition.rs: -------------------------------------------------------------------------------- 1 | /// 2 | /// Block Partitioning Algorithm 3 | /// See 4 | /// 5 | /// 6 | /// This function implements the block partitioning algorithm as defined in RFC 5052. 7 | /// The algorithm is used to partition a large amount of data into smaller blocks that can be transmitted or encoded more efficiently. 8 | /// 9 | /// # Arguments 10 | /// 11 | /// * b: Maximum Source Block Length, i.e., the maximum number of source symbols per source block. 12 | /// 13 | /// * l: Transfer Length in octets. 14 | /// 15 | /// * e: Encoding Symbol Length in octets. 16 | /// 17 | /// # Returns 18 | /// 19 | /// The function returns a tuple of four values: 20 | /// * a_large: The length of each of the larger source blocks in symbols. 21 | /// * a_small: The length of each of the smaller source blocks in symbols. 22 | /// * nb_a_large: The number of blocks composed of a_large symbols. 23 | /// * nb_blocks: The total number of blocks. 24 | /// 25 | pub fn block_partitioning(b: u64, l: u64, e: u64) -> (u64, u64, u64, u64) { 26 | if b == 0 { 27 | log::warn!("Maximum Source Block Length is 0"); 28 | return (0, 0, 0, 0); 29 | } 30 | 31 | if e == 0 { 32 | log::error!("Encoding Symbol Length is 0"); 33 | return (0, 0, 0, 0); 34 | } 35 | 36 | let t = num_integer::div_ceil(l, e); 37 | let n = num_integer::div_ceil(t, b); 38 | log::debug!("t={} n={} b={} l={} e={}", t, n, b, l, e); 39 | if n == 0 { 40 | return (0, 0, 0, 0); 41 | } 42 | 43 | let a_large = num_integer::div_ceil(t, n); 44 | let a_small = num_integer::div_floor(t, n); 45 | let nb_a_large = t - (a_small * n); 46 | let nb_blocks = n; 47 | 48 | (a_large, a_small, nb_a_large, nb_blocks) 49 | } 50 | 51 | /// Calculates the size of a block in octets. 52 | /// 53 | /// # Arguments 54 | /// 55 | /// * `a_large`: The length of each of the larger source blocks in symbols. 56 | /// * `a_small`: The length of each of the smaller source blocks in symbols. 57 | /// * `nb_a_large`: The number of blocks composed of `a_large` symbols. 58 | /// * `l`: Transfer length in octets. 59 | /// * `e`: Encoding symbol length in octets. 60 | /// * `sbn`: Source block number. 61 | /// 62 | /// # Returns 63 | /// 64 | /// The size of the block in octets. 65 | /// 66 | pub fn block_length(a_large: u64, a_small: u64, nb_a_large: u64, l: u64, e: u64, sbn: u32) -> u64 { 67 | let sbn = sbn as u64; 68 | 69 | let large_block_size = a_large * e; 70 | let small_block_size = a_small * e; 71 | 72 | if sbn + 1 < nb_a_large { 73 | return large_block_size; 74 | } 75 | 76 | if sbn + 1 == nb_a_large { 77 | let large_size = nb_a_large * large_block_size; 78 | if large_size <= l { 79 | return large_block_size; 80 | } 81 | 82 | // Should never happen ? 83 | return l - ((nb_a_large - 1) * large_block_size); 84 | } 85 | 86 | let l = l - (nb_a_large * large_block_size); 87 | let sbn = sbn - nb_a_large; 88 | let small_size = (sbn + 1) * small_block_size; 89 | if small_size <= l { 90 | return small_block_size; 91 | } 92 | 93 | l - (sbn * small_block_size) 94 | } 95 | 96 | #[cfg(test)] 97 | mod tests { 98 | 99 | #[test] 100 | pub fn partition_empty_file() { 101 | crate::tests::init(); 102 | let (a_large, a_small, nb_a_large, nb_blocks) = super::block_partitioning(64, 0, 1024); 103 | log::info!( 104 | "a_large={} a_small={} nb_a_large={} nb_blocks={}", 105 | a_large, 106 | a_small, 107 | nb_a_large, 108 | nb_blocks 109 | ); 110 | assert!(nb_blocks == 0); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/py/sender/senderpy.rs: -------------------------------------------------------------------------------- 1 | use pyo3::exceptions::PyTypeError; 2 | use pyo3::prelude::*; 3 | use std::time::SystemTime; 4 | 5 | use super::config; 6 | use super::oti; 7 | 8 | #[pyclass] 9 | #[derive(Debug)] 10 | pub struct Sender(crate::sender::Sender); 11 | 12 | #[pymethods] 13 | impl Sender { 14 | #[new] 15 | pub fn new(tsi: u64, oti: &oti::Oti, config: &config::Config) -> Self { 16 | Self { 17 | 0: crate::sender::Sender::new( 18 | crate::core::UDPEndpoint::new(None, "224.0.0.1".to_owned(), 0), // FIXME 19 | tsi, 20 | &oti.0, 21 | &config.0, 22 | ), 23 | } 24 | } 25 | 26 | #[pyo3(signature = (content, content_type, content_location, oti=None))] 27 | fn add_object_from_buffer( 28 | &mut self, 29 | content: &[u8], 30 | content_type: &str, 31 | content_location: &str, 32 | oti: Option<&oti::Oti>, 33 | ) -> PyResult { 34 | let content_location = 35 | url::Url::parse(content_location).map_err(|e| PyTypeError::new_err(e.to_string()))?; 36 | 37 | let oti = oti.map(|o| o.0.clone()); 38 | let object = crate::sender::ObjectDesc::create_from_buffer( 39 | content.to_vec(), 40 | content_type, 41 | &content_location, 42 | 1, 43 | None, 44 | None, 45 | None, 46 | None, 47 | crate::core::lct::Cenc::Null, 48 | true, 49 | oti, 50 | true, 51 | ) 52 | .map_err(|e| PyTypeError::new_err(e.0.to_string()))?; 53 | 54 | self.0 55 | .add_object(0, object) 56 | .map_err(|e| PyTypeError::new_err(e.0.to_string())) 57 | } 58 | 59 | #[pyo3(signature = (filepath, cenc, content_type, content_location=None, oti=None))] 60 | fn add_file( 61 | &mut self, 62 | filepath: &str, 63 | cenc: u8, 64 | content_type: &str, 65 | content_location: Option<&str>, 66 | oti: Option<&oti::Oti>, 67 | ) -> PyResult { 68 | let cenc = cenc 69 | .try_into() 70 | .map_err(|_| PyTypeError::new_err("Unknown cenc"))?; 71 | let content_location = content_location.map(|content_location| { 72 | url::Url::parse(content_location).map_err(|e| PyTypeError::new_err(e.to_string())) 73 | }); 74 | let content_location = match content_location { 75 | Some(Err(e)) => return Err(PyTypeError::new_err(e)), 76 | Some(Ok(url)) => Some(url), 77 | None => None, 78 | }; 79 | 80 | let oti = oti.map(|o| o.0.clone()); 81 | let object = crate::sender::ObjectDesc::create_from_file( 82 | std::path::Path::new(filepath), 83 | content_location.as_ref(), 84 | content_type, 85 | true, 86 | 1, 87 | None, 88 | None, 89 | None, 90 | None, 91 | cenc, 92 | true, 93 | oti, 94 | true, 95 | ) 96 | .map_err(|e| PyTypeError::new_err(e.0.to_string()))?; 97 | 98 | self.0 99 | .add_object(0, object) 100 | .map_err(|e| PyTypeError::new_err(e.0.to_string())) 101 | } 102 | 103 | fn remove_object(&mut self, toi: u128) -> bool { 104 | self.0.remove_object(toi) 105 | } 106 | 107 | fn nb_objects(&self) -> usize { 108 | self.0.nb_objects() 109 | } 110 | 111 | fn publish(&mut self) -> PyResult<()> { 112 | self.0 113 | .publish(SystemTime::now()) 114 | .map_err(|e| PyTypeError::new_err(e.0.to_string())) 115 | } 116 | 117 | fn read(&mut self) -> PyResult>> { 118 | Ok(self.0.read(SystemTime::now())) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/sender/toiallocator.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, Mutex}; 2 | 3 | use rand::Rng; 4 | 5 | use crate::common::lct; 6 | 7 | use super::TOIMaxLength; 8 | 9 | #[derive(Debug)] 10 | struct ToiAllocatorInternal { 11 | toi_reserved: std::collections::HashSet, 12 | toi: u128, 13 | toi_max_length: TOIMaxLength, 14 | } 15 | 16 | #[derive(Debug)] 17 | pub struct ToiAllocator { 18 | internal: Mutex, 19 | } 20 | 21 | /// Struct containing a TOI 22 | #[derive(Debug)] 23 | pub struct Toi { 24 | allocator: Arc, 25 | value: u128, 26 | } 27 | 28 | impl Drop for Toi { 29 | fn drop(&mut self) { 30 | self.allocator.release(self.value); 31 | } 32 | } 33 | 34 | impl Toi { 35 | /// Get Value of TOI 36 | pub fn get(&self) -> u128 { 37 | self.value 38 | } 39 | } 40 | 41 | impl ToiAllocatorInternal { 42 | fn new(toi_max_length: TOIMaxLength, toi_initial_value: Option) -> Self { 43 | let mut toi = match toi_initial_value { 44 | Some(0) => 1, 45 | Some(n) => n, 46 | None => { 47 | let mut rng = rand::rng(); 48 | rng.random() 49 | } 50 | }; 51 | 52 | toi = Self::to_max_length(toi, toi_max_length); 53 | if toi == lct::TOI_FDT { 54 | toi += 1; 55 | } 56 | 57 | Self { 58 | toi_reserved: std::collections::HashSet::new(), 59 | toi, 60 | toi_max_length, 61 | } 62 | } 63 | 64 | fn to_max_length(toi: u128, toi_max_length: TOIMaxLength) -> u128 { 65 | match toi_max_length { 66 | TOIMaxLength::ToiMax16 => toi & 0xFFFFu128, 67 | TOIMaxLength::ToiMax32 => toi & 0xFFFFFFFFu128, 68 | TOIMaxLength::ToiMax48 => toi & 0xFFFFFFFFFFFFu128, 69 | TOIMaxLength::ToiMax64 => toi & 0xFFFFFFFFFFFFFFFFu128, 70 | TOIMaxLength::ToiMax80 => toi & 0xFFFFFFFFFFFFFFFFFFFFu128, 71 | TOIMaxLength::ToiMax112 => toi, 72 | } 73 | } 74 | 75 | fn allocate(&mut self) -> u128 { 76 | let ret = self.toi; 77 | assert!(!self.toi_reserved.contains(&ret)); 78 | self.toi_reserved.insert(ret); 79 | 80 | loop { 81 | self.toi = Self::to_max_length(self.toi + 1, self.toi_max_length); 82 | if self.toi == lct::TOI_FDT { 83 | self.toi = 1; 84 | } 85 | 86 | if !self.toi_reserved.contains(&self.toi) { 87 | break; 88 | } 89 | 90 | log::warn!("TOI {} is already used by a file or reserved", self.toi) 91 | } 92 | ret 93 | } 94 | 95 | fn release(&mut self, toi: u128) { 96 | let success = self.toi_reserved.remove(&toi); 97 | debug_assert!(success); 98 | } 99 | } 100 | 101 | impl ToiAllocator { 102 | pub fn new(toi_max_length: TOIMaxLength, toi_initial_value: Option) -> Arc { 103 | Arc::new(Self { 104 | internal: Mutex::new(ToiAllocatorInternal::new(toi_max_length, toi_initial_value)), 105 | }) 106 | } 107 | 108 | pub fn allocate(allocator: &Arc) -> Box { 109 | let mut db = allocator.internal.lock().unwrap(); 110 | let toi = db.allocate(); 111 | Box::new(Toi { 112 | allocator: allocator.clone(), 113 | value: toi, 114 | }) 115 | } 116 | 117 | pub fn allocate_toi_fdt(allocator: &Arc) -> Box { 118 | Box::new(Toi { 119 | allocator: allocator.clone(), 120 | value: 0, 121 | }) 122 | } 123 | 124 | pub fn release(&self, toi: u128) { 125 | if toi == lct::TOI_FDT { 126 | return; 127 | } 128 | { 129 | let mut db = self.internal.lock().unwrap(); 130 | db.release(toi); 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/py/sender/oti.rs: -------------------------------------------------------------------------------- 1 | use pyo3::{exceptions::PyTypeError, prelude::*}; 2 | 3 | #[pyclass] 4 | #[derive(Debug)] 5 | pub struct Oti(pub crate::core::Oti); 6 | 7 | #[pymethods] 8 | impl Oti { 9 | #[new] 10 | pub fn new() -> Self { 11 | Self { 12 | 0: Default::default(), 13 | } 14 | } 15 | 16 | #[staticmethod] 17 | fn new_no_code( 18 | encoding_symbol_length: u16, 19 | maximum_source_block_length: u16, 20 | ) -> PyResult { 21 | Ok(Self { 22 | 0: crate::core::Oti::new_no_code(encoding_symbol_length, maximum_source_block_length), 23 | }) 24 | } 25 | 26 | #[staticmethod] 27 | fn new_reed_solomon_rs28( 28 | encoding_symbol_length: u16, 29 | maximum_source_block_length: u8, 30 | max_number_of_parity_symbols: u8, 31 | ) -> PyResult { 32 | let oti = crate::core::Oti::new_reed_solomon_rs28( 33 | encoding_symbol_length, 34 | maximum_source_block_length, 35 | max_number_of_parity_symbols, 36 | ) 37 | .map_err(|e| PyTypeError::new_err(e.0.to_string()))?; 38 | Ok(Self { 0: oti }) 39 | } 40 | 41 | #[staticmethod] 42 | fn new_reed_solomon_rs28_under_specified( 43 | encoding_symbol_length: u16, 44 | maximum_source_block_length: u16, 45 | max_number_of_parity_symbols: u16, 46 | ) -> PyResult { 47 | let oti = crate::core::Oti::new_reed_solomon_rs28_under_specified( 48 | encoding_symbol_length, 49 | maximum_source_block_length, 50 | max_number_of_parity_symbols, 51 | ) 52 | .map_err(|e| PyTypeError::new_err(e.0.to_string()))?; 53 | Ok(Self { 0: oti }) 54 | } 55 | 56 | #[getter] 57 | fn get_max_transfer_length(&self) -> PyResult { 58 | Ok(self.0.max_transfer_length()) 59 | } 60 | 61 | #[getter] 62 | fn get_fec_encoding_id(&self) -> PyResult { 63 | Ok(self.0.fec_encoding_id as u8) 64 | } 65 | 66 | #[setter] 67 | fn set_fec_encoding_id(&mut self, value: u8) -> PyResult<()> { 68 | let encoding_id: crate::core::FECEncodingID = value 69 | .try_into() 70 | .map_err(|_| PyTypeError::new_err("Invalid FEC Encoding ID"))?; 71 | self.0.fec_encoding_id = encoding_id; 72 | Ok(()) 73 | } 74 | 75 | #[getter] 76 | fn get_fec_instance_id(&self) -> PyResult { 77 | Ok(self.0.fec_instance_id) 78 | } 79 | 80 | #[setter] 81 | fn set_fec_instance_id(&mut self, value: u16) -> PyResult<()> { 82 | self.0.fec_instance_id = value; 83 | Ok(()) 84 | } 85 | 86 | #[getter] 87 | fn get_maximum_source_block_length(&self) -> PyResult { 88 | Ok(self.0.maximum_source_block_length) 89 | } 90 | 91 | #[setter] 92 | fn set_maximum_source_block_length(&mut self, value: u32) -> PyResult<()> { 93 | self.0.maximum_source_block_length = value; 94 | Ok(()) 95 | } 96 | 97 | #[getter] 98 | fn get_encoding_symbol_length(&self) -> PyResult { 99 | Ok(self.0.encoding_symbol_length) 100 | } 101 | 102 | #[setter] 103 | fn set_encoding_symbol_length(&mut self, value: u16) -> PyResult<()> { 104 | self.0.encoding_symbol_length = value; 105 | Ok(()) 106 | } 107 | 108 | #[getter] 109 | fn get_max_number_of_parity_symbols(&self) -> PyResult { 110 | Ok(self.0.encoding_symbol_length) 111 | } 112 | 113 | #[setter] 114 | fn set_max_number_of_parity_symbols(&mut self, value: u32) -> PyResult<()> { 115 | self.0.max_number_of_parity_symbols = value; 116 | Ok(()) 117 | } 118 | 119 | #[getter] 120 | fn get_inband_fti(&self) -> PyResult { 121 | Ok(self.0.inband_fti) 122 | } 123 | 124 | #[setter] 125 | fn set_inband_fti(&mut self, value: bool) -> PyResult<()> { 126 | self.0.inband_fti = value; 127 | Ok(()) 128 | } 129 | 130 | // TODO 131 | // reed_solomon_scheme_specific 132 | // raptorq_scheme_specific 133 | } 134 | -------------------------------------------------------------------------------- /src/receiver/blockdecoder.rs: -------------------------------------------------------------------------------- 1 | use crate::common::{ 2 | alc, 3 | oti::{self, SchemeSpecific}, 4 | }; 5 | use crate::error::FluteError; 6 | use crate::fec; 7 | use crate::fec::nocode; 8 | use crate::fec::rscodec; 9 | use crate::fec::FecDecoder; 10 | use crate::tools::error::Result; 11 | 12 | #[derive(Debug)] 13 | pub struct BlockDecoder { 14 | pub completed: bool, 15 | pub initialized: bool, 16 | pub block_size: usize, 17 | decoder: Option>, 18 | } 19 | 20 | impl BlockDecoder { 21 | pub fn new() -> BlockDecoder { 22 | BlockDecoder { 23 | completed: false, 24 | initialized: false, 25 | decoder: None, 26 | block_size: 0, 27 | } 28 | } 29 | 30 | pub fn init( 31 | &mut self, 32 | oti: &oti::Oti, 33 | nb_source_symbols: u32, 34 | block_size: usize, 35 | sbn: u32, 36 | ) -> Result<()> { 37 | if self.initialized { 38 | return Ok(()); 39 | } 40 | 41 | match oti.fec_encoding_id { 42 | oti::FECEncodingID::NoCode => { 43 | let codec = nocode::NoCodeDecoder::new(nb_source_symbols as usize); 44 | self.decoder = Some(Box::new(codec)); 45 | } 46 | oti::FECEncodingID::ReedSolomonGF28 => { 47 | let codec = rscodec::RSGalois8Codec::new( 48 | nb_source_symbols as usize, 49 | oti.max_number_of_parity_symbols as usize, 50 | oti.encoding_symbol_length as usize, 51 | )?; 52 | self.decoder = Some(Box::new(codec)); 53 | } 54 | oti::FECEncodingID::ReedSolomonGF28UnderSpecified => { 55 | let codec = rscodec::RSGalois8Codec::new( 56 | nb_source_symbols as usize, 57 | oti.max_number_of_parity_symbols as usize, 58 | oti.encoding_symbol_length as usize, 59 | )?; 60 | self.decoder = Some(Box::new(codec)); 61 | } 62 | oti::FECEncodingID::ReedSolomonGF2M => { 63 | log::warn!("Not implemented") 64 | } 65 | oti::FECEncodingID::RaptorQ => { 66 | if let Some(SchemeSpecific::RaptorQ(scheme)) = oti.scheme_specific.as_ref() { 67 | let codec = fec::raptorq::RaptorQDecoder::new( 68 | sbn, 69 | nb_source_symbols as usize, 70 | oti.encoding_symbol_length as usize, 71 | scheme, 72 | ); 73 | self.decoder = Some(Box::new(codec)); 74 | } else { 75 | return Err(FluteError::new("RaptorQ Scheme not found")); 76 | } 77 | } 78 | oti::FECEncodingID::Raptor => { 79 | if oti.scheme_specific.is_none() { 80 | return Err(FluteError::new("Raptor Scheme not found")); 81 | } 82 | 83 | let codec = fec::raptor::RaptorDecoder::new(nb_source_symbols as usize, block_size); 84 | self.decoder = Some(Box::new(codec)); 85 | } 86 | } 87 | 88 | self.initialized = true; 89 | self.block_size = block_size; 90 | Ok(()) 91 | } 92 | 93 | pub fn source_block(&self) -> Result<&[u8]> { 94 | if self.decoder.is_none() { 95 | return Err(FluteError::new("Fail to decode block")); 96 | } 97 | 98 | self.decoder.as_ref().unwrap().source_block() 99 | } 100 | 101 | pub fn deallocate(&mut self) { 102 | self.decoder = None; 103 | self.block_size = 0; 104 | } 105 | 106 | pub fn push(&mut self, pkt: &alc::AlcPkt, payload_id: &alc::PayloadID) { 107 | debug_assert!(self.initialized); 108 | 109 | if self.completed { 110 | return; 111 | } 112 | 113 | debug_assert!(self.decoder.is_some()); 114 | if self.decoder.is_none() { 115 | log::error!("Decoder has been deallocated"); 116 | return; 117 | } 118 | 119 | let payload = &pkt.data[pkt.data_payload_offset..]; 120 | let decoder = self.decoder.as_mut().unwrap(); 121 | decoder.push_symbol(payload, payload_id.esi); 122 | 123 | if decoder.can_decode() { 124 | self.completed = decoder.decode(); 125 | if self.completed { 126 | log::debug!("Block completed"); 127 | } 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/receiver/writer/objectwriterbuffer.rs: -------------------------------------------------------------------------------- 1 | use super::{ObjectMetadata, ObjectWriter, ObjectWriterBuilder, ObjectWriterBuilderResult}; 2 | use crate::{common::udpendpoint::UDPEndpoint, tools::error::Result}; 3 | use std::{cell::RefCell, rc::Rc, time::SystemTime}; 4 | 5 | /// 6 | /// Write objects received by the `receiver` to a buffers 7 | /// 8 | #[derive(Debug)] 9 | pub struct ObjectWriterBufferBuilder { 10 | /// List of all objects received 11 | pub objects: RefCell>>>, 12 | /// True when MD5 check is enabled 13 | pub enable_md5_check: bool, 14 | } 15 | 16 | /// 17 | /// Write a FLUTE object to a buffer 18 | /// 19 | #[derive(Debug)] 20 | struct ObjectWriterBufferWrapper { 21 | inner: Rc>, 22 | enable_md5_check: bool, 23 | } 24 | 25 | #[derive(Debug)] 26 | /// Object stored in a buffer 27 | pub struct ObjectWriterBuffer { 28 | /// true when the object is fully received 29 | pub complete: bool, 30 | /// true when an error occured during the reception 31 | pub error: bool, 32 | /// buffer containing the data of the object 33 | pub data: Vec, 34 | /// Metadata of the object 35 | pub meta: ObjectMetadata, 36 | /// Time when the object reception started 37 | pub start_time: SystemTime, 38 | /// Time when the object reception ended 39 | pub end_time: Option, 40 | } 41 | 42 | impl ObjectWriterBufferBuilder { 43 | /// Return a new `ObjectWriterBuffer` 44 | pub fn new(enable_md5_check: bool) -> ObjectWriterBufferBuilder { 45 | ObjectWriterBufferBuilder { 46 | objects: RefCell::new(Vec::new()), 47 | enable_md5_check, 48 | } 49 | } 50 | } 51 | 52 | impl Default for ObjectWriterBufferBuilder { 53 | fn default() -> Self { 54 | Self::new(true) 55 | } 56 | } 57 | 58 | impl ObjectWriterBuilder for ObjectWriterBufferBuilder { 59 | fn new_object_writer( 60 | &self, 61 | _endpoint: &UDPEndpoint, 62 | _tsi: &u64, 63 | _toi: &u128, 64 | meta: &ObjectMetadata, 65 | now: std::time::SystemTime, 66 | ) -> ObjectWriterBuilderResult { 67 | let obj = Rc::new(RefCell::new(ObjectWriterBuffer { 68 | complete: false, 69 | error: false, 70 | data: Vec::new(), 71 | meta: meta.clone(), 72 | start_time: now, 73 | end_time: None, 74 | })); 75 | 76 | let obj_wrapper = Box::new(ObjectWriterBufferWrapper { 77 | inner: obj.clone(), 78 | enable_md5_check: self.enable_md5_check, 79 | }); 80 | self.objects.borrow_mut().push(obj); 81 | ObjectWriterBuilderResult::StoreObject(obj_wrapper) 82 | } 83 | 84 | fn update_cache_control( 85 | &self, 86 | _endpoint: &UDPEndpoint, 87 | _tsi: &u64, 88 | _toi: &u128, 89 | _meta: &ObjectMetadata, 90 | _now: std::time::SystemTime, 91 | ) { 92 | } 93 | 94 | fn fdt_received( 95 | &self, 96 | _endpoint: &UDPEndpoint, 97 | _tsi: &u64, 98 | _fdt_xml: &str, 99 | _expires: std::time::SystemTime, 100 | _meta: &ObjectMetadata, 101 | _transfer_duration: std::time::Duration, 102 | _now: std::time::SystemTime, 103 | _ext_time: Option, 104 | ) { 105 | } 106 | } 107 | 108 | impl ObjectWriter for ObjectWriterBufferWrapper { 109 | fn open(&self, _now: SystemTime) -> Result<()> { 110 | Ok(()) 111 | } 112 | 113 | fn write(&self, _sbn: u32, data: &[u8], _now: SystemTime) -> Result<()> { 114 | let mut inner = self.inner.borrow_mut(); 115 | inner.data.extend(data); 116 | Ok(()) 117 | } 118 | 119 | fn complete(&self, now: SystemTime) { 120 | let mut inner = self.inner.borrow_mut(); 121 | log::info!("Object complete !"); 122 | inner.complete = true; 123 | inner.end_time = Some(now); 124 | } 125 | 126 | fn error(&self, now: SystemTime) { 127 | let mut inner = self.inner.borrow_mut(); 128 | log::error!("Object received with error"); 129 | inner.error = true; 130 | inner.end_time = Some(now); 131 | } 132 | 133 | fn interrupted(&self, now: SystemTime) { 134 | let mut inner = self.inner.borrow_mut(); 135 | log::error!("Object reception interrupted"); 136 | inner.error = true; 137 | inner.end_time = Some(now); 138 | } 139 | 140 | fn enable_md5_check(&self) -> bool { 141 | self.enable_md5_check 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/common/alccodec/alcrs28.rs: -------------------------------------------------------------------------------- 1 | use super::AlcCodec; 2 | use crate::{ 3 | common::{alc, lct, oti, pkt}, 4 | error::FluteError, 5 | }; 6 | 7 | pub struct AlcRS28 {} 8 | 9 | impl AlcCodec for AlcRS28 { 10 | fn add_fti(&self, data: &mut Vec, oti: &oti::Oti, transfer_length: u64) { 11 | /*0 1 2 3 12 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 13 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 14 | | HET = 64 | HEL = 3 | | 15 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + 16 | | Transfer Length (L) | 17 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 18 | | Encoding Symbol Length (E) | MaxBlkLen (B) | max_n | 19 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+*/ 20 | 21 | let ext_header_l: u64 = 22 | (lct::Ext::Fti as u64) << 56 | 3u64 << 48 | transfer_length & 0xFFFFFFFFFFFF; 23 | let max_n: u32 = 24 | (oti.max_number_of_parity_symbols + oti.maximum_source_block_length) & 0xFF; 25 | let e_b_n: u32 = (oti.encoding_symbol_length as u32) << 16 26 | | (oti.maximum_source_block_length & 0xFF) << 8 27 | | max_n; 28 | data.extend(ext_header_l.to_be_bytes()); 29 | data.extend(e_b_n.to_be_bytes()); 30 | lct::inc_hdr_len(data, 3); 31 | } 32 | 33 | fn get_fti( 34 | &self, 35 | data: &[u8], 36 | lct_header: &lct::LCTHeader, 37 | ) -> crate::error::Result> { 38 | let fti = match lct::get_ext(data, lct_header, lct::Ext::Fti as u8)? { 39 | Some(fti) => fti, 40 | None => return Ok(None), 41 | }; 42 | 43 | if fti.len() != 12 { 44 | return Err(FluteError::new("Wrong extension size")); 45 | } 46 | 47 | debug_assert!(fti[0] == lct::Ext::Fti as u8); 48 | if fti[1] != 3 { 49 | return Err(FluteError::new("Wrong header extension")); 50 | } 51 | 52 | let transfer_length = 53 | u64::from_be_bytes(fti[0..8].as_ref().try_into().unwrap()) & 0xFFFFFFFFFFFF; 54 | let encoding_symbol_length = u16::from_be_bytes(fti[8..10].as_ref().try_into().unwrap()); 55 | 56 | let maximum_source_block_length = fti[10]; 57 | let num_encoding_symbols = fti[11]; 58 | 59 | let oti = oti::Oti { 60 | fec_encoding_id: oti::FECEncodingID::ReedSolomonGF28, 61 | fec_instance_id: 0, 62 | maximum_source_block_length: maximum_source_block_length as u32, 63 | encoding_symbol_length, 64 | max_number_of_parity_symbols: num_encoding_symbols as u32 65 | - maximum_source_block_length as u32, 66 | scheme_specific: None, 67 | inband_fti: true, 68 | }; 69 | 70 | Ok(Some((oti, transfer_length))) 71 | } 72 | 73 | fn add_fec_payload_id(&self, data: &mut Vec, _oti: &oti::Oti, pkt: &pkt::Pkt) { 74 | /* 75 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 76 | | Source Block Number (24) | Enc. Symb. ID | 77 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 78 | */ 79 | 80 | let sbn = pkt.sbn & 0xFFFFFF; 81 | let esi = pkt.esi & 0xFF; 82 | 83 | let header: u32 = (sbn << 8) | esi & 0xFF; 84 | data.extend(header.to_be_bytes()); 85 | } 86 | 87 | fn get_fec_payload_id( 88 | &self, 89 | pkt: &alc::AlcPkt, 90 | _oti: &oti::Oti, 91 | ) -> crate::error::Result { 92 | self.get_fec_inline_payload_id(pkt) 93 | } 94 | 95 | fn get_fec_inline_payload_id(&self, pkt: &alc::AlcPkt) -> crate::error::Result { 96 | let data = &pkt.data[pkt.data_alc_header_offset..pkt.data_payload_offset]; 97 | let arr: [u8; 4] = match data.try_into() { 98 | Ok(arr) => arr, 99 | Err(e) => return Err(FluteError::new(e.to_string())), 100 | }; 101 | let payload_id_header = u32::from_be_bytes(arr); 102 | let sbn = payload_id_header >> 8; 103 | let esi = payload_id_header & 0xFF; 104 | Ok(alc::PayloadID { 105 | esi, 106 | sbn, 107 | source_block_length: None, 108 | }) 109 | } 110 | 111 | fn fec_payload_id_block_length(&self) -> usize { 112 | 4 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/common/alccodec/alcnocode.rs: -------------------------------------------------------------------------------- 1 | use super::AlcCodec; 2 | use crate::{ 3 | common::{alc, lct, oti, pkt}, 4 | error::FluteError, 5 | }; 6 | 7 | pub struct AlcNoCode {} 8 | 9 | impl AlcCodec for AlcNoCode { 10 | fn add_fti(&self, data: &mut Vec, oti: &oti::Oti, transfer_length: u64) { 11 | // https://tools.ietf.org/html/rfc5445 12 | /* 13 | +- 14 | | FTI <127 8bits | LEN (8bit) | 15 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 16 | | Transfer Length | 17 | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 18 | | | Reserved | 19 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-++ 20 | | Encoding Symbol Length | Max. Source Block Length (MSB)| 21 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 22 | | Max. Source Block Length (LSB)| 23 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+*/ 24 | let ext_header: u16 = (lct::Ext::Fti as u16) << 8 | 4u16; 25 | let transfer_header: u64 = transfer_length << 16; 26 | let esl: u16 = oti.encoding_symbol_length; 27 | let sbl_msb: u16 = ((oti.maximum_source_block_length >> 16) & 0xFFFF) as u16; 28 | let sbl_lsb: u16 = (oti.maximum_source_block_length & 0xFFFF) as u16; 29 | 30 | data.extend(ext_header.to_be_bytes()); 31 | data.extend(transfer_header.to_be_bytes()); 32 | data.extend(esl.to_be_bytes()); 33 | data.extend(sbl_msb.to_be_bytes()); 34 | data.extend(sbl_lsb.to_be_bytes()); 35 | lct::inc_hdr_len(data, 4); 36 | } 37 | 38 | fn get_fti( 39 | &self, 40 | data: &[u8], 41 | lct_header: &lct::LCTHeader, 42 | ) -> crate::error::Result> { 43 | let fti = match lct::get_ext(data, lct_header, lct::Ext::Fti as u8)? { 44 | Some(fti) => fti, 45 | None => return Ok(None), 46 | }; 47 | 48 | if fti.len() != 16 { 49 | return Err(FluteError::new("Wrong extension size")); 50 | } 51 | 52 | debug_assert!(fti[0] == lct::Ext::Fti as u8); 53 | if fti[1] != 4 { 54 | return Err(FluteError::new(format!( 55 | "Wrong exten header size {} != 4 for FTI", 56 | fti[1] 57 | ))); 58 | } 59 | 60 | let transfer_length = u64::from_be_bytes(fti[2..10].as_ref().try_into().unwrap()) >> 16; 61 | let encoding_symbol_length = u16::from_be_bytes(fti[10..12].as_ref().try_into().unwrap()); 62 | let maximum_source_block_length = 63 | u32::from_be_bytes(fti[12..16].as_ref().try_into().unwrap()); 64 | 65 | let oti = oti::Oti { 66 | fec_encoding_id: oti::FECEncodingID::NoCode, 67 | fec_instance_id: 0, 68 | maximum_source_block_length, 69 | encoding_symbol_length, 70 | max_number_of_parity_symbols: 0, 71 | scheme_specific: None, 72 | inband_fti: true, 73 | }; 74 | 75 | Ok(Some((oti, transfer_length))) 76 | } 77 | 78 | fn get_fec_payload_id( 79 | &self, 80 | pkt: &alc::AlcPkt, 81 | _oti: &oti::Oti, 82 | ) -> crate::error::Result { 83 | self.get_fec_inline_payload_id(pkt) 84 | } 85 | 86 | fn get_fec_inline_payload_id(&self, pkt: &alc::AlcPkt) -> crate::error::Result { 87 | let data = &pkt.data[pkt.data_alc_header_offset..pkt.data_payload_offset]; 88 | let arr: [u8; 4] = match data.try_into() { 89 | Ok(arr) => arr, 90 | Err(e) => return Err(FluteError::new(e.to_string())), 91 | }; 92 | let payload_id_header = u32::from_be_bytes(arr); 93 | let sbn = payload_id_header >> 16; 94 | let esi = payload_id_header & 0xFFFF; 95 | Ok(alc::PayloadID { 96 | esi, 97 | sbn, 98 | source_block_length: None, 99 | }) 100 | } 101 | 102 | fn add_fec_payload_id(&self, data: &mut Vec, _oti: &oti::Oti, pkt: &pkt::Pkt) { 103 | let sbn = pkt.sbn; 104 | let esi = pkt.esi; 105 | /* 106 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 107 | | Source Block Number 16 bits | Enc. Symb. ID 16 bits | 108 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 109 | */ 110 | let header: u32 = ((sbn & 0xFFFF) << 16) | esi & 0xFFFF; 111 | data.extend(header.to_be_bytes()); 112 | } 113 | 114 | fn fec_payload_id_block_length(&self) -> usize { 115 | 4 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/sender/sendersession.rs: -------------------------------------------------------------------------------- 1 | use super::blockencoder::BlockEncoder; 2 | use super::fdt::Fdt; 3 | use super::filedesc::FileDesc; 4 | #[cfg(feature = "opentelemetry")] 5 | use super::objectsenderlogger::ObjectSenderLogger; 6 | use super::Profile; 7 | use crate::common::alc; 8 | use crate::core::UDPEndpoint; 9 | use std::sync::Arc; 10 | use std::time::SystemTime; 11 | 12 | #[allow(dead_code)] 13 | #[derive(Debug)] 14 | pub struct SenderSession { 15 | priority: u32, 16 | endpoint: UDPEndpoint, 17 | tsi: u64, 18 | file: Option>, 19 | encoder: Option, 20 | interleave_blocks: usize, 21 | transfer_fdt_only: bool, 22 | profile: Profile, 23 | #[cfg(feature = "opentelemetry")] 24 | logger: Option, 25 | } 26 | 27 | impl SenderSession { 28 | pub fn new( 29 | priority: u32, 30 | tsi: u64, 31 | interleave_blocks: usize, 32 | transfer_fdt_only: bool, 33 | profile: Profile, 34 | endpoint: UDPEndpoint, 35 | ) -> SenderSession { 36 | SenderSession { 37 | priority, 38 | endpoint, 39 | tsi, 40 | file: None, 41 | encoder: None, 42 | interleave_blocks, 43 | transfer_fdt_only, 44 | profile, 45 | #[cfg(feature = "opentelemetry")] 46 | logger: None, 47 | } 48 | } 49 | 50 | pub fn run(&mut self, fdt: &mut Fdt, now: SystemTime) -> Option> { 51 | loop { 52 | if self.encoder.is_none() { 53 | self.get_next(fdt, now); 54 | } 55 | 56 | if !self.transfer_fdt_only { 57 | // Stop emitting packets if a new FDT is needed 58 | if fdt.need_transfer_fdt() { 59 | return None; 60 | } 61 | } 62 | 63 | let encoder = self.encoder.as_mut()?; 64 | 65 | debug_assert!(self.file.is_some()); 66 | let file = self.file.as_ref().unwrap(); 67 | let must_stop_transfer = !self.transfer_fdt_only 68 | && file.can_transfer_be_stopped() 69 | && !fdt.is_added(file.toi); 70 | 71 | if must_stop_transfer { 72 | log::debug!("File has already been transferred and is removed from the FDT, stop the transfer {}", file.object.content_location.to_string()); 73 | } 74 | 75 | if let Some(next_timestamp) = file.get_next_transfer_timestamp() { 76 | if next_timestamp > now { 77 | return None; 78 | } 79 | } 80 | 81 | let pkt = encoder.read(must_stop_transfer); 82 | if pkt.is_none() { 83 | self.release_file(fdt, now); 84 | continue; 85 | } 86 | 87 | file.inc_next_transfer_timestamp(); 88 | let pkt = pkt.as_ref().unwrap(); 89 | return Some(alc::new_alc_pkt( 90 | &file.oti, 91 | &0u128, 92 | self.tsi, 93 | pkt, 94 | self.profile, 95 | now, 96 | )); 97 | } 98 | } 99 | 100 | fn get_next(&mut self, fdt: &mut Fdt, now: SystemTime) { 101 | self.encoder = None; 102 | if self.transfer_fdt_only { 103 | self.file = fdt.get_next_fdt_transfer(now); 104 | } else { 105 | self.file = fdt.get_next_file_transfer(self.priority, now); 106 | } 107 | if self.file.is_none() { 108 | return; 109 | } 110 | 111 | #[cfg(feature = "opentelemetry")] 112 | if !self.transfer_fdt_only { 113 | let file = self.file.as_ref().unwrap(); 114 | if file.total_nb_transfer() == 0 { 115 | self.logger = Some(ObjectSenderLogger::new( 116 | &self.endpoint, 117 | self.tsi, 118 | file.toi, 119 | file.object.optel_propagator.as_ref(), 120 | )); 121 | } 122 | } 123 | 124 | let file = self.file.as_ref().unwrap().clone(); 125 | let is_last_transfer = file.is_last_transfer(); 126 | let block_encoder = BlockEncoder::new(file, self.interleave_blocks, is_last_transfer); 127 | if block_encoder.is_err() { 128 | log::error!("Fail to open Block Encoder"); 129 | self.release_file(fdt, now); 130 | return; 131 | } 132 | 133 | self.encoder = block_encoder.ok(); 134 | } 135 | 136 | fn release_file(&mut self, fdt: &mut Fdt, now: SystemTime) { 137 | if let Some(file) = &self.file { 138 | fdt.transfer_done(file.clone(), now) 139 | }; 140 | 141 | self.file = None; 142 | self.encoder = None; 143 | 144 | #[cfg(feature = "opentelemetry")] 145 | { 146 | self.logger = None; 147 | } 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/receiver/objectreceiverlogger.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use opentelemetry::{ 4 | global::{self, BoxedSpan}, 5 | propagation::{Extractor, Injector}, 6 | trace::{Span, SpanKind, Status, TraceContextExt, Tracer}, 7 | Context, KeyValue, 8 | }; 9 | 10 | use crate::common::udpendpoint::UDPEndpoint; 11 | 12 | pub struct ObjectReceiverLogger { 13 | cx: Context, 14 | } 15 | 16 | impl std::fmt::Debug for ObjectReceiverLogger { 17 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 18 | f.debug_struct("ObjectReceiverLogger").finish() 19 | } 20 | } 21 | 22 | struct HeaderExtractor<'a>(pub &'a HashMap); 23 | impl Extractor for HeaderExtractor<'_> { 24 | fn get(&self, key: &str) -> Option<&str> { 25 | self.0.get(key).map(|s| s.as_str()) 26 | } 27 | 28 | fn keys(&self) -> Vec<&str> { 29 | self.0.keys().map(|s| s.as_str()).collect() 30 | } 31 | } 32 | 33 | struct HeaderInjector<'a>(pub &'a mut HashMap); 34 | impl Injector for HeaderInjector<'_> { 35 | fn set(&mut self, key: &str, value: String) { 36 | self.0.insert(key.to_string(), value); 37 | } 38 | } 39 | 40 | impl ObjectReceiverLogger { 41 | fn extract_context_from_propagator(req: &HashMap) -> Context { 42 | global::get_text_map_propagator(|propagator| propagator.extract(&HeaderExtractor(req))) 43 | } 44 | 45 | pub fn new( 46 | endpoint: &UDPEndpoint, 47 | tsi: u64, 48 | toi: u128, 49 | propagator: Option<&HashMap>, 50 | ) -> Self { 51 | let tracer = global::tracer("FluteLogger"); 52 | let name = match toi { 53 | 0 => "FDT", 54 | _ => "FLUTEObject", 55 | }; 56 | 57 | let mut span; 58 | if let Some(propagator) = propagator { 59 | let parent_cx = Self::extract_context_from_propagator(propagator); 60 | span = tracer 61 | .span_builder(name) 62 | .with_kind(SpanKind::Consumer) 63 | .start_with_context(&tracer, &parent_cx) 64 | } else { 65 | span = tracer 66 | .span_builder(name) 67 | .with_kind(SpanKind::Consumer) 68 | .start(&tracer); 69 | } 70 | 71 | span.set_attribute(KeyValue::new( 72 | opentelemetry_semantic_conventions::attribute::NETWORK_TRANSPORT, 73 | "flute", 74 | )); 75 | 76 | span.set_attribute(KeyValue::new( 77 | opentelemetry_semantic_conventions::attribute::NETWORK_PEER_ADDRESS, 78 | endpoint.destination_group_address.clone(), 79 | )); 80 | 81 | span.set_attribute(KeyValue::new( 82 | opentelemetry_semantic_conventions::attribute::NETWORK_PEER_PORT, 83 | endpoint.port as i64, 84 | )); 85 | 86 | if let Some(source_address) = endpoint.source_address.as_ref() { 87 | span.set_attribute(KeyValue::new( 88 | opentelemetry_semantic_conventions::attribute::NETWORK_LOCAL_ADDRESS, 89 | source_address.clone(), 90 | )); 91 | } 92 | 93 | span.set_attribute(KeyValue::new("flute.toi", toi.to_string())); 94 | span.set_attribute(KeyValue::new("flute.tsi", tsi.to_string())); 95 | 96 | let cx = Context::default().with_span(span); 97 | Self { cx } 98 | } 99 | 100 | pub fn get_propagator(&self) -> HashMap { 101 | let propagator = global::get_text_map_propagator(|propagator| { 102 | let mut headers = HashMap::new(); 103 | propagator.inject_context(&self.cx, &mut HeaderInjector(&mut headers)); 104 | headers 105 | }); 106 | 107 | propagator 108 | } 109 | /* 110 | pub fn block_span(&mut self) -> BoxedSpan { 111 | let tracer = global::tracer("FluteLogger"); 112 | tracer.start_with_context("block", &self.cx) 113 | } 114 | */ 115 | 116 | pub fn fdt_attached(&mut self) -> BoxedSpan { 117 | let tracer = global::tracer("FluteLogger"); 118 | tracer.start_with_context("fdt_attached", &self.cx) 119 | } 120 | 121 | pub fn complete(&mut self) -> BoxedSpan { 122 | let tracer = global::tracer("FluteLogger"); 123 | 124 | let span = self.cx.span(); 125 | span.set_status(Status::Ok); 126 | 127 | tracer.start_with_context("complete", &self.cx) 128 | } 129 | 130 | pub fn interrupted(&mut self, description: &str) -> BoxedSpan { 131 | let tracer = global::tracer("FluteLogger"); 132 | 133 | let span = self.cx.span(); 134 | span.set_status(Status::Ok); 135 | 136 | span.set_attribute(KeyValue::new("description", description.to_string())); 137 | tracer.start_with_context("interrupted", &self.cx) 138 | } 139 | 140 | pub fn error(&mut self, description: &str) -> BoxedSpan { 141 | let tracer = global::tracer("FluteLogger"); 142 | 143 | let span = self.cx.span(); 144 | span.set_status(Status::Error { 145 | description: std::borrow::Cow::Owned(description.to_string()), 146 | }); 147 | 148 | span.set_attribute(KeyValue::new("error_description", description.to_string())); 149 | tracer.start_with_context("error", &self.cx) 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/fec/raptorq.rs: -------------------------------------------------------------------------------- 1 | use crate::common::oti::RaptorQSchemeSpecific; 2 | use crate::error::{FluteError, Result}; 3 | 4 | use super::{FecDecoder, FecEncoder, FecShard}; 5 | 6 | pub struct RaptorQEncoder { 7 | config: raptorq::ObjectTransmissionInformation, 8 | nb_parity_symbols: usize, 9 | } 10 | 11 | #[derive(Debug)] 12 | struct RaptorFecShard { 13 | pkt: raptorq::EncodingPacket, 14 | } 15 | 16 | impl FecShard for RaptorFecShard { 17 | fn data(&self) -> &[u8] { 18 | self.pkt.data() 19 | } 20 | fn esi(&self) -> u32 { 21 | self.pkt.payload_id().encoding_symbol_id() 22 | } 23 | } 24 | 25 | impl RaptorQEncoder { 26 | pub fn new( 27 | nb_source_symbols: usize, 28 | nb_parity_symbols: usize, 29 | encoding_symbol_length: usize, 30 | scheme: &RaptorQSchemeSpecific, 31 | ) -> Self { 32 | RaptorQEncoder { 33 | nb_parity_symbols, 34 | config: raptorq::ObjectTransmissionInformation::new( 35 | (nb_source_symbols * encoding_symbol_length) as u64, 36 | encoding_symbol_length as u16, 37 | 1, 38 | scheme.sub_blocks_length, 39 | scheme.symbol_alignment, 40 | ), 41 | } 42 | } 43 | } 44 | 45 | impl FecEncoder for RaptorQEncoder { 46 | fn encode(&self, data: &[u8]) -> crate::error::Result>> { 47 | let symbol_aligned = data.len() % self.config.symbol_size() as usize; 48 | let encoder = match data.len() % self.config.symbol_size() as usize { 49 | 0 => raptorq::SourceBlockEncoder::new(0, &self.config.clone(), data), 50 | _ => { 51 | let mut data = data.to_vec(); 52 | data.resize( 53 | data.len() + (self.config.symbol_size() as usize - symbol_aligned), 54 | 0, 55 | ); 56 | raptorq::SourceBlockEncoder::new(0, &self.config.clone(), &data) 57 | } 58 | }; 59 | 60 | let src_pkt = encoder.source_packets(); 61 | let repair_pkt = encoder.repair_packets(0, self.nb_parity_symbols as u32); 62 | let mut output: Vec> = Vec::new(); 63 | 64 | for pkt in src_pkt { 65 | output.push(Box::new(RaptorFecShard { pkt })); 66 | } 67 | 68 | for pkt in repair_pkt { 69 | output.push(Box::new(RaptorFecShard { pkt })); 70 | } 71 | 72 | Ok(output) 73 | } 74 | } 75 | 76 | pub struct RaptorQDecoder { 77 | decoder: raptorq::SourceBlockDecoder, 78 | data: Option>, 79 | sbn: u32, 80 | } 81 | 82 | impl RaptorQDecoder { 83 | pub fn new( 84 | sbn: u32, 85 | nb_source_symbols: usize, 86 | encoding_symbol_length: usize, 87 | scheme: &RaptorQSchemeSpecific, 88 | ) -> RaptorQDecoder { 89 | let config = raptorq::ObjectTransmissionInformation::new( 90 | (nb_source_symbols * encoding_symbol_length) as u64, 91 | encoding_symbol_length as u16, 92 | 1, 93 | scheme.sub_blocks_length, 94 | scheme.symbol_alignment, 95 | ); 96 | 97 | let block_length = nb_source_symbols as u64 * encoding_symbol_length as u64; 98 | let decoder = raptorq::SourceBlockDecoder::new(sbn as u8, &config, block_length); 99 | RaptorQDecoder { 100 | decoder, 101 | data: None, 102 | sbn, 103 | } 104 | } 105 | } 106 | 107 | impl FecDecoder for RaptorQDecoder { 108 | fn push_symbol(&mut self, encoding_symbol: &[u8], esi: u32) { 109 | if self.data.is_some() { 110 | return; 111 | } 112 | 113 | let pkt = raptorq::EncodingPacket::new( 114 | raptorq::PayloadId::new(self.sbn as u8, esi), 115 | encoding_symbol.to_vec(), 116 | ); 117 | 118 | self.data = self.decoder.decode(vec![pkt]); 119 | } 120 | 121 | fn can_decode(&self) -> bool { 122 | self.data.is_some() 123 | } 124 | 125 | fn decode(&mut self) -> bool { 126 | self.data.is_some() 127 | } 128 | 129 | fn source_block(&self) -> Result<&[u8]> { 130 | if self.data.is_none() { 131 | return Err(FluteError::new("Source block not decoded")); 132 | } 133 | 134 | Ok(self.data.as_ref().unwrap()) 135 | } 136 | } 137 | 138 | #[cfg(test)] 139 | mod tests { 140 | use crate::{common::oti::RaptorQSchemeSpecific, fec::FecEncoder}; 141 | 142 | #[test] 143 | pub fn test_raptorq_encode() { 144 | crate::tests::init(); 145 | 146 | let nb_source_symbols = 10usize; 147 | let nb_parity_symbols = 2usize; 148 | let symbols_length = 1024usize; 149 | 150 | let data = vec![0xAAu8; nb_source_symbols * symbols_length]; 151 | 152 | let scheme = RaptorQSchemeSpecific { 153 | source_blocks_length: 1, 154 | sub_blocks_length: 1, 155 | symbol_alignment: 8, 156 | }; 157 | 158 | let r = super::RaptorQEncoder::new( 159 | nb_source_symbols, 160 | nb_parity_symbols, 161 | symbols_length, 162 | &scheme, 163 | ); 164 | let encoded_data = r.encode(data.as_ref()).unwrap(); 165 | log::info!("NB source symbols={}", encoded_data.len()); 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /assets/xsd/FLUTE-FDT-3GPP-Main.xsd: -------------------------------------------------------------------------------- 1 | 2 | 16 | 18 | 20 | 22 | 24 | 26 | 28 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 62 | 64 | 65 | 66 | 67 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /src/fec/rscodec.rs: -------------------------------------------------------------------------------- 1 | use crate::tools::error::{FluteError, Result}; 2 | 3 | use super::{DataFecShard, FecDecoder, FecEncoder, FecShard}; 4 | 5 | #[derive(Debug)] 6 | pub struct RSCodecParam { 7 | nb_source_symbols: usize, 8 | nb_parity_symbols: usize, 9 | encoding_symbol_length: usize, 10 | } 11 | 12 | #[derive(Debug)] 13 | pub struct RSGalois8Codec { 14 | params: RSCodecParam, 15 | rs: reed_solomon_erasure::galois_8::ReedSolomon, 16 | decode_shards: Vec>>, 17 | decode_block: Option>, 18 | nb_source_symbols_received: usize, 19 | nb_encoding_symbols_received: usize, 20 | } 21 | 22 | impl RSCodecParam { 23 | fn create_shards(&self, data: &[u8]) -> Result>> { 24 | let mut shards: Vec> = data 25 | .chunks(self.encoding_symbol_length) 26 | .map(|chunck| chunck.to_vec()) 27 | .collect(); 28 | 29 | let last = shards.last_mut().unwrap(); 30 | if last.len() < self.encoding_symbol_length { 31 | last.resize(self.encoding_symbol_length, 0) 32 | } 33 | if shards.len() != self.nb_source_symbols { 34 | return Err(FluteError::new(format!( 35 | "nb source symbols is {} instead of {}", 36 | shards.len(), 37 | self.nb_source_symbols 38 | ))); 39 | } 40 | 41 | for _ in 0..self.nb_parity_symbols { 42 | shards.push(vec![0; self.encoding_symbol_length]); 43 | } 44 | Ok(shards) 45 | } 46 | } 47 | 48 | impl RSGalois8Codec { 49 | pub fn new( 50 | nb_source_symbols: usize, 51 | nb_parity_symbols: usize, 52 | encoding_symbol_length: usize, 53 | ) -> Result { 54 | let rs = 55 | reed_solomon_erasure::galois_8::ReedSolomon::new(nb_source_symbols, nb_parity_symbols) 56 | .map_err(|_| FluteError::new("Fail to create RS codec"))?; 57 | 58 | Ok(RSGalois8Codec { 59 | params: RSCodecParam { 60 | nb_source_symbols, 61 | nb_parity_symbols, 62 | encoding_symbol_length, 63 | }, 64 | rs, 65 | decode_shards: vec![None; nb_source_symbols + nb_parity_symbols], 66 | decode_block: None, 67 | nb_source_symbols_received: 0, 68 | nb_encoding_symbols_received: 0, 69 | }) 70 | } 71 | } 72 | 73 | impl FecDecoder for RSGalois8Codec { 74 | fn push_symbol(&mut self, encoding_symbol: &[u8], esi: u32) { 75 | if self.decode_block.is_some() { 76 | return; 77 | } 78 | 79 | log::info!("Receive ESI {}", esi); 80 | if self.decode_shards.len() <= esi as usize { 81 | return; 82 | } 83 | 84 | if self.decode_shards[esi as usize].is_some() { 85 | return; 86 | } 87 | 88 | self.decode_shards[esi as usize] = Some(encoding_symbol.to_vec()); 89 | if esi < self.params.nb_source_symbols as u32 { 90 | self.nb_source_symbols_received += 1; 91 | } 92 | self.nb_encoding_symbols_received += 1; 93 | } 94 | 95 | fn can_decode(&self) -> bool { 96 | self.nb_encoding_symbols_received >= self.params.nb_source_symbols 97 | } 98 | 99 | fn decode(&mut self) -> bool { 100 | if self.decode_block.is_some() { 101 | return true; 102 | } 103 | 104 | if self.nb_source_symbols_received < self.params.nb_source_symbols { 105 | match self.rs.reconstruct(&mut self.decode_shards) { 106 | Ok(_) => { 107 | log::info!("Reconstruct with success !"); 108 | } 109 | Err(e) => { 110 | log::error!("{:?}", e); 111 | return false; 112 | } 113 | }; 114 | } 115 | 116 | let mut output = Vec::new(); 117 | for i in 0..self.params.nb_source_symbols { 118 | if self.decode_shards[i].is_none() { 119 | log::error!("BUG? a shard is missing"); 120 | return false; 121 | } 122 | output.extend(self.decode_shards[i].as_ref().unwrap()); 123 | } 124 | 125 | self.decode_block = Some(output); 126 | true 127 | } 128 | 129 | fn source_block(&self) -> Result<&[u8]> { 130 | match self.decode_block.as_ref() { 131 | Some(e) => Ok(e), 132 | None => Err(FluteError::new("Block not decoded")), 133 | } 134 | } 135 | } 136 | 137 | impl FecEncoder for RSGalois8Codec { 138 | fn encode(&self, data: &[u8]) -> Result>> { 139 | let mut shards = self.params.create_shards(data)?; 140 | self.rs 141 | .encode(&mut shards) 142 | .map_err(|_| FluteError::new("Fail to encode RS"))?; 143 | 144 | let shards: Vec> = shards 145 | .into_iter() 146 | .enumerate() 147 | .map(|(index, shard)| { 148 | Box::new(DataFecShard { 149 | shard, 150 | index: index as u32, 151 | }) as Box 152 | }) 153 | .collect(); 154 | 155 | Ok(shards) 156 | } 157 | } 158 | 159 | #[cfg(test)] 160 | mod tests { 161 | use crate::fec::FecEncoder; 162 | #[test] 163 | pub fn test_encoder() { 164 | crate::tests::init(); 165 | let data = vec![1, 2, 3, 4, 5]; 166 | let encoder = super::RSGalois8Codec::new(2, 3, 4).unwrap(); 167 | let _shards = encoder.encode(&data).unwrap(); 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /py_tests/__init__.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | import logging 3 | 4 | 5 | def init(): 6 | FORMAT = '%(levelname)s %(name)s %(asctime)-15s %(filename)s:%(lineno)d %(message)s' 7 | logging.basicConfig(format=FORMAT) 8 | logging.getLogger().setLevel(logging.DEBUG) 9 | 10 | 11 | class SenderReceiverTestCase(TestCase): 12 | 13 | 14 | init() 15 | 16 | def test_create_sender(self): 17 | from flute import sender 18 | print("------- test_create_sender--------") 19 | config = sender.Config() 20 | oti = sender.Oti.new_no_code(1400, 64) 21 | flute_sender = sender.Sender(1, oti, config) 22 | 23 | buf = bytes(b'hello') 24 | flute_sender.add_object_from_buffer(buf, "text", "file://hello.txt", None) 25 | flute_sender.publish() 26 | 27 | while True: 28 | pkt = flute_sender.read() 29 | if pkt == None: 30 | break 31 | 32 | print("Received pkt of " + str(len(pkt)) + " bytes") 33 | 34 | print("File transmitted !") 35 | 36 | def test_create_receiver(self): 37 | from flute import receiver 38 | print("------- test_create_receiver--------") 39 | writer = receiver.ObjectWriterBuilder.new_buffer() 40 | config = receiver.Config() 41 | udp_endpoint = receiver.UDPEndpoint("224.0.0.1", 1234) 42 | flute_receiver = receiver.Receiver(udp_endpoint, 1, writer, config) 43 | print("Flute Receiver created !") 44 | 45 | def test_create_multireceiver(self): 46 | from flute import receiver 47 | print("------- test_create_multireceiver--------") 48 | 49 | writer = receiver.ObjectWriterBuilder.new_buffer() 50 | config = receiver.Config() 51 | 52 | flute_receiver = receiver.MultiReceiver(writer, config) 53 | 54 | print("Flute Receiver created !") 55 | 56 | 57 | def test_send_receiver(self): 58 | from flute import sender, receiver 59 | 60 | print("------- test_send_receiver--------") 61 | 62 | tsi = 1 63 | 64 | sender_config = sender.Config() 65 | oti = sender.Oti.new_no_code(1400, 64) 66 | flute_sender = sender.Sender(tsi, oti, sender_config) 67 | 68 | receiver_writer = receiver.ObjectWriterBuilder.new_buffer() 69 | receiver_config = receiver.Config() 70 | udp_endpoint = receiver.UDPEndpoint("224.0.0.1", 1234) 71 | flute_receiver = receiver.Receiver(udp_endpoint, tsi, receiver_writer, receiver_config) 72 | 73 | buf = bytes(b'hello world') 74 | flute_sender.add_object_from_buffer(buf, "text", "file://hello.txt", None) 75 | flute_sender.publish() 76 | 77 | while True: 78 | pkt = flute_sender.read() 79 | if pkt == None: 80 | break 81 | 82 | flute_receiver.push(bytes(pkt)) 83 | 84 | def test_send_multi_receiver(self): 85 | from flute import sender, receiver 86 | 87 | print("------- test_send_multi_receiver--------") 88 | 89 | tsi = 1 90 | 91 | sender_config = sender.Config() 92 | oti = sender.Oti.new_no_code(1400, 64) 93 | flute_sender = sender.Sender(tsi, oti, sender_config) 94 | 95 | receiver_writer = receiver.ObjectWriterBuilder.new_buffer() 96 | receiver_config = receiver.Config() 97 | flute_receiver = receiver.MultiReceiver(receiver_writer, receiver_config) 98 | 99 | buf = bytes(b'hello world') 100 | flute_sender.add_object_from_buffer(buf, "text", "file://hello.txt", None) 101 | flute_sender.publish() 102 | 103 | udp_endpoint = receiver.UDPEndpoint("224.0.0.1", 1234) 104 | 105 | while True: 106 | pkt = flute_sender.read() 107 | if pkt == None: 108 | break 109 | 110 | flute_receiver.push(udp_endpoint, bytes(pkt)) 111 | 112 | def test_remove_object(self): 113 | from flute import sender 114 | print("------- test_remove_object--------") 115 | config = sender.Config() 116 | oti = sender.Oti.new_no_code(1400, 64) 117 | flute_sender = sender.Sender(1, oti, config) 118 | 119 | buf = bytes(b'hello') 120 | toi = flute_sender.add_object_from_buffer(buf, "text", "file://hello.txt", None) 121 | print("object with TOI " + str(toi) + " added") 122 | assert(flute_sender.nb_objects() == 1) 123 | 124 | success = flute_sender.remove_object(toi) 125 | assert(success == True) 126 | assert(flute_sender.nb_objects() == 0) 127 | 128 | 129 | def test_lct(self): 130 | from flute import sender, receiver 131 | 132 | print("------- test_lct--------") 133 | 134 | tsi = 1234 135 | 136 | sender_config = sender.Config() 137 | oti = sender.Oti.new_no_code(1400, 64) 138 | flute_sender = sender.Sender(tsi, oti, sender_config) 139 | 140 | receiver_writer = receiver.ObjectWriterBuilder.new_buffer() 141 | receiver_config = receiver.Config() 142 | udp_endpoint = receiver.UDPEndpoint("224.0.0.1", 1234) 143 | flute_receiver = receiver.Receiver(udp_endpoint, tsi, receiver_writer, receiver_config) 144 | 145 | buf = bytes(b'hello world') 146 | flute_sender.add_object_from_buffer(buf, "text", "file://hello.txt", None) 147 | flute_sender.publish() 148 | 149 | pkt = flute_sender.read() 150 | lct = receiver.LCTHeader(bytes(pkt)) 151 | assert(lct.cci == 0) 152 | assert(lct.tsi == 1234) 153 | assert(lct.toi == 0) 154 | assert(lct.sbn == 0) 155 | assert(lct.esi == 0) 156 | 157 | if __name__ == '__main__': 158 | unittest.main() 159 | -------------------------------------------------------------------------------- /src/common/alccodec/alcraptor.rs: -------------------------------------------------------------------------------- 1 | use super::AlcCodec; 2 | use crate::{ 3 | common::{ 4 | alc, lct, 5 | oti::{self, SchemeSpecific}, 6 | pkt, 7 | }, 8 | error::FluteError, 9 | }; 10 | 11 | pub struct AlcRaptor {} 12 | 13 | impl AlcCodec for AlcRaptor { 14 | fn add_fti(&self, data: &mut Vec, oti: &oti::Oti, transfer_length: u64) { 15 | /* 16 | +- 17 | | FTI <127 8bits| LEN (8bit) | 18 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 19 | | Transfer Length (F) | 20 | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 21 | | | Reserved | Symbol Size (T) | 22 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 23 | | Z | N | Al | 24 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 25 | | PADDING (16 bits) ?? | 26 | 27 | Transfer Length (F): 40-bit unsigned integer 28 | Symbol Size (T): 16-bit unsigned integer. 29 | The number of source blocks (Z): 16-bit unsigned integer. 30 | The number of sub-blocks (N): 8-bit unsigned integer. 31 | A symbol alignment parameter (Al): 8-bit unsigned integer. 32 | */ 33 | let len: u8 = 4; 34 | let ext_header: u16 = (lct::Ext::Fti as u16) << 8 | len as u16; 35 | let transfer_header: u64 = 36 | (transfer_length << 24) | (oti.encoding_symbol_length as u64 & 0xFFFF); 37 | 38 | debug_assert!(oti.scheme_specific.is_some()); 39 | if let SchemeSpecific::Raptor(raptor) = oti.scheme_specific.as_ref().unwrap() { 40 | let padding: u16 = 0; 41 | data.extend(ext_header.to_be_bytes()); 42 | data.extend(transfer_header.to_be_bytes()); 43 | data.extend(raptor.source_blocks_length.to_be_bytes()); 44 | data.extend(raptor.sub_blocks_length.to_be_bytes()); 45 | data.extend(raptor.symbol_alignment.to_be_bytes()); 46 | data.extend(padding.to_be_bytes()); 47 | lct::inc_hdr_len(data, len); 48 | } else { 49 | debug_assert!(false); 50 | } 51 | } 52 | 53 | fn get_fti( 54 | &self, 55 | data: &[u8], 56 | lct_header: &lct::LCTHeader, 57 | ) -> crate::error::Result> { 58 | let fti = match lct::get_ext(data, lct_header, lct::Ext::Fti as u8)? { 59 | Some(fti) => fti, 60 | None => return Ok(None), 61 | }; 62 | 63 | if fti.len() != 16 { 64 | return Err(FluteError::new("Wrong extension size")); 65 | } 66 | 67 | let transfer_length = u64::from_be_bytes(fti[2..10].as_ref().try_into().unwrap()) >> 24; 68 | let symbol_size = u16::from_be_bytes(fti[8..10].as_ref().try_into().unwrap()); 69 | let z = u16::from_be_bytes(fti[10..12].as_ref().try_into().unwrap()); 70 | let n = fti[12]; 71 | let al = fti[13]; 72 | 73 | if symbol_size == 0 { 74 | return Err(FluteError::new("Symbol size is null")); 75 | } 76 | 77 | if z == 0 { 78 | return Err(FluteError::new("Z is null")); 79 | } 80 | 81 | if al == 0 { 82 | return Err(FluteError::new("AL must be at least 1")); 83 | } 84 | 85 | if symbol_size % al as u16 != 0 { 86 | return Err(FluteError::new("Symbol size is not properly aligned")); 87 | } 88 | 89 | let block_size = num_integer::div_ceil(transfer_length, z as u64); 90 | let maximum_source_block_length = num_integer::div_ceil(block_size, symbol_size as u64); 91 | 92 | let oti = oti::Oti { 93 | fec_encoding_id: oti::FECEncodingID::Raptor, 94 | fec_instance_id: 0, 95 | maximum_source_block_length: maximum_source_block_length as u32, 96 | encoding_symbol_length: symbol_size, 97 | max_number_of_parity_symbols: 0, // Unknown for RaptorQ 98 | scheme_specific: Some(SchemeSpecific::Raptor(oti::RaptorSchemeSpecific { 99 | source_blocks_length: z, 100 | sub_blocks_length: n, 101 | symbol_alignment: al, 102 | })), 103 | inband_fti: true, 104 | }; 105 | 106 | Ok(Some((oti, transfer_length))) 107 | } 108 | 109 | fn add_fec_payload_id(&self, data: &mut Vec, _oti: &oti::Oti, pkt: &pkt::Pkt) { 110 | /* 111 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 112 | | Source Block Number | Encoding Symbol ID | 113 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 114 | */ 115 | 116 | let payload_id = (pkt.sbn & 0xFFFFu32) << 16 | pkt.esi & 0xFFFFu32; 117 | data.extend(payload_id.to_be_bytes()); 118 | } 119 | 120 | fn get_fec_payload_id( 121 | &self, 122 | pkt: &alc::AlcPkt, 123 | _oti: &oti::Oti, 124 | ) -> crate::error::Result { 125 | self.get_fec_inline_payload_id(pkt) 126 | } 127 | 128 | fn get_fec_inline_payload_id(&self, pkt: &alc::AlcPkt) -> crate::error::Result { 129 | let data = &pkt.data[pkt.data_alc_header_offset..pkt.data_payload_offset]; 130 | let arr: [u8; 4] = match data.try_into() { 131 | Ok(arr) => arr, 132 | Err(e) => return Err(FluteError::new(e.to_string())), 133 | }; 134 | let payload_id_header = u32::from_be_bytes(arr); 135 | let sbn = payload_id_header >> 16; 136 | let esi = payload_id_header & 0xFFFF; 137 | Ok(alc::PayloadID { 138 | esi, 139 | sbn, 140 | source_block_length: None, 141 | }) 142 | } 143 | 144 | fn fec_payload_id_block_length(&self) -> usize { 145 | 4 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /src/sender/block.rs: -------------------------------------------------------------------------------- 1 | use crate::common::oti::{self, Oti, SchemeSpecific}; 2 | use crate::fec::{self, FecShard}; 3 | use crate::fec::{DataFecShard, FecEncoder}; 4 | use crate::tools::error::{FluteError, Result}; 5 | 6 | #[derive(Debug)] 7 | pub struct Block { 8 | sbn: u32, 9 | read_index: u32, 10 | shards: Vec>, 11 | pub nb_source_symbols: usize, 12 | } 13 | 14 | pub struct EncodingSymbol<'a> { 15 | pub sbn: u32, 16 | pub esi: u32, 17 | pub symbols: &'a [u8], 18 | pub is_source_symbol: bool, 19 | } 20 | 21 | impl Block { 22 | pub fn new_from_buffer( 23 | sbn: u32, 24 | buffer: &[u8], 25 | block_length: u64, 26 | oti: &Oti, 27 | ) -> Result> { 28 | if oti.encoding_symbol_length == 0 { 29 | return Err(FluteError::new("Encoding symbol length is null")); 30 | } 31 | 32 | let nb_source_symbols: usize = 33 | num_integer::div_ceil(buffer.len(), oti.encoding_symbol_length as usize); 34 | log::debug!( 35 | "nb_source_symbols={} encoding_symbol_length={}", 36 | nb_source_symbols, 37 | oti.encoding_symbol_length 38 | ); 39 | let shards: Vec> = match oti.fec_encoding_id { 40 | oti::FECEncodingID::NoCode => Block::create_shards_no_code(oti, buffer), 41 | oti::FECEncodingID::ReedSolomonGF28 => Block::create_shards_reed_solomon_gf8( 42 | oti, 43 | nb_source_symbols, 44 | block_length as usize, 45 | buffer, 46 | )?, 47 | oti::FECEncodingID::ReedSolomonGF28UnderSpecified => { 48 | Block::create_shards_reed_solomon_gf8( 49 | oti, 50 | nb_source_symbols, 51 | block_length as usize, 52 | buffer, 53 | )? 54 | } 55 | oti::FECEncodingID::ReedSolomonGF2M => return Err(FluteError::new("Not implemented")), 56 | oti::FECEncodingID::RaptorQ => { 57 | Block::create_shards_raptorq(oti, nb_source_symbols, block_length as usize, buffer)? 58 | } 59 | oti::FECEncodingID::Raptor => { 60 | Block::create_shards_raptor(oti, nb_source_symbols, block_length as usize, buffer)? 61 | } 62 | }; 63 | 64 | Ok(Box::new(Block { 65 | sbn, 66 | read_index: 0, 67 | shards, 68 | nb_source_symbols, 69 | })) 70 | } 71 | 72 | pub fn is_empty(&self) -> bool { 73 | self.read_index as usize == self.shards.len() 74 | } 75 | 76 | pub fn read<'a>(&'a mut self) -> Option<(EncodingSymbol<'a>, bool)> { 77 | if self.is_empty() { 78 | return None; 79 | } 80 | let shard = self.shards[self.read_index as usize].as_ref(); 81 | let esi = shard.esi(); 82 | let is_source_symbol = (esi as usize) < self.nb_source_symbols; 83 | let symbol = EncodingSymbol { 84 | sbn: self.sbn, 85 | esi: shard.esi(), 86 | symbols: shard.data(), 87 | is_source_symbol, 88 | }; 89 | self.read_index += 1; 90 | Some((symbol, self.is_empty())) 91 | } 92 | 93 | fn create_shards_no_code(oti: &Oti, buffer: &[u8]) -> Vec> { 94 | buffer 95 | .chunks(oti.encoding_symbol_length as usize) 96 | .enumerate() 97 | .map(|(index, chunk)| { 98 | Box::new(DataFecShard::new(chunk, index as u32)) as Box 99 | }) 100 | .collect() 101 | } 102 | 103 | fn create_shards_reed_solomon_gf8( 104 | oti: &Oti, 105 | nb_source_symbols: usize, 106 | block_length: usize, 107 | buffer: &[u8], 108 | ) -> Result>> { 109 | debug_assert!(nb_source_symbols <= oti.maximum_source_block_length as usize); 110 | debug_assert!(nb_source_symbols <= block_length); 111 | let encoder = fec::rscodec::RSGalois8Codec::new( 112 | nb_source_symbols, 113 | oti.max_number_of_parity_symbols as usize, 114 | oti.encoding_symbol_length as usize, 115 | )?; 116 | let shards = encoder.encode(buffer)?; 117 | Ok(shards) 118 | } 119 | 120 | fn create_shards_raptorq( 121 | oti: &Oti, 122 | nb_source_symbols: usize, 123 | block_length: usize, 124 | buffer: &[u8], 125 | ) -> Result>> { 126 | debug_assert!(nb_source_symbols <= oti.maximum_source_block_length as usize); 127 | debug_assert!(nb_source_symbols <= block_length); 128 | debug_assert!(oti.scheme_specific.is_some()); 129 | 130 | if let Some(SchemeSpecific::RaptorQ(scheme)) = oti.scheme_specific.as_ref() { 131 | let encoder = fec::raptorq::RaptorQEncoder::new( 132 | nb_source_symbols, 133 | oti.max_number_of_parity_symbols as usize, 134 | oti.encoding_symbol_length as usize, 135 | scheme, 136 | ); 137 | 138 | let shards = encoder.encode(buffer)?; 139 | Ok(shards) 140 | } else { 141 | Err(FluteError::new("Scheme specific for Raptorq not defined")) 142 | } 143 | } 144 | 145 | fn create_shards_raptor( 146 | oti: &Oti, 147 | nb_source_symbols: usize, 148 | block_length: usize, 149 | buffer: &[u8], 150 | ) -> Result>> { 151 | debug_assert!(nb_source_symbols <= oti.maximum_source_block_length as usize); 152 | debug_assert!(nb_source_symbols <= block_length); 153 | debug_assert!(oti.scheme_specific.is_some()); 154 | 155 | let encoder = fec::raptor::RaptorEncoder::new( 156 | nb_source_symbols, 157 | oti.max_number_of_parity_symbols as usize, 158 | ); 159 | let shards = encoder.encode(buffer)?; 160 | Ok(shards) 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /src/common/alccodec/alcraptorq.rs: -------------------------------------------------------------------------------- 1 | use super::AlcCodec; 2 | use crate::{ 3 | common::{ 4 | alc, lct, 5 | oti::{self, SchemeSpecific}, 6 | pkt, 7 | }, 8 | error::FluteError, 9 | }; 10 | 11 | pub struct AlcRaptorQ {} 12 | 13 | impl AlcCodec for AlcRaptorQ { 14 | fn add_fti(&self, data: &mut Vec, oti: &oti::Oti, transfer_length: u64) { 15 | /* 16 | +- 17 | | FTI <127 8bits| LEN (8bit) | 18 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 19 | | Transfer Length (F) | 20 | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 21 | | | Reserved | Symbol Size (T) | 22 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 23 | | Z | N | Al | 24 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 25 | | PADDING (16 bits) ?? | 26 | 27 | Transfer Length (F): 40-bit unsigned integer 28 | Symbol Size (T): 16-bit unsigned integer. 29 | The number of source blocks (Z): 8-bit unsigned integer. 30 | The number of sub-blocks (N): 16-bit unsigned integer. 31 | A symbol alignment parameter (Al): 8-bit unsigned integer. 32 | */ 33 | let len: u8 = 4; 34 | let ext_header: u16 = (lct::Ext::Fti as u16) << 8 | len as u16; 35 | let transfer_header: u64 = 36 | (transfer_length << 24) | (oti.encoding_symbol_length as u64 & 0xFFFF); 37 | 38 | debug_assert!(oti.scheme_specific.is_some()); 39 | if let SchemeSpecific::RaptorQ(raptorq) = oti.scheme_specific.as_ref().unwrap() { 40 | let padding: u16 = 0; 41 | data.extend(ext_header.to_be_bytes()); 42 | data.extend(transfer_header.to_be_bytes()); 43 | data.push(raptorq.source_blocks_length); 44 | data.extend(raptorq.sub_blocks_length.to_be_bytes()); 45 | data.push(raptorq.symbol_alignment); 46 | data.extend(padding.to_be_bytes()); 47 | lct::inc_hdr_len(data, len); 48 | } else { 49 | debug_assert!(false); 50 | } 51 | } 52 | 53 | fn get_fti( 54 | &self, 55 | data: &[u8], 56 | lct_header: &lct::LCTHeader, 57 | ) -> crate::error::Result> { 58 | let fti = match lct::get_ext(data, lct_header, lct::Ext::Fti as u8)? { 59 | Some(fti) => fti, 60 | None => return Ok(None), 61 | }; 62 | 63 | if fti.len() != 16 { 64 | return Err(FluteError::new("Wrong extension size")); 65 | } 66 | 67 | let transfer_length = u64::from_be_bytes(fti[2..10].as_ref().try_into().unwrap()) >> 24; 68 | let symbol_size = u16::from_be_bytes(fti[8..10].as_ref().try_into().unwrap()); 69 | let z = fti[10]; 70 | let n = u16::from_be_bytes(fti[11..13].as_ref().try_into().unwrap()); 71 | let al = fti[13]; 72 | log::debug!( 73 | "length={} sym={} z={} n={} al={}", 74 | transfer_length, 75 | symbol_size, 76 | z, 77 | n, 78 | al 79 | ); 80 | 81 | if symbol_size == 0 { 82 | return Err(FluteError::new("Symbol size is null")); 83 | } 84 | 85 | if z == 0 { 86 | return Err(FluteError::new("Z is null")); 87 | } 88 | 89 | if al == 0 { 90 | return Err(FluteError::new("AL must be at least 1")); 91 | } 92 | 93 | if symbol_size % al as u16 != 0 { 94 | return Err(FluteError::new("Symbol size is not properly aligned")); 95 | } 96 | 97 | let block_size = num_integer::div_ceil(transfer_length, z as u64); 98 | let maximum_source_block_length = num_integer::div_ceil(block_size, symbol_size as u64); 99 | 100 | let oti = oti::Oti { 101 | fec_encoding_id: oti::FECEncodingID::RaptorQ, 102 | fec_instance_id: 0, 103 | maximum_source_block_length: maximum_source_block_length as u32, 104 | encoding_symbol_length: symbol_size, 105 | max_number_of_parity_symbols: 0, // Unknown for RaptorQ 106 | scheme_specific: Some(SchemeSpecific::RaptorQ(oti::RaptorQSchemeSpecific { 107 | source_blocks_length: z, 108 | sub_blocks_length: n, 109 | symbol_alignment: al, 110 | })), 111 | inband_fti: true, 112 | }; 113 | 114 | Ok(Some((oti, transfer_length))) 115 | } 116 | 117 | fn add_fec_payload_id(&self, data: &mut Vec, _oti: &oti::Oti, pkt: &pkt::Pkt) { 118 | /* 119 | 0 1 2 3 120 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 121 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 122 | | SBN | Encoding Symbol ID | 123 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 124 | */ 125 | 126 | let payload_id = (pkt.sbn & 0xFFu32) << 24 | pkt.esi & 0xFFFFFFu32; 127 | data.extend(payload_id.to_be_bytes()); 128 | } 129 | 130 | fn get_fec_payload_id( 131 | &self, 132 | pkt: &alc::AlcPkt, 133 | _oti: &oti::Oti, 134 | ) -> crate::error::Result { 135 | self.get_fec_inline_payload_id(pkt) 136 | } 137 | 138 | fn get_fec_inline_payload_id(&self, pkt: &alc::AlcPkt) -> crate::error::Result { 139 | let data = &pkt.data[pkt.data_alc_header_offset..pkt.data_payload_offset]; 140 | let arr: [u8; 4] = match data.try_into() { 141 | Ok(arr) => arr, 142 | Err(e) => return Err(FluteError::new(e.to_string())), 143 | }; 144 | let payload_id_header = u32::from_be_bytes(arr); 145 | let sbn = payload_id_header >> 24; 146 | let esi = payload_id_header & 0xFFFFFF; 147 | Ok(alc::PayloadID { 148 | esi, 149 | sbn, 150 | source_block_length: None, 151 | }) 152 | } 153 | 154 | fn fec_payload_id_block_length(&self) -> usize { 155 | 4 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/common/alccodec/alcrs28underspecified.rs: -------------------------------------------------------------------------------- 1 | use super::AlcCodec; 2 | use crate::{ 3 | common::{alc, lct, oti, pkt}, 4 | error::FluteError, 5 | }; 6 | 7 | pub struct AlcRS28UnderSpecified {} 8 | 9 | impl AlcCodec for AlcRS28UnderSpecified { 10 | fn add_fti(&self, data: &mut Vec, oti: &oti::Oti, transfer_length: u64) { 11 | /* 12 | * 0 1 2 3 13 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 14 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 15 | | Transfer Length | 16 | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 17 | | | FEC Instance ID | 18 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 19 | | Encoding Symbol Length | Maximum Source Block Length | 20 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 21 | | Max. Num. of Encoding Symbols | 22 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 23 | */ 24 | 25 | let ext_header: u16 = (lct::Ext::Fti as u16) << 8 | 4u16; 26 | let transfer_header_fec_id: u64 = (transfer_length << 16) | oti.fec_instance_id as u64; 27 | let esl: u16 = oti.encoding_symbol_length; 28 | let sbl: u16 = ((oti.maximum_source_block_length) & 0xFFFF) as u16; 29 | let mne: u16 = (oti.max_number_of_parity_symbols + oti.maximum_source_block_length) as u16; 30 | 31 | data.extend(ext_header.to_be_bytes()); 32 | data.extend(transfer_header_fec_id.to_be_bytes()); 33 | data.extend(esl.to_be_bytes()); 34 | data.extend(sbl.to_be_bytes()); 35 | data.extend(mne.to_be_bytes()); 36 | lct::inc_hdr_len(data, 4); 37 | } 38 | 39 | fn get_fti( 40 | &self, 41 | data: &[u8], 42 | lct_header: &lct::LCTHeader, 43 | ) -> crate::error::Result> { 44 | let fti = match lct::get_ext(data, lct_header, lct::Ext::Fti as u8)? { 45 | Some(fti) => fti, 46 | None => return Ok(None), 47 | }; 48 | 49 | if fti.len() != 16 { 50 | return Err(FluteError::new("Wrong extension size")); 51 | } 52 | 53 | debug_assert!(fti[0] == lct::Ext::Fti as u8); 54 | if fti[1] != 4 { 55 | return Err(FluteError::new(format!( 56 | "Wrong exten header size {} != 4 for FTI", 57 | fti[1] 58 | ))); 59 | } 60 | 61 | let transfer_length = u64::from_be_bytes(fti[2..10].as_ref().try_into().unwrap()) >> 16; 62 | let fec_instance_id = u16::from_be_bytes(fti[8..10].as_ref().try_into().unwrap()); 63 | let encoding_symbol_length = u16::from_be_bytes(fti[10..12].as_ref().try_into().unwrap()); 64 | let maximum_source_block_length = 65 | u16::from_be_bytes(fti[12..14].as_ref().try_into().unwrap()); 66 | let num_encoding_symbols = u16::from_be_bytes(fti[14..16].as_ref().try_into().unwrap()); 67 | 68 | let oti = oti::Oti { 69 | fec_encoding_id: oti::FECEncodingID::ReedSolomonGF28UnderSpecified, 70 | fec_instance_id, 71 | maximum_source_block_length: maximum_source_block_length as u32, 72 | encoding_symbol_length, 73 | max_number_of_parity_symbols: (num_encoding_symbols as u32).checked_sub(maximum_source_block_length as u32).unwrap_or_default(), 74 | scheme_specific: None, 75 | inband_fti: true, 76 | }; 77 | 78 | Ok(Some((oti, transfer_length))) 79 | } 80 | 81 | fn add_fec_payload_id(&self, data: &mut Vec, _oti: &oti::Oti, pkt: &pkt::Pkt) { 82 | /*0 1 2 3 83 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 84 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 85 | | Source Block Number | 86 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 87 | | Source Block Length | Encoding Symbol ID | 88 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 89 | */ 90 | let sbn = pkt.sbn; 91 | let source_block_length = pkt.source_block_length as u16; 92 | let esi = pkt.esi as u16; 93 | 94 | data.extend(sbn.to_be_bytes()); 95 | data.extend(source_block_length.to_be_bytes()); 96 | data.extend(esi.to_be_bytes()); 97 | } 98 | 99 | fn get_fec_payload_id( 100 | &self, 101 | pkt: &alc::AlcPkt, 102 | _oti: &oti::Oti, 103 | ) -> crate::error::Result { 104 | self.get_fec_inline_payload_id(pkt) 105 | } 106 | 107 | fn get_fec_inline_payload_id(&self, pkt: &alc::AlcPkt) -> crate::error::Result { 108 | let data = &pkt.data[pkt.data_alc_header_offset..pkt.data_payload_offset]; 109 | let arr: [u8; 8] = match data.try_into() { 110 | Ok(arr) => arr, 111 | Err(e) => return Err(FluteError::new(e.to_string())), 112 | }; 113 | 114 | /*0 1 2 3 115 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 116 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 117 | | Source Block Number | 118 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 119 | | Source Block Length | Encoding Symbol ID | 120 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 121 | */ 122 | let payload_id_header = u64::from_be_bytes(arr); 123 | let sbn = ((payload_id_header >> 32) & 0xFFFFFFFF) as u32; 124 | let source_block_length = ((payload_id_header >> 16) & 0xFFFF) as u32; 125 | let esi = ((payload_id_header) & 0xFFFF) as u32; 126 | 127 | Ok(alc::PayloadID { 128 | sbn, 129 | esi, 130 | source_block_length: Some(source_block_length), 131 | }) 132 | } 133 | 134 | fn fec_payload_id_block_length(&self) -> usize { 135 | 8 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/common/alccodec/alcrs2m.rs: -------------------------------------------------------------------------------- 1 | use super::AlcCodec; 2 | use crate::{ 3 | common::{ 4 | alc, lct, 5 | oti::{self, ReedSolomonGF2MSchemeSpecific, SchemeSpecific}, 6 | pkt, 7 | }, 8 | error::FluteError, 9 | }; 10 | 11 | pub struct AlcRS2m {} 12 | 13 | impl AlcCodec for AlcRS2m { 14 | fn add_fti(&self, data: &mut Vec, oti: &oti::Oti, transfer_length: u64) { 15 | /* 0 1 2 3 16 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 17 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 18 | | HET = 64 | HEL = 4 | | 19 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + 20 | | Transfer Length (L) | 21 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 22 | | m | G | Encoding Symbol Length (E) | 23 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 24 | | Max Source Block Length (B) | Max Nb Enc. Symbols (max_n) | 25 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+*/ 26 | 27 | if let SchemeSpecific::ReedSolomon(scheme_specific) = oti.scheme_specific.as_ref().unwrap() 28 | { 29 | let ext_header_l: u64 = 30 | (lct::Ext::Fti as u64) << 56 | 4u64 << 48 | transfer_length & 0xFFFFFFFFFFFF; 31 | 32 | let b = oti.maximum_source_block_length as u16; 33 | let max_n = (oti.max_number_of_parity_symbols + oti.maximum_source_block_length) as u16; 34 | 35 | data.extend(ext_header_l.to_be_bytes()); 36 | data.push(scheme_specific.m); 37 | data.push(scheme_specific.g); 38 | data.extend(oti.encoding_symbol_length.to_be_bytes()); 39 | data.extend(b.to_be_bytes()); 40 | data.extend(max_n.to_be_bytes()); 41 | lct::inc_hdr_len(data, 4); 42 | } else { 43 | debug_assert!(false); 44 | } 45 | } 46 | 47 | fn get_fti( 48 | &self, 49 | data: &[u8], 50 | lct_header: &lct::LCTHeader, 51 | ) -> crate::error::Result> { 52 | let fti = match lct::get_ext(data, lct_header, lct::Ext::Fti as u8)? { 53 | Some(fti) => fti, 54 | None => return Ok(None), 55 | }; 56 | 57 | if fti.len() != 16 { 58 | return Err(FluteError::new("Wrong extension size")); 59 | } 60 | 61 | debug_assert!(fti[0] == lct::Ext::Fti as u8); 62 | if fti[1] != 4 { 63 | return Err(FluteError::new("Wrong extension")); 64 | } 65 | 66 | let transfer_length = 67 | u64::from_be_bytes(fti[0..8].as_ref().try_into().unwrap()) & 0xFFFFFFFFFFFF; 68 | let m = fti[8]; 69 | let g = fti[9]; 70 | let encoding_symbol_length = u16::from_be_bytes(fti[10..12].as_ref().try_into().unwrap()); 71 | let b = u16::from_be_bytes(fti[12..14].as_ref().try_into().unwrap()); 72 | let max_n = u16::from_be_bytes(fti[14..16].as_ref().try_into().unwrap()); 73 | let max_number_of_parity_symbols = (max_n as u32).saturating_sub(b as u32); 74 | 75 | let oti = oti::Oti { 76 | fec_encoding_id: oti::FECEncodingID::ReedSolomonGF2M, 77 | fec_instance_id: 0, 78 | maximum_source_block_length: b as u32, 79 | encoding_symbol_length, 80 | max_number_of_parity_symbols, 81 | scheme_specific: Some(SchemeSpecific::ReedSolomon(ReedSolomonGF2MSchemeSpecific { 82 | g: match g { 83 | 0 => 1, 84 | g => g, 85 | }, 86 | m: match m { 87 | 0 => 8, 88 | m => m, 89 | }, 90 | })), 91 | inband_fti: true, 92 | }; 93 | 94 | Ok(Some((oti, transfer_length))) 95 | } 96 | 97 | fn add_fec_payload_id(&self, data: &mut Vec, oti: &oti::Oti, pkt: &pkt::Pkt) { 98 | let m = oti 99 | .scheme_specific 100 | .as_ref() 101 | .map(|f| match f { 102 | SchemeSpecific::ReedSolomon(s) => s.m, 103 | _ => 8, 104 | }) 105 | .unwrap_or(8); 106 | 107 | let sbn = pkt.sbn; 108 | let esi = pkt.esi; 109 | 110 | /* 111 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 112 | | Source Block Number (32-m | Enc. Symb. ID | 113 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 114 | */ 115 | let header: u32 = (sbn << m) | esi & 0xFF; 116 | data.extend(header.to_be_bytes()); 117 | } 118 | 119 | fn get_fec_payload_id( 120 | &self, 121 | pkt: &alc::AlcPkt, 122 | oti: &oti::Oti, 123 | ) -> crate::error::Result { 124 | /* 125 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 126 | | Source Block Number (32-m | Enc. Symb. ID | 127 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 128 | */ 129 | let data = &pkt.data[pkt.data_alc_header_offset..pkt.data_payload_offset]; 130 | let arr: [u8; 4] = match data.try_into() { 131 | Ok(arr) => arr, 132 | Err(e) => return Err(FluteError::new(e.to_string())), 133 | }; 134 | let payload_id_header = u32::from_be_bytes(arr); 135 | 136 | let m = oti 137 | .scheme_specific 138 | .as_ref() 139 | .map(|f| match f { 140 | SchemeSpecific::ReedSolomon(s) => s.m, 141 | _ => 8, 142 | }) 143 | .unwrap_or(8); 144 | 145 | let sbn = payload_id_header >> m; 146 | let esi_mask = (1u32 << m) - 1u32; 147 | let esi = payload_id_header & esi_mask; 148 | 149 | Ok(alc::PayloadID { 150 | esi, 151 | sbn, 152 | source_block_length: None, 153 | }) 154 | } 155 | 156 | fn get_fec_inline_payload_id( 157 | &self, 158 | _pkt: &alc::AlcPkt, 159 | ) -> crate::error::Result { 160 | Err(FluteError::new("not supported")) 161 | } 162 | 163 | fn fec_payload_id_block_length(&self) -> usize { 164 | 4 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/receiver/blockwriter.rs: -------------------------------------------------------------------------------- 1 | use std::time::SystemTime; 2 | 3 | use base64::Engine; 4 | 5 | use crate::error::{FluteError, Result}; 6 | 7 | use crate::common::lct; 8 | 9 | use super::{ 10 | blockdecoder::BlockDecoder, 11 | uncompress::{Decompress, DecompressGzip}, 12 | uncompress::{DecompressDeflate, DecompressZlib}, 13 | writer::ObjectWriter, 14 | }; 15 | 16 | pub struct BlockWriter { 17 | sbn: u32, 18 | bytes_left: usize, 19 | content_length_left: Option, 20 | cenc: lct::Cenc, 21 | decoder: Option>, 22 | buffer: Vec, 23 | md5_context: Option, 24 | md5: Option, 25 | } 26 | 27 | impl std::fmt::Debug for BlockWriter { 28 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 29 | f.debug_struct("BlockWriter") 30 | .field("sbn", &self.sbn) 31 | .field("bytes_left", &self.bytes_left) 32 | .field("cenc", &self.cenc) 33 | .field("decoder", &self.decoder) 34 | .field("buffer", &self.buffer) 35 | .field("md5_context", &self.md5_context.is_some()) 36 | .field("md5", &self.md5) 37 | .finish() 38 | } 39 | } 40 | 41 | impl BlockWriter { 42 | pub fn new( 43 | transfer_length: usize, 44 | content_length: Option, 45 | cenc: lct::Cenc, 46 | md5: bool, 47 | ) -> BlockWriter { 48 | BlockWriter { 49 | sbn: 0, 50 | bytes_left: transfer_length, 51 | content_length_left: content_length, 52 | cenc, 53 | decoder: None, 54 | buffer: Vec::new(), 55 | md5_context: match md5 { 56 | true => Some(md5::Context::new()), 57 | false => None, 58 | }, 59 | md5: None, 60 | } 61 | } 62 | 63 | pub fn check_md5(&self, md5: &str) -> bool { 64 | self.md5.as_ref().map(|m| m.eq(md5)).unwrap_or(true) 65 | } 66 | 67 | pub fn get_md5(&self) -> Option<&str> { 68 | self.md5.as_deref() 69 | } 70 | 71 | pub fn write( 72 | &mut self, 73 | sbn: u32, 74 | block: &BlockDecoder, 75 | writer: &dyn ObjectWriter, 76 | now: SystemTime, 77 | ) -> Result { 78 | if self.sbn != sbn { 79 | return Ok(false); 80 | } 81 | debug_assert!(block.completed); 82 | let data = block.source_block()?; 83 | 84 | // Detect the size of the last symbol 85 | let data = match self.bytes_left > data.len() { 86 | true => data, 87 | false => &data[..self.bytes_left], 88 | }; 89 | 90 | if self.cenc == lct::Cenc::Null { 91 | self.write_pkt_cenc_null(data, writer, now)?; 92 | } else { 93 | self.decode_write_pkt(data, writer, now)?; 94 | } 95 | 96 | debug_assert!(data.len() <= self.bytes_left); 97 | self.bytes_left -= data.len(); 98 | 99 | self.sbn += 1; 100 | 101 | if self.is_completed() { 102 | // All blocks have been received -> flush the decoder 103 | if self.decoder.is_some() { 104 | self.decoder.as_mut().unwrap().finish(); 105 | self.decoder_read(writer, now)?; 106 | } 107 | 108 | let output = self.md5_context.take().map(|ctx| ctx.finalize().0); 109 | self.md5 = 110 | output.map(|output| base64::engine::general_purpose::STANDARD.encode(output)); 111 | } 112 | 113 | Ok(true) 114 | } 115 | 116 | fn init_decoder(&mut self, data: &[u8]) { 117 | debug_assert!(self.decoder.is_none()); 118 | self.decoder = match self.cenc { 119 | lct::Cenc::Null => None, 120 | lct::Cenc::Zlib => Some(Box::new(DecompressZlib::new(data))), 121 | lct::Cenc::Deflate => Some(Box::new(DecompressDeflate::new(data))), 122 | lct::Cenc::Gzip => Some(Box::new(DecompressGzip::new(data))), 123 | }; 124 | self.buffer.resize(data.len(), 0); 125 | } 126 | 127 | fn write_pkt_cenc_null( 128 | &mut self, 129 | data: &[u8], 130 | writer: &dyn ObjectWriter, 131 | now: SystemTime, 132 | ) -> Result<()> { 133 | if let Some(ctx) = self.md5_context.as_mut() { 134 | ctx.consume(data) 135 | } 136 | writer.write(self.sbn, data, now) 137 | } 138 | 139 | fn decode_write_pkt( 140 | &mut self, 141 | pkt: &[u8], 142 | writer: &dyn ObjectWriter, 143 | now: SystemTime, 144 | ) -> Result<()> { 145 | if self.decoder.is_none() { 146 | self.init_decoder(pkt); 147 | self.decoder_read(writer, now)?; 148 | return Ok(()); 149 | } 150 | 151 | let mut offset: usize = 0; 152 | loop { 153 | let size = self.decoder.as_mut().unwrap().write(&pkt[offset..])?; 154 | self.decoder_read(writer, now)?; 155 | offset += size; 156 | if offset == pkt.len() { 157 | break; 158 | } 159 | } 160 | Ok(()) 161 | } 162 | 163 | fn decoder_read(&mut self, writer: &dyn ObjectWriter, now: SystemTime) -> Result<()> { 164 | let decoder = self.decoder.as_mut().unwrap(); 165 | 166 | if self.content_length_left == Some(0) { 167 | return Ok(()); 168 | } 169 | 170 | loop { 171 | let size = match decoder.read(&mut self.buffer) { 172 | Ok(res) => res, 173 | Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => return Ok(()), 174 | Err(e) => return Err(FluteError::new(e)), 175 | }; 176 | 177 | if size == 0 { 178 | return Ok(()); 179 | } 180 | 181 | if let Some(ctx) = self.md5_context.as_mut() { 182 | ctx.consume(&self.buffer[..size]) 183 | } 184 | 185 | writer.write(self.sbn, &self.buffer[..size], now)?; 186 | 187 | if let Some(content_length_left) = self.content_length_left.as_mut() { 188 | *content_length_left = content_length_left.saturating_sub(size); 189 | if *content_length_left == 0 { 190 | return Ok(()); 191 | } 192 | } 193 | } 194 | } 195 | 196 | pub fn left(&self) -> usize { 197 | self.bytes_left 198 | } 199 | 200 | pub fn is_completed(&self) -> bool { 201 | self.bytes_left == 0 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /src/receiver/writer/objectwriterfs.rs: -------------------------------------------------------------------------------- 1 | use super::{ObjectMetadata, ObjectWriter, ObjectWriterBuilder, ObjectWriterBuilderResult}; 2 | use crate::{ 3 | common::udpendpoint::UDPEndpoint, 4 | error::{FluteError, Result}, 5 | }; 6 | use std::{cell::RefCell, io::Write, time::SystemTime}; 7 | 8 | /// 9 | /// Write objects received by the `receiver` to a filesystem 10 | /// 11 | #[derive(Debug)] 12 | pub struct ObjectWriterFSBuilder { 13 | dest: std::path::PathBuf, 14 | enable_md5_check: bool, 15 | } 16 | 17 | impl ObjectWriterFSBuilder { 18 | /// Return a new `ObjectWriterBuffer` 19 | pub fn new(dest: &std::path::Path, enable_md5_check: bool) -> Result { 20 | if !dest.is_dir() { 21 | return Err(FluteError::new(format!("{:?} is not a directory", dest))); 22 | } 23 | 24 | Ok(ObjectWriterFSBuilder { 25 | dest: dest.to_path_buf(), 26 | enable_md5_check, 27 | }) 28 | } 29 | } 30 | 31 | impl ObjectWriterBuilder for ObjectWriterFSBuilder { 32 | fn new_object_writer( 33 | &self, 34 | _endpoint: &UDPEndpoint, 35 | _tsi: &u64, 36 | _toi: &u128, 37 | meta: &ObjectMetadata, 38 | _now: std::time::SystemTime, 39 | ) -> ObjectWriterBuilderResult { 40 | ObjectWriterBuilderResult::StoreObject(Box::new(ObjectWriterFS { 41 | dest: self.dest.clone(), 42 | inner: RefCell::new(ObjectWriterFSInner { 43 | destination: None, 44 | writer: None, 45 | }), 46 | meta: meta.clone(), 47 | enable_md5_check: self.enable_md5_check, 48 | })) 49 | } 50 | 51 | fn update_cache_control( 52 | &self, 53 | _endpoint: &UDPEndpoint, 54 | _tsi: &u64, 55 | _toi: &u128, 56 | _meta: &ObjectMetadata, 57 | _now: std::time::SystemTime, 58 | ) { 59 | } 60 | 61 | fn fdt_received( 62 | &self, 63 | _endpoint: &UDPEndpoint, 64 | _tsi: &u64, 65 | _fdt_xml: &str, 66 | _expires: std::time::SystemTime, 67 | _meta: &ObjectMetadata, 68 | _transfer_duration: std::time::Duration, 69 | _now: std::time::SystemTime, 70 | _ext_time: Option, 71 | ) { 72 | } 73 | } 74 | 75 | /// 76 | /// Write an object to a file system. 77 | /// Uses the content-location to create the destination path of the object. 78 | /// If the destination path does not exists, the folder hierarchy is created. 79 | /// Existing files will be overwritten by this object. 80 | /// 81 | #[derive(Debug)] 82 | pub struct ObjectWriterFS { 83 | /// Folder destination were the object will be written 84 | dest: std::path::PathBuf, 85 | inner: RefCell, 86 | meta: ObjectMetadata, 87 | enable_md5_check: bool, 88 | } 89 | 90 | /// 91 | /// 92 | #[derive(Debug)] 93 | pub struct ObjectWriterFSInner { 94 | destination: Option, 95 | writer: Option>, 96 | } 97 | 98 | impl ObjectWriter for ObjectWriterFS { 99 | fn open(&self, _now: SystemTime) -> Result<()> { 100 | let url = url::Url::parse(&self.meta.content_location); 101 | let content_location_path = match &url { 102 | Ok(url) => url.path(), 103 | Err(e) => match e { 104 | url::ParseError::RelativeUrlWithoutBase => &self.meta.content_location, 105 | url::ParseError::RelativeUrlWithCannotBeABaseBase => &self.meta.content_location, 106 | _ => { 107 | log::error!( 108 | "Fail to parse content location {:?} {:?}", 109 | self.meta.content_location, 110 | e 111 | ); 112 | return Err(FluteError::new(format!( 113 | "Fail to parse content location {:?} {:?}", 114 | self.meta.content_location, e 115 | ))); 116 | } 117 | }, 118 | }; 119 | let relative_path = content_location_path 120 | .strip_prefix('/') 121 | .unwrap_or(content_location_path); 122 | let destination = self.dest.join(relative_path); 123 | log::info!( 124 | "Create destination {:?} {:?} {:?}", 125 | self.dest, 126 | relative_path, 127 | destination 128 | ); 129 | let parent = destination.parent(); 130 | if parent.is_some() { 131 | let parent = parent.unwrap(); 132 | if !parent.is_dir() { 133 | std::fs::create_dir_all(parent)?; 134 | } 135 | } 136 | 137 | let file = std::fs::File::create(&destination)?; 138 | let mut inner = self.inner.borrow_mut(); 139 | inner.writer = Some(std::io::BufWriter::new(file)); 140 | inner.destination = Some(destination.to_path_buf()); 141 | Ok(()) 142 | } 143 | 144 | fn write(&self, _sbn: u32, data: &[u8], _now: SystemTime) -> Result<()> { 145 | let mut inner = self.inner.borrow_mut(); 146 | if inner.writer.is_none() { 147 | return Ok(()); 148 | } 149 | inner 150 | .writer 151 | .as_mut() 152 | .unwrap() 153 | .write_all(data) 154 | .map_err(|e| { 155 | log::error!("Fail to write data to file {:?} {:?}", inner.destination, e); 156 | FluteError::new(format!( 157 | "Fail to write data to file {:?} {:?}", 158 | inner.destination, e 159 | )) 160 | })?; 161 | Ok(()) 162 | } 163 | 164 | fn complete(&self, _now: SystemTime) { 165 | let mut inner = self.inner.borrow_mut(); 166 | if inner.writer.is_none() { 167 | return; 168 | } 169 | 170 | println!("File {:?} is completed !", inner.destination); 171 | inner.writer.as_mut().unwrap().flush().ok(); 172 | inner.writer = None; 173 | inner.destination = None 174 | } 175 | 176 | fn error(&self, _now: SystemTime) { 177 | let mut inner = self.inner.borrow_mut(); 178 | inner.writer = None; 179 | if inner.destination.is_some() { 180 | log::error!("Remove file {:?}", inner.destination); 181 | std::fs::remove_file(inner.destination.as_ref().unwrap()).ok(); 182 | inner.destination = None; 183 | } 184 | } 185 | 186 | fn interrupted(&self, now: SystemTime) { 187 | self.error(now); 188 | } 189 | 190 | fn enable_md5_check(&self) -> bool { 191 | self.enable_md5_check 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /src/sender/compress.rs: -------------------------------------------------------------------------------- 1 | //! This module provides functions for compressing data using various compression algorithms. 2 | //! 3 | //! The supported compression algorithms are: 4 | //! - Zlib 5 | //! - Deflate 6 | //! - Gzip 7 | //! 8 | //! The module offers two main functions for compression: 9 | //! - `compress_buffer`: Compresses a byte slice and returns the compressed data as a vector of bytes. 10 | //! - `compress_stream`: Compresses data from an input stream and writes the compressed data to an output stream. 11 | //! 12 | //! # Examples 13 | //! 14 | //! ## Compressing a Byte Slice 15 | //! 16 | //! ```rust 17 | //! use flute::sender::compress::compress_buffer; 18 | //! use flute::core::lct::Cenc; 19 | //! 20 | //! let data = b"example data"; 21 | //! let compressed_data = compress_buffer(data, Cenc::Gzip).expect("Compression failed"); 22 | //! ``` 23 | //! 24 | //! ## Compressing Data from a Stream 25 | //! 26 | //! ```rust 27 | //! use flute::sender::compress::compress_stream; 28 | //! use flute::core::lct::Cenc; 29 | //! use std::io::Cursor; 30 | //! 31 | //! let data = b"example data"; 32 | //! let mut input = Cursor::new(data); 33 | //! let mut output = Vec::new(); 34 | //! compress_stream(&mut input, Cenc::Gzip, &mut output).expect("Compression failed"); 35 | //! ``` 36 | 37 | use crate::common::lct; 38 | use crate::tools::error::{FluteError, Result}; 39 | use flate2::write::{DeflateEncoder, GzEncoder, ZlibEncoder}; 40 | use std::io::Write; 41 | 42 | /// Compresses the given data using the specified compression encoding. 43 | /// 44 | /// # Arguments 45 | /// 46 | /// * `data` - A byte slice containing the data to be compressed. 47 | /// * `cenc` - The compression encoding to use. This can be one of the following: 48 | /// - `lct::Cenc::Null`: No compression (returns an error). 49 | /// - `lct::Cenc::Zlib`: Compress using the Zlib algorithm. 50 | /// - `lct::Cenc::Deflate`: Compress using the Deflate algorithm. 51 | /// - `lct::Cenc::Gzip`: Compress using the Gzip algorithm. 52 | /// 53 | /// # Returns 54 | /// 55 | /// A `Result` containing a vector of compressed bytes on success, or a `FluteError` on failure. 56 | /// 57 | /// # Errors 58 | /// 59 | /// This function will return an error if the specified compression encoding is `lct::Cenc::Null` 60 | /// or if there is an issue during the compression process. 61 | /// 62 | /// # Examples 63 | /// 64 | /// ``` 65 | /// use flute::sender::compress::compress_buffer; 66 | /// use flute::core::lct::Cenc; 67 | /// 68 | /// let data = b"example data"; 69 | /// let compressed_data = compress_buffer(data, Cenc::Gzip).expect("Compression failed"); 70 | /// ``` 71 | pub fn compress_buffer(data: &[u8], cenc: lct::Cenc) -> Result> { 72 | match cenc { 73 | lct::Cenc::Null => Err(FluteError::new("Null compression ?")), 74 | lct::Cenc::Zlib => compress_zlib(data), 75 | lct::Cenc::Deflate => compress_deflate(data), 76 | lct::Cenc::Gzip => compress_gzip(data), 77 | } 78 | } 79 | 80 | /// Compresses data from an input stream and writes the compressed data to an output stream 81 | /// using the specified compression encoding. 82 | /// 83 | /// # Arguments 84 | /// 85 | /// * `input` - A mutable reference to a type that implements the `std::io::Read` trait, representing the input stream. 86 | /// * `cenc` - The compression encoding to use. This can be one of the following: 87 | /// - `lct::Cenc::Null`: No compression (returns an error). 88 | /// - `lct::Cenc::Zlib`: Compress using the Zlib algorithm. 89 | /// - `lct::Cenc::Deflate`: Compress using the Deflate algorithm. 90 | /// - `lct::Cenc::Gzip`: Compress using the Gzip algorithm. 91 | /// * `output` - A mutable reference to a type that implements the `std::io::Write` trait, representing the output stream. 92 | /// 93 | /// # Returns 94 | /// 95 | /// A `Result` containing `()` on success, or a `FluteError` on failure. 96 | /// 97 | /// # Errors 98 | /// 99 | /// This function will return an error if the specified compression encoding is `lct::Cenc::Null` 100 | /// or if there is an issue during the compression process. 101 | /// ``` 102 | pub fn compress_stream( 103 | input: &mut dyn std::io::Read, 104 | cenc: lct::Cenc, 105 | output: &mut dyn std::io::Write, 106 | ) -> Result<()> { 107 | match cenc { 108 | lct::Cenc::Null => Err(FluteError::new("Null compression ?")), 109 | lct::Cenc::Zlib => stream_compress_zlib(input, output), 110 | lct::Cenc::Deflate => stream_compress_deflate(input, output), 111 | lct::Cenc::Gzip => stream_compress_gzip(input, output), 112 | } 113 | } 114 | 115 | fn compress_gzip(data: &[u8]) -> Result> { 116 | log::debug!("Create GZIP encoder"); 117 | let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); 118 | encoder.write_all(data)?; 119 | let output = encoder.finish()?; 120 | Ok(output) 121 | } 122 | 123 | fn compress_deflate(data: &[u8]) -> Result> { 124 | let mut encoder = DeflateEncoder::new(Vec::new(), flate2::Compression::default()); 125 | encoder.write_all(data)?; 126 | let output = encoder.finish()?; 127 | Ok(output) 128 | } 129 | 130 | fn compress_zlib(data: &[u8]) -> Result> { 131 | let mut encoder = ZlibEncoder::new(Vec::new(), flate2::Compression::default()); 132 | encoder.write_all(data)?; 133 | let output = encoder.finish()?; 134 | Ok(output) 135 | } 136 | 137 | fn stream_compress_gzip( 138 | input: &mut dyn std::io::Read, 139 | output: &mut dyn std::io::Write, 140 | ) -> Result<()> { 141 | log::debug!("Create GZIP encoder"); 142 | let mut encoder = GzEncoder::new(output, flate2::Compression::default()); 143 | let mut buffer = vec![0; 1024 * 1024]; 144 | loop { 145 | let read = input.read(&mut buffer)?; 146 | if read == 0 { 147 | break; 148 | } 149 | encoder.write_all(&buffer[..read])?; 150 | } 151 | 152 | encoder.finish()?; 153 | Ok(()) 154 | } 155 | 156 | fn stream_compress_zlib( 157 | input: &mut dyn std::io::Read, 158 | output: &mut dyn std::io::Write, 159 | ) -> Result<()> { 160 | log::debug!("Create GZIP encoder"); 161 | let mut encoder = ZlibEncoder::new(output, flate2::Compression::default()); 162 | let mut buffer = vec![0; 1024 * 1024]; 163 | loop { 164 | let read = input.read(&mut buffer)?; 165 | if read == 0 { 166 | break; 167 | } 168 | encoder.write_all(&buffer[..read])?; 169 | } 170 | 171 | encoder.finish()?; 172 | Ok(()) 173 | } 174 | 175 | fn stream_compress_deflate( 176 | input: &mut dyn std::io::Read, 177 | output: &mut dyn std::io::Write, 178 | ) -> Result<()> { 179 | log::debug!("Create GZIP encoder"); 180 | let mut encoder = DeflateEncoder::new(output, flate2::Compression::default()); 181 | let mut buffer = vec![0; 1024 * 1024]; 182 | loop { 183 | let read = input.read(&mut buffer)?; 184 | if read == 0 { 185 | break; 186 | } 187 | encoder.write_all(&buffer[..read])?; 188 | } 189 | 190 | encoder.finish()?; 191 | Ok(()) 192 | } 193 | -------------------------------------------------------------------------------- /src/tools/ringbuffer.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug)] 2 | pub struct RingBuffer { 3 | buffer: Vec, 4 | producer: usize, 5 | consumer: usize, 6 | finish: bool, 7 | } 8 | 9 | impl RingBuffer { 10 | pub fn new(size: usize) -> Self { 11 | let buffer = vec![0; size]; 12 | Self { 13 | buffer, 14 | producer: 0, 15 | consumer: 0, 16 | finish: false, 17 | } 18 | } 19 | 20 | pub fn finish(&mut self) { 21 | self.finish = true; 22 | } 23 | 24 | fn write_size(&self) -> usize { 25 | if self.producer < self.consumer { 26 | return self.consumer - self.producer - 1; 27 | } 28 | 29 | self.buffer.len() - self.producer + self.consumer - 1 30 | } 31 | 32 | fn read_size(&self) -> usize { 33 | if self.consumer <= self.producer { 34 | return self.producer - self.consumer; 35 | } 36 | 37 | self.buffer.len() - self.consumer + self.producer 38 | } 39 | } 40 | 41 | impl std::io::Read for RingBuffer { 42 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 43 | let mut max_size = self.read_size(); 44 | if max_size > buf.len() { 45 | max_size = buf.len(); 46 | } 47 | 48 | if max_size == 0 { 49 | if self.finish { 50 | return Ok(0); 51 | } 52 | return Err(std::io::Error::new( 53 | std::io::ErrorKind::WouldBlock, 54 | "waiting for more data", 55 | )); 56 | } 57 | 58 | if self.consumer < self.producer { 59 | buf[..max_size] 60 | .copy_from_slice(&self.buffer[self.consumer..(self.consumer + max_size)]); 61 | self.consumer += max_size; 62 | return Ok(max_size); 63 | } 64 | 65 | let end_size = self.buffer.len() - self.consumer; 66 | if end_size >= max_size { 67 | buf[..max_size] 68 | .copy_from_slice(&self.buffer[self.consumer..(self.consumer + max_size)]); 69 | self.consumer += max_size; 70 | debug_assert!(self.consumer <= self.buffer.len()); 71 | 72 | if self.consumer == self.buffer.len() { 73 | self.consumer = 0; 74 | } 75 | 76 | return Ok(max_size); 77 | } 78 | 79 | buf[..end_size].copy_from_slice(&self.buffer[self.consumer..(self.consumer + end_size)]); 80 | let left = max_size - end_size; 81 | self.consumer = 0; 82 | 83 | buf[end_size..(end_size + left)].copy_from_slice(&self.buffer[..left]); 84 | self.consumer += left; 85 | debug_assert!(self.consumer <= self.producer); 86 | debug_assert!(self.consumer != self.buffer.len()); 87 | Ok(max_size) 88 | } 89 | } 90 | 91 | impl std::io::Write for RingBuffer { 92 | fn write(&mut self, buf: &[u8]) -> std::io::Result { 93 | let mut max_size = self.write_size(); 94 | if max_size == 0 { 95 | return Ok(0); 96 | } 97 | 98 | if max_size > buf.len() { 99 | max_size = buf.len(); 100 | } 101 | 102 | if self.consumer > self.producer { 103 | self.buffer[self.producer..(self.producer + max_size)] 104 | .copy_from_slice(&buf[..max_size]); 105 | 106 | self.producer += max_size; 107 | debug_assert!(self.consumer > self.producer); 108 | debug_assert!(self.producer != self.buffer.len()); 109 | return Ok(max_size); 110 | } 111 | 112 | let end_size = self.buffer.len() - self.producer; 113 | if end_size >= max_size { 114 | self.buffer[self.producer..(self.producer + max_size)] 115 | .copy_from_slice(&buf[..max_size]); 116 | self.producer += max_size; 117 | 118 | if self.producer == self.buffer.len() { 119 | self.producer = 0; 120 | } 121 | 122 | return Ok(max_size); 123 | } 124 | 125 | self.buffer[self.producer..(self.producer + end_size)].copy_from_slice(&buf[..end_size]); 126 | self.producer = 0; 127 | let left_size = max_size - end_size; 128 | self.buffer[..left_size].copy_from_slice(&buf[end_size..(end_size + left_size)]); 129 | self.producer += left_size; 130 | debug_assert!(self.producer < self.consumer); 131 | Ok(max_size) 132 | } 133 | 134 | fn flush(&mut self) -> std::io::Result<()> { 135 | Ok(()) 136 | } 137 | } 138 | 139 | #[cfg(test)] 140 | mod tests { 141 | 142 | use std::io::Read; 143 | use std::io::Write; 144 | 145 | #[test] 146 | pub fn ringbuffer() { 147 | crate::tests::init(); 148 | const RING_SIZE: usize = 1024; 149 | let mut ring = super::RingBuffer::new(RING_SIZE); 150 | 151 | let mut buffer: Vec = vec![0; RING_SIZE / 3]; 152 | 153 | assert!(ring.write_size() < RING_SIZE); 154 | 155 | let wsize = ring.write(buffer.as_ref()).unwrap(); 156 | assert!(wsize == buffer.len()); 157 | 158 | let rsize = ring.read_size(); 159 | assert!(rsize == buffer.len()); 160 | 161 | let rsize = ring.read(buffer.as_mut()).unwrap(); 162 | assert!(rsize == buffer.len()); 163 | assert!(ring.read_size() == 0); 164 | 165 | ring.write(&buffer[0..buffer.len()]).unwrap(); 166 | ring.write(&buffer[0..buffer.len() / 2]).unwrap(); 167 | 168 | let rsize = ring.read(buffer.as_mut()).unwrap(); 169 | assert!(rsize == buffer.len()); 170 | 171 | loop { 172 | let wsize = ring.write(buffer.as_ref()).unwrap(); 173 | if wsize == 0 { 174 | break; 175 | } 176 | } 177 | } 178 | 179 | #[test] 180 | pub fn ringbuffer_2() { 181 | crate::tests::init(); 182 | const RING_SIZE: usize = 3; 183 | let mut buffer: Vec = vec![0; 1024]; 184 | let mut ring = super::RingBuffer::new(RING_SIZE); 185 | 186 | for _ in 0..10 { 187 | let wsize = ring.write(buffer.as_ref()).unwrap(); 188 | assert!(wsize == RING_SIZE - 1); 189 | 190 | let rsize = ring.read_size(); 191 | assert!(rsize == RING_SIZE - 1); 192 | 193 | let rsize = ring.read(buffer.as_mut()).unwrap(); 194 | assert!(rsize == RING_SIZE - 1); 195 | 196 | let wouldblock = ring.read(buffer.as_mut()); 197 | assert!(wouldblock.is_err()); 198 | assert!(wouldblock.err().unwrap().kind() == std::io::ErrorKind::WouldBlock); 199 | } 200 | } 201 | 202 | #[test] 203 | pub fn ringbuffer_3() { 204 | crate::tests::init(); 205 | const RING_SIZE: usize = 11; 206 | let mut wbuffer: Vec = vec![0xAA; 9]; 207 | let mut rbuffer: Vec = vec![0; 1]; 208 | 209 | let mut ring = super::RingBuffer::new(RING_SIZE); 210 | 211 | ring.write(wbuffer.as_ref()).unwrap(); 212 | ring.read(wbuffer.as_mut()).unwrap(); 213 | assert!(ring.read_size() == 0); 214 | 215 | ring.write(wbuffer.as_ref()).unwrap(); 216 | 217 | for _ in 0..25 { 218 | ring.write(wbuffer.as_ref()).unwrap(); 219 | ring.flush().unwrap(); 220 | loop { 221 | match ring.read(rbuffer.as_mut()) { 222 | Ok(res) => { 223 | assert!(res == 1); 224 | assert!(rbuffer[0] == 0xAA); 225 | } 226 | Err(e) => { 227 | assert!(e.kind() == std::io::ErrorKind::WouldBlock); 228 | break; 229 | } 230 | } 231 | } 232 | rbuffer.fill(0); 233 | } 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /src/receiver/writer/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! Write FLUTE objects to their final destination 3 | //! 4 | //! # Example 5 | //! 6 | //! ``` 7 | //! use flute::receiver::writer; 8 | //! 9 | //! let enable_md5_check = true; 10 | //! let writer = writer::ObjectWriterFSBuilder::new(&std::path::Path::new("./destination_dir"), enable_md5_check).ok(); 11 | //! ``` 12 | //! 13 | 14 | use std::collections::HashMap; 15 | use std::time::Duration; 16 | use std::time::SystemTime; 17 | 18 | use crate::common::udpendpoint::UDPEndpoint; 19 | use crate::core::lct::Cenc; 20 | use crate::core::Oti; 21 | use crate::tools::error::Result; 22 | 23 | /// 24 | /// Cache-Duration for an object. 25 | /// 26 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] 27 | pub enum ObjectCacheControl { 28 | /// No cache duration, the object should not be cached 29 | NoCache, 30 | /// Should be cached permanently 31 | MaxStale, 32 | /// Cache duration with a specific time 33 | ExpiresAt(SystemTime), 34 | /// When hint when there is not cache-control directive for this object 35 | ExpiresAtHint(SystemTime), 36 | } 37 | 38 | impl ObjectCacheControl { 39 | /// Hint if the cache control should be updated 40 | pub fn should_update(&self, cache_control: ObjectCacheControl) -> bool { 41 | match self { 42 | ObjectCacheControl::NoCache => cache_control != ObjectCacheControl::NoCache, 43 | ObjectCacheControl::MaxStale => cache_control != ObjectCacheControl::MaxStale, 44 | ObjectCacheControl::ExpiresAt(expires_at) => { 45 | if let ObjectCacheControl::ExpiresAt(d) = cache_control { 46 | let diff = match d < *expires_at { 47 | true => expires_at.duration_since(d).unwrap_or_default(), 48 | false => d.duration_since(*expires_at).unwrap_or_default(), 49 | }; 50 | return diff > std::time::Duration::from_secs(1); 51 | } 52 | 53 | true 54 | } 55 | ObjectCacheControl::ExpiresAtHint(expires_at) => { 56 | if let ObjectCacheControl::ExpiresAtHint(d) = cache_control { 57 | let diff = match d < *expires_at { 58 | true => expires_at.duration_since(d).unwrap_or_default(), 59 | false => d.duration_since(*expires_at).unwrap_or_default(), 60 | }; 61 | return diff > std::time::Duration::from_secs(1); 62 | } 63 | 64 | true 65 | } 66 | } 67 | } 68 | } 69 | 70 | /// 71 | /// Struct representing metadata for an object. 72 | /// 73 | #[derive(Debug, Clone)] 74 | pub struct ObjectMetadata { 75 | /// URI that can be used as an identifier for this object 76 | pub content_location: String, 77 | /// Final size of this object 78 | pub content_length: Option, 79 | /// Transfer length (compressed) of this object 80 | pub transfer_length: Option, 81 | /// The content type of this object. 82 | /// This field describes the format of the object's content, 83 | /// and can be used to determine how to handle or process the object. 84 | pub content_type: Option, 85 | /// Object Cache Control 86 | pub cache_control: ObjectCacheControl, 87 | /// List of groups 88 | pub groups: Option>, 89 | /// Object MD5 90 | pub md5: Option, 91 | /// Opentelemetry propagation context 92 | pub optel_propagator: Option>, 93 | /// Object Transmission Information (OTI) of the received object 94 | pub oti: Option, 95 | /// CENC information 96 | pub cenc: Option, 97 | /// ETag 98 | pub e_tag: Option, 99 | } 100 | 101 | /// 102 | /// Represents the result when the creation of an `ObjectWriter` is requested. 103 | /// 104 | #[derive(Debug)] 105 | pub enum ObjectWriterBuilderResult { 106 | /// Indicates that the object must be stored using the provided writer. 107 | StoreObject(Box), 108 | /// Indicates that the object has already been received and does not need to be stored again. 109 | ObjectAlreadyReceived, 110 | /// Indicates that an error occurred and the object cannot be stored. 111 | Abort, 112 | } 113 | 114 | /// 115 | /// A trait for building an `ObjectWriter` 116 | /// 117 | pub trait ObjectWriterBuilder { 118 | /// Return a new object writer that will be used to store the received object to its final destination 119 | fn new_object_writer( 120 | &self, 121 | endpoint: &UDPEndpoint, 122 | tsi: &u64, 123 | toi: &u128, 124 | meta: &ObjectMetadata, 125 | now: std::time::SystemTime, 126 | ) -> ObjectWriterBuilderResult; 127 | /// Triggered when the cache control of an object is updated 128 | fn update_cache_control( 129 | &self, 130 | endpoint: &UDPEndpoint, 131 | tsi: &u64, 132 | toi: &u128, 133 | meta: &ObjectMetadata, 134 | now: std::time::SystemTime, 135 | ); 136 | /// Called when an FDT is received 137 | fn fdt_received( 138 | &self, 139 | endpoint: &UDPEndpoint, 140 | tsi: &u64, 141 | fdt_xml: &str, 142 | expires: std::time::SystemTime, 143 | meta: &ObjectMetadata, 144 | transfer_duration: Duration, 145 | now: std::time::SystemTime, 146 | ext_time: Option, 147 | ); 148 | } 149 | 150 | /// 151 | /// A trait for writing an object to its final destination. 152 | /// 153 | pub trait ObjectWriter { 154 | /// Open the destination 155 | /// 156 | /// Returns `Ok(())` if the destination is opened successfully, or an error if it fails. 157 | fn open(&self, now: SystemTime) -> Result<()>; 158 | /// Writes a data block associated with a specific Source Block Number (SBN). 159 | /// 160 | /// # Arguments 161 | /// 162 | /// * `sbn` - The Source Block Number identifying the data block's origin or sequence. 163 | /// * `data` - A byte slice representing the content to be written. 164 | /// * `now` - The current system time, typically used for timestamping or aging logic. 165 | /// 166 | /// Returns `Ok(())` if the destination is opened successfully, or an error if it fails. 167 | /// In case of an error, the object will move to error state 168 | fn write(&self, sbn: u32, data: &[u8], now: SystemTime) -> Result<()>; 169 | /// Called when all the data has been written 170 | fn complete(&self, now: SystemTime); 171 | /// Called when an error occurred during the reception of this object 172 | fn error(&self, now: SystemTime); 173 | /// Called when the sender has interrupted the transmission of this object 174 | fn interrupted(&self, now: SystemTime); 175 | /// Indicates whether MD5 checksum verification is enabled for this object. 176 | /// 177 | /// - `true`: The MD5 checksum will be verified. If the checksum is invalid, 178 | /// the object will transition to an error state. 179 | /// - `false`: The MD5 checksum will be skipped. Even if the checksum is invalid 180 | /// or missing, the object will proceed to a complete state without error. 181 | fn enable_md5_check(&self) -> bool; 182 | } 183 | 184 | impl std::fmt::Debug for dyn ObjectWriterBuilder { 185 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 186 | write!(f, "ObjectWriterBuilder {{ }}") 187 | } 188 | } 189 | 190 | impl std::fmt::Debug for dyn ObjectWriter { 191 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 192 | write!(f, "ObjectWriter {{ }}") 193 | } 194 | } 195 | 196 | mod objectwriterbuffer; 197 | mod objectwriterfs; 198 | 199 | pub use objectwriterbuffer::ObjectWriterBuffer; 200 | pub use objectwriterbuffer::ObjectWriterBufferBuilder; 201 | 202 | pub use objectwriterfs::ObjectWriterFS; 203 | pub use objectwriterfs::ObjectWriterFSBuilder; 204 | -------------------------------------------------------------------------------- /examples/flute-receiver/src/msocket.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | net::Ipv4Addr, 3 | os::fd::{AsRawFd, FromRawFd}, 4 | str::FromStr, 5 | }; 6 | 7 | use pnet::util::Octets; 8 | 9 | use libc::{ 10 | ip_mreq_source as IpMreqSource, IPPROTO_IP, IP_ADD_SOURCE_MEMBERSHIP, IP_DROP_SOURCE_MEMBERSHIP, 11 | }; 12 | 13 | const fn to_in_addr(addr: &Ipv4Addr) -> libc::in_addr { 14 | libc::in_addr { 15 | s_addr: u32::from_ne_bytes(addr.octets()), 16 | } 17 | } 18 | 19 | fn get_errno() -> i32 { 20 | unsafe { *libc::__errno_location() } 21 | } 22 | 23 | fn get_error_msg(errno_value: i32) -> Option { 24 | let error_message = unsafe { libc::strerror(errno_value) }; 25 | if error_message.is_null() { 26 | return None; 27 | } 28 | let c_str = unsafe { std::ffi::CStr::from_ptr(error_message) }; 29 | Some(c_str.to_string_lossy().into_owned()) 30 | } 31 | 32 | pub struct MSocket { 33 | pub sock: std::net::UdpSocket, 34 | source_addr: Option, 35 | group_addr: Ipv4Addr, 36 | interface: Ipv4Addr, 37 | } 38 | 39 | impl MSocket { 40 | pub fn new( 41 | endpoint: &flute::core::UDPEndpoint, 42 | eth: Option<&str>, 43 | nonblocking: bool, 44 | ) -> std::io::Result { 45 | log::info!("Create new Multicast Socket endpoint to {:?}", endpoint); 46 | 47 | let group_addr = match Ipv4Addr::from_str(&endpoint.destination_group_address) { 48 | Ok(res) => res, 49 | Err(_) => { 50 | return Err(std::io::Error::new( 51 | std::io::ErrorKind::Other, 52 | format!( 53 | "Fail to parse ip addr {}", 54 | endpoint.destination_group_address 55 | ), 56 | )) 57 | } 58 | }; 59 | 60 | let socket_fd = unsafe { libc::socket(libc::AF_INET, libc::SOCK_DGRAM, 0) }; 61 | if socket_fd == -1 { 62 | return Err(std::io::Error::new( 63 | std::io::ErrorKind::Other, 64 | "Fail to create UDP socket", 65 | )); 66 | } 67 | 68 | Self::set_reuse_address(socket_fd, true)?; 69 | Self::set_reuse_port(socket_fd, true)?; 70 | Self::set_receive_buffer_size(socket_fd, 1024 * 1024)?; 71 | Self::bind_socket(socket_fd, &group_addr, endpoint.port)?; 72 | 73 | let sock = unsafe { std::net::UdpSocket::from_raw_fd(socket_fd) }; 74 | sock.set_nonblocking(nonblocking)?; 75 | 76 | let interface = match eth { 77 | Some(res) => Ipv4Addr::from_str(res) 78 | .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?, 79 | None => Ipv4Addr::UNSPECIFIED, 80 | }; 81 | 82 | let source_addr = match &endpoint.source_address { 83 | Some(res) => Some( 84 | Ipv4Addr::from_str(res) 85 | .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?, 86 | ), 87 | None => None, 88 | }; 89 | 90 | if source_addr.is_some() && Self::is_ssm_addr(&group_addr) { 91 | Self::join_ssm( 92 | socket_fd, 93 | source_addr.as_ref().unwrap(), 94 | &group_addr, 95 | &interface, 96 | )?; 97 | } else { 98 | log::info!("Join multicast on interface {}", interface); 99 | sock.join_multicast_v4(&group_addr, &interface)?; 100 | } 101 | 102 | Ok(MSocket { 103 | sock, 104 | source_addr, 105 | group_addr, 106 | interface, 107 | }) 108 | } 109 | 110 | fn is_ssm_addr(group_addr: &Ipv4Addr) -> bool { 111 | group_addr.octets()[0] == 232 112 | } 113 | 114 | fn bind_socket(socket_fd: i32, address: &Ipv4Addr, port: u16) -> std::io::Result<()> { 115 | let sockaddr = libc::sockaddr_in { 116 | sin_family: libc::AF_INET as u16, 117 | sin_port: u16::from_ne_bytes(port.octets()), 118 | sin_addr: libc::in_addr { 119 | s_addr: u32::from_ne_bytes(address.octets()), 120 | }, 121 | sin_zero: [0; 8], 122 | }; 123 | 124 | let sockaddr_ptr = &sockaddr as *const libc::sockaddr_in as *const libc::sockaddr; 125 | let sockaddr_len = std::mem::size_of::() as libc::socklen_t; 126 | 127 | let ret = unsafe { libc::bind(socket_fd, sockaddr_ptr, sockaddr_len) }; 128 | 129 | if ret == -1 { 130 | return Err(std::io::Error::new( 131 | std::io::ErrorKind::Other, 132 | format!("Fail to bind socket {:?}", get_error_msg(get_errno())), 133 | )); 134 | } 135 | 136 | Ok(()) 137 | } 138 | 139 | fn join_ssm( 140 | sock: i32, 141 | source: &Ipv4Addr, 142 | group: &Ipv4Addr, 143 | interface: &Ipv4Addr, 144 | ) -> std::io::Result<()> { 145 | log::debug!("Join SSM {} {} {}", source, group, interface); 146 | let mreqs = IpMreqSource { 147 | imr_multiaddr: to_in_addr(group), 148 | imr_interface: to_in_addr(interface), 149 | imr_sourceaddr: to_in_addr(source), 150 | }; 151 | Self::setsockopt(sock, IPPROTO_IP, IP_ADD_SOURCE_MEMBERSHIP, mreqs) 152 | } 153 | 154 | fn leave_ssm( 155 | sock: i32, 156 | source: &Ipv4Addr, 157 | group: &Ipv4Addr, 158 | interface: &Ipv4Addr, 159 | ) -> std::io::Result<()> { 160 | log::debug!("Leave SSM {} {} {}", source, group, interface); 161 | let mreqs = IpMreqSource { 162 | imr_multiaddr: to_in_addr(group), 163 | imr_interface: to_in_addr(interface), 164 | imr_sourceaddr: to_in_addr(source), 165 | }; 166 | Self::setsockopt(sock, IPPROTO_IP, IP_DROP_SOURCE_MEMBERSHIP, mreqs) 167 | } 168 | 169 | fn set_reuse_address(sock: i32, reuse: bool) -> std::io::Result<()> { 170 | Self::setsockopt( 171 | sock, 172 | libc::SOL_SOCKET, 173 | libc::SO_REUSEADDR, 174 | match reuse { 175 | true => 1 as i32, 176 | false => 0 as i32, 177 | }, 178 | ) 179 | } 180 | 181 | fn set_reuse_port(sock: i32, reuse: bool) -> std::io::Result<()> { 182 | Self::setsockopt( 183 | sock, 184 | libc::SOL_SOCKET, 185 | libc::SO_REUSEPORT, 186 | match reuse { 187 | true => 1 as i32, 188 | false => 0 as i32, 189 | }, 190 | ) 191 | } 192 | 193 | fn set_receive_buffer_size(sock: i32, size: usize) -> std::io::Result<()> { 194 | Self::setsockopt(sock, libc::SOL_SOCKET, libc::SO_RCVBUF, size) 195 | } 196 | 197 | fn setsockopt( 198 | sock: libc::c_int, 199 | level: libc::c_int, 200 | name: libc::c_int, 201 | data: T, 202 | ) -> std::io::Result<()> { 203 | let data_ptr: *const libc::c_void = &data as *const _ as *const libc::c_void; 204 | let ret = unsafe { 205 | libc::setsockopt( 206 | sock as libc::c_int, 207 | level, 208 | name, 209 | data_ptr, 210 | std::mem::size_of::() as libc::socklen_t, 211 | ) 212 | }; 213 | match ret { 214 | 0 => Ok(()), 215 | _ => Err(std::io::Error::new( 216 | std::io::ErrorKind::Other, 217 | format!( 218 | "Fail to set opt {} errno={:?}", 219 | ret, 220 | get_error_msg(get_errno()) 221 | ), 222 | )), 223 | } 224 | } 225 | } 226 | 227 | impl Drop for MSocket { 228 | fn drop(&mut self) { 229 | let fd = self.sock.as_raw_fd(); 230 | if self.source_addr.is_some() && Self::is_ssm_addr(&self.group_addr) { 231 | Self::leave_ssm( 232 | fd, 233 | self.source_addr.as_ref().unwrap(), 234 | &self.group_addr, 235 | &self.interface, 236 | ) 237 | .ok(); 238 | } else { 239 | log::info!("Leave Multicast V4 on interface {}", self.interface); 240 | self.sock 241 | .leave_multicast_v4(&self.group_addr, &self.interface) 242 | .ok(); 243 | } 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /src/sender/blockencoder.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use super::filedesc; 4 | use super::objectdesc::ObjectDataSource; 5 | use crate::common::{partition, pkt}; 6 | use crate::error::FluteError; 7 | use crate::tools::error::Result; 8 | 9 | #[derive(Debug)] 10 | pub struct BlockEncoder { 11 | file: Arc, 12 | curr_content_offset: u64, 13 | curr_sbn: u32, 14 | a_large: u64, 15 | a_small: u64, 16 | nb_a_large: u64, 17 | nb_blocks: u64, 18 | blocks: Vec>, 19 | block_multiplex_windows: usize, 20 | block_multiplex_index: usize, 21 | read_end: bool, 22 | source_size_transferred: usize, 23 | nb_pkt_sent: usize, 24 | stopped: bool, 25 | closabled_object: bool, 26 | } 27 | 28 | use super::block::Block; 29 | 30 | impl BlockEncoder { 31 | pub fn new( 32 | file: Arc, 33 | block_multiplex_windows: usize, 34 | closabled_object: bool, 35 | ) -> Result { 36 | match &file.object.source { 37 | ObjectDataSource::Buffer(_) => {} 38 | ObjectDataSource::Stream(stream) => { 39 | stream.lock().unwrap().seek(std::io::SeekFrom::Start(0))?; 40 | } 41 | } 42 | 43 | let mut block = BlockEncoder { 44 | file, 45 | curr_content_offset: 0, 46 | curr_sbn: 0, 47 | a_large: 0, 48 | a_small: 0, 49 | nb_a_large: 0, 50 | nb_blocks: 0, 51 | blocks: Vec::new(), 52 | block_multiplex_windows, 53 | block_multiplex_index: 0, 54 | read_end: false, 55 | source_size_transferred: 0, 56 | nb_pkt_sent: 0, 57 | stopped: false, 58 | closabled_object, 59 | }; 60 | block.block_partitioning(); 61 | Ok(block) 62 | } 63 | 64 | pub fn read(&mut self, force_close_object: bool) -> Option { 65 | if self.stopped { 66 | return None; 67 | } 68 | 69 | if force_close_object { 70 | self.stopped = true; 71 | } 72 | 73 | loop { 74 | self.read_window(); 75 | 76 | if self.blocks.is_empty() { 77 | if self.nb_pkt_sent == 0 { 78 | log::debug!("Empty file ? Send a pkt containing close object flag"); 79 | self.nb_pkt_sent += 1; 80 | 81 | debug_assert!(self.file.object.transfer_length == 0); 82 | return Some(pkt::Pkt { 83 | payload: Vec::new(), 84 | transfer_length: self.file.object.transfer_length, 85 | esi: 0, 86 | sbn: 0, 87 | toi: self.file.toi, 88 | fdt_id: self.file.fdt_id, 89 | cenc: self.file.object.cenc, 90 | inband_cenc: self.file.object.inband_cenc, 91 | close_object: true, 92 | source_block_length: 0, 93 | sender_current_time: self.file.sender_current_time, 94 | }); 95 | } 96 | 97 | return None; 98 | } 99 | 100 | if self.block_multiplex_index >= self.blocks.len() { 101 | self.block_multiplex_index = 0; 102 | } 103 | 104 | let block = &mut self.blocks[self.block_multiplex_index]; 105 | let symbol = block.read(); 106 | if symbol.is_none() { 107 | self.blocks.remove(self.block_multiplex_index); 108 | continue; 109 | } 110 | 111 | let (symbol, is_last_symbol) = symbol.as_ref().unwrap(); 112 | 113 | self.block_multiplex_index += 1; 114 | if symbol.is_source_symbol { 115 | self.source_size_transferred += symbol.symbols.len(); 116 | } 117 | 118 | self.nb_pkt_sent += 1; 119 | 120 | let is_last_packet = (self.source_size_transferred 121 | >= self.file.object.transfer_length as usize) 122 | && *is_last_symbol; 123 | 124 | return Some(pkt::Pkt { 125 | payload: symbol.symbols.to_vec(), 126 | transfer_length: self.file.object.transfer_length, 127 | esi: symbol.esi, 128 | sbn: symbol.sbn, 129 | toi: self.file.toi, 130 | fdt_id: self.file.fdt_id, 131 | cenc: self.file.object.cenc, 132 | inband_cenc: self.file.object.inband_cenc, 133 | close_object: force_close_object || (self.closabled_object && is_last_packet), 134 | source_block_length: block.nb_source_symbols as u32, 135 | sender_current_time: self.file.sender_current_time, 136 | }); 137 | } 138 | } 139 | 140 | fn block_partitioning(&mut self) { 141 | let oti = &self.file.oti; 142 | (self.a_large, self.a_small, self.nb_a_large, self.nb_blocks) = 143 | partition::block_partitioning( 144 | oti.maximum_source_block_length as u64, 145 | self.file.object.transfer_length, 146 | oti.encoding_symbol_length as u64, 147 | ); 148 | } 149 | 150 | fn read_block(&mut self) -> Result<()> { 151 | debug_assert!(!self.read_end); 152 | let source = &self.file.object.source; 153 | match source { 154 | ObjectDataSource::Buffer(_) => self.read_block_buffer(), 155 | ObjectDataSource::Stream(_) => self.read_block_stream(), 156 | } 157 | } 158 | 159 | fn read_block_buffer(&mut self) -> Result<()> { 160 | log::debug!("Read block nb {}", self.curr_sbn); 161 | 162 | let content = match &self.file.object.source { 163 | ObjectDataSource::Buffer(buffer) => Ok(buffer), 164 | _ => Err(FluteError::new("Not a data source buffer")), 165 | }?; 166 | 167 | let oti = &self.file.oti; 168 | let block_length = match self.curr_sbn as u64 { 169 | value if value < self.nb_a_large => self.a_large, 170 | _ => self.a_small, 171 | }; 172 | 173 | let offset_start = self.curr_content_offset as usize; 174 | let mut offset_end = 175 | offset_start + (block_length * oti.encoding_symbol_length as u64) as usize; 176 | if offset_end > content.len() { 177 | offset_end = content.len(); 178 | } 179 | 180 | let buffer = &content.as_slice()[offset_start..offset_end]; 181 | let block = Block::new_from_buffer(self.curr_sbn, buffer, block_length, oti)?; 182 | self.blocks.push(block); 183 | self.curr_sbn += 1; 184 | self.read_end = offset_end == content.len(); 185 | self.curr_content_offset = offset_end as u64; 186 | log::debug!( 187 | "offset={}/{} end={}", 188 | self.curr_content_offset, 189 | content.len(), 190 | self.read_end 191 | ); 192 | Ok(()) 193 | } 194 | 195 | fn read_block_stream(&mut self) -> Result<()> { 196 | log::info!("Read block nb {}", self.curr_sbn); 197 | 198 | let mut stream = match &self.file.object.source { 199 | ObjectDataSource::Stream(stream) => Ok(stream.lock().unwrap()), 200 | _ => Err(FluteError::new("Not a data source stream")), 201 | }?; 202 | 203 | let oti = &self.file.oti; 204 | let block_length = match self.curr_sbn as u64 { 205 | value if value < self.nb_a_large => self.a_large, 206 | _ => self.a_small, 207 | }; 208 | let mut buffer: Vec = 209 | vec![0; block_length as usize * oti.encoding_symbol_length as usize]; 210 | let result = match stream.read(&mut buffer) { 211 | Ok(s) => s, 212 | Err(e) => { 213 | log::error!("Fail to read file {:?}", e.to_string()); 214 | self.read_end = true; 215 | return Ok(()); 216 | } 217 | }; 218 | 219 | if result == 0 { 220 | self.read_end = true; 221 | return Ok(()); 222 | } 223 | 224 | buffer.truncate(result); 225 | 226 | let block = Block::new_from_buffer(self.curr_sbn, &buffer, block_length, oti)?; 227 | self.blocks.push(block); 228 | self.curr_sbn += 1; 229 | self.curr_content_offset += buffer.len() as u64; 230 | Ok(()) 231 | } 232 | 233 | fn read_window(&mut self) { 234 | while !self.read_end && (self.blocks.len() < self.block_multiplex_windows) { 235 | match self.read_block() { 236 | Ok(_) => {} 237 | Err(_) => self.read_end = true, // TODO handle error 238 | }; 239 | } 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /src/receiver/fdtreceiver.rs: -------------------------------------------------------------------------------- 1 | use super::objectreceiver; 2 | use super::writer::{ObjectWriterBuilder, ObjectWriterBuilderResult}; 3 | use crate::common::udpendpoint::UDPEndpoint; 4 | use crate::common::{alc, fdtinstance::FdtInstance, lct}; 5 | use crate::{receiver::writer::ObjectMetadata, tools}; 6 | use crate::{receiver::writer::ObjectWriter, tools::error::Result}; 7 | use std::{cell::RefCell, rc::Rc, time::SystemTime}; 8 | 9 | #[derive(Clone, Copy, PartialEq, Debug)] 10 | pub enum FDTState { 11 | Receiving, 12 | Complete, 13 | Error, 14 | Expired, 15 | } 16 | 17 | pub struct FdtReceiver { 18 | pub fdt_id: u32, 19 | obj: Option>, 20 | inner: Rc>, 21 | fdt_instance: Option, 22 | sender_current_time_offset: Option, 23 | sender_current_time_late: bool, 24 | pub ext_time: Option, 25 | pub reception_start_time: SystemTime, 26 | enable_expired_check: bool, 27 | meta: Option, 28 | } 29 | 30 | impl std::fmt::Debug for FdtReceiver { 31 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 32 | f.debug_struct("FdtReceiver") 33 | .field("fdt_id", &self.fdt_id) 34 | .field("obj", &self.obj) 35 | .field("inner", &self.inner) 36 | .field("fdt_instance", &self.fdt_instance) 37 | .field( 38 | "sender_current_time_offset", 39 | &self.sender_current_time_offset, 40 | ) 41 | .field("sender_current_time_late", &self.sender_current_time_late) 42 | .field("receiver_start_time", &self.reception_start_time) 43 | .finish() 44 | } 45 | } 46 | 47 | #[derive(Debug)] 48 | struct FdtWriter { 49 | inner: Rc>, 50 | } 51 | 52 | struct FdtWriterBuilder { 53 | inner: Rc>, 54 | } 55 | 56 | #[derive(Debug)] 57 | struct FdtWriterInner { 58 | data: Vec, 59 | fdt: Option, 60 | expires: Option, 61 | state: FDTState, 62 | } 63 | 64 | impl FdtReceiver { 65 | pub fn new( 66 | endpoint: &UDPEndpoint, 67 | tsi: u64, 68 | fdt_id: u32, 69 | enable_expired_check: bool, 70 | now: SystemTime, 71 | ) -> FdtReceiver { 72 | let inner = Rc::new(RefCell::new(FdtWriterInner { 73 | data: Vec::new(), 74 | fdt: None, 75 | state: FDTState::Receiving, 76 | expires: None, 77 | })); 78 | 79 | let fdt_builder = Rc::new(FdtWriterBuilder::new(inner.clone())); 80 | 81 | FdtReceiver { 82 | fdt_id, 83 | obj: Some(Box::new(objectreceiver::ObjectReceiver::new( 84 | endpoint, 85 | tsi, 86 | &lct::TOI_FDT, 87 | Some(fdt_id), 88 | fdt_builder, 89 | 1024 * 1024, 90 | now, 91 | ))), 92 | inner: inner.clone(), 93 | fdt_instance: None, 94 | sender_current_time_offset: None, 95 | sender_current_time_late: true, 96 | reception_start_time: now, 97 | enable_expired_check, 98 | meta: None, 99 | ext_time: None, 100 | } 101 | } 102 | 103 | pub fn push(&mut self, pkt: &alc::AlcPkt, now: std::time::SystemTime) { 104 | if let Ok(Some(res)) = alc::get_sender_current_time(pkt) { 105 | self.ext_time = Some(res); 106 | if res < now { 107 | self.sender_current_time_late = true; 108 | self.sender_current_time_offset = Some(now.duration_since(res).unwrap()) 109 | } else { 110 | self.sender_current_time_late = false; 111 | self.sender_current_time_offset = Some(res.duration_since(now).unwrap()) 112 | } 113 | } 114 | 115 | if let Some(obj) = self.obj.as_mut() { 116 | obj.push(pkt, now); 117 | match obj.state { 118 | objectreceiver::State::Receiving => {} 119 | objectreceiver::State::Completed => { 120 | self.meta = Some(obj.create_meta()); 121 | self.obj = None 122 | } 123 | objectreceiver::State::Interrupted => { 124 | self.inner.borrow_mut().state = FDTState::Error 125 | } 126 | objectreceiver::State::Error => self.inner.borrow_mut().state = FDTState::Error, 127 | } 128 | } 129 | } 130 | 131 | pub fn get_server_time(&self, now: std::time::SystemTime) -> std::time::SystemTime { 132 | if let Some(offset) = self.sender_current_time_offset { 133 | if self.sender_current_time_late { 134 | return now - offset; 135 | } else { 136 | return now + offset; 137 | } 138 | } 139 | 140 | now 141 | } 142 | 143 | pub fn state(&self) -> FDTState { 144 | self.inner.borrow().state 145 | } 146 | 147 | pub fn fdt_instance(&mut self) -> Option<&FdtInstance> { 148 | if self.fdt_instance.is_none() { 149 | let inner = self.inner.borrow(); 150 | let instance = inner.fdt.as_ref(); 151 | self.fdt_instance = instance.cloned(); 152 | } 153 | self.fdt_instance.as_ref() 154 | } 155 | 156 | pub fn fdt_xml_str(&self) -> Option { 157 | let inner = self.inner.borrow(); 158 | String::from_utf8(inner.data.clone()).ok() 159 | } 160 | 161 | pub fn fdt_meta(&self) -> Option<&ObjectMetadata> { 162 | self.meta.as_ref() 163 | } 164 | 165 | pub fn update_expired_state(&self, now: SystemTime) { 166 | if self.state() != FDTState::Complete { 167 | return; 168 | } 169 | 170 | if self.enable_expired_check && self.is_expired(now) { 171 | let mut inner = self.inner.borrow_mut(); 172 | inner.state = FDTState::Expired; 173 | } 174 | } 175 | 176 | fn is_expired(&self, now: SystemTime) -> bool { 177 | let inner = self.inner.borrow(); 178 | let expires = match inner.expires { 179 | Some(expires) => expires, 180 | None => return true, 181 | }; 182 | 183 | self.get_server_time(now) > expires 184 | } 185 | 186 | pub fn get_expiration_time(&self) -> Option { 187 | let inner = self.inner.borrow(); 188 | inner.expires 189 | } 190 | } 191 | 192 | impl FdtWriterBuilder { 193 | fn new(inner: Rc>) -> Self { 194 | FdtWriterBuilder { inner } 195 | } 196 | } 197 | 198 | impl ObjectWriterBuilder for FdtWriterBuilder { 199 | fn new_object_writer( 200 | &self, 201 | _endpoint: &UDPEndpoint, 202 | _tsi: &u64, 203 | _toi: &u128, 204 | _meta: &ObjectMetadata, 205 | _now: std::time::SystemTime, 206 | ) -> ObjectWriterBuilderResult { 207 | ObjectWriterBuilderResult::StoreObject(Box::new(FdtWriter { 208 | inner: self.inner.clone(), 209 | })) 210 | } 211 | 212 | fn update_cache_control( 213 | &self, 214 | _endpoint: &UDPEndpoint, 215 | _tsi: &u64, 216 | _toi: &u128, 217 | _meta: &ObjectMetadata, 218 | _now: std::time::SystemTime, 219 | ) { 220 | } 221 | 222 | fn fdt_received( 223 | &self, 224 | _endpoint: &UDPEndpoint, 225 | _tsi: &u64, 226 | _fdt_xml: &str, 227 | _expires: std::time::SystemTime, 228 | __meta: &ObjectMetadata, 229 | _transfer_duration: std::time::Duration, 230 | _now: std::time::SystemTime, 231 | _ext_time: Option, 232 | ) { 233 | } 234 | } 235 | 236 | impl ObjectWriter for FdtWriter { 237 | fn open(&self, _now: SystemTime) -> Result<()> { 238 | Ok(()) 239 | } 240 | 241 | fn write(&self, _sbn: u32, data: &[u8], _now: SystemTime) -> Result<()> { 242 | let mut inner = self.inner.borrow_mut(); 243 | inner.data.extend(data); 244 | Ok(()) 245 | } 246 | 247 | fn complete(&self, _now: SystemTime) { 248 | let mut inner = self.inner.borrow_mut(); 249 | match FdtInstance::parse(&inner.data) { 250 | Ok(inst) => { 251 | inner.expires = match inst.expires.parse::() { 252 | Ok(seconds_ntp) => tools::ntp_to_system_time((seconds_ntp as u64) << 32).ok(), 253 | _ => None, 254 | }; 255 | inner.fdt = Some(inst); 256 | inner.state = FDTState::Complete 257 | } 258 | Err(_) => inner.state = FDTState::Error, 259 | }; 260 | } 261 | 262 | fn error(&self, _now: SystemTime) { 263 | let mut inner = self.inner.borrow_mut(); 264 | inner.state = FDTState::Error; 265 | } 266 | 267 | fn interrupted(&self, _now: SystemTime) { 268 | let mut inner = self.inner.borrow_mut(); 269 | inner.state = FDTState::Error; 270 | } 271 | 272 | fn enable_md5_check(&self) -> bool { 273 | false 274 | } 275 | } 276 | --------------------------------------------------------------------------------