├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── format.sh ├── json.hpp ├── m3u8.hpp ├── m3u8.re2c.cpp ├── main.cpp ├── netutils.hpp ├── netutils.re2c.cpp ├── tsanalyser.cpp └── tsanalyser.hpp /.gitignore: -------------------------------------------------------------------------------- 1 | CMakeCache.txt 2 | CMakeFiles 3 | CMakeScripts 4 | Testing 5 | Makefile 6 | cmake_install.cmake 7 | install_manifest.txt 8 | compile_commands.json 9 | CTestTestfile.cmake 10 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.1) 2 | project(muxincstreamvalidator VERSION 1.0.0) 3 | 4 | set(CMAKE_CXX_STANDARD 11) 5 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 6 | set(CMAKE_CXX_EXTENSIONS OFF) 7 | set(CMAKE_BUILD_TYPE Debug) 8 | 9 | find_package(CURL REQUIRED) 10 | find_program(RE2C re2c) 11 | 12 | if(RE2C) 13 | add_custom_command( OUTPUT netutils.cpp DEPENDS netutils.re2c.cpp 14 | COMMAND ${RE2C} -i -o netutils.cpp netutils.re2c.cpp) 15 | add_custom_command( OUTPUT m3u8.cpp DEPENDS m3u8.re2c.cpp 16 | COMMAND ${RE2C} -i -o m3u8.cpp m3u8.re2c.cpp) 17 | else() 18 | error( "re2c required to compile" ) 19 | endif() 20 | 21 | include_directories(${CURL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/nlohmann_json/single_include/) 22 | add_executable(muxincstreamvalidator main.cpp m3u8.cpp netutils.cpp tsanalyser.cpp) 23 | target_link_libraries(muxincstreamvalidator ${CURL_LIBRARIES}) 24 | install (TARGETS muxincstreamvalidator DESTINATION bin) 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Mux 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![HLS Tools](https://banner.mux.dev/HLS%20Tools.svg) 2 | 3 | # hlstools 4 | 5 | Tools for analyzing and processing HLS streams. See our blog post on [measuring package overhead](https://mux.com/blog/quantifying-packaging-overhead-2/) for more details on how we use this internally. 6 | 7 | A central repository for tools to analyse and process HLS streams. Think Apple's [HTTP Live Streaming Tools](https://developer.apple.com/documentation/http_live_streaming/about_apple_s_http_live_streaming_tools) but with just one of the packages...for now. :imp: 8 | 9 | * `muxincstreamvalidator` - A tool to analyse HLS streams. Similar to Apple's mediastreamvalidator, but works on non-Apple platforms and supports measuring TS packaging overhead. 10 | 11 | 12 | These tools use the excellent [JSON for Modern C++](https://github.com/nlohmann/json) and is available under the [MIT](./LICENSE). 13 | -------------------------------------------------------------------------------- /format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd "$(dirname "$0")" 3 | find . -maxdepth 1 \( -name '*.cpp' -o -name '*.hpp' \) -exec clang-format -i -style=WebKit {} \; 4 | -------------------------------------------------------------------------------- /m3u8.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "json.hpp" 3 | #include 4 | 5 | // https://tools.ietf.org/html/rfc8216 6 | // https://video-dev.github.io/hlsjs-rfcs/docs/0001-lhls 7 | 8 | class m3u8 { 9 | public: 10 | static nlohmann::json parse(const char* m3u8); 11 | static nlohmann::json parse(const std::string& m3u8) { return parse(m3u8.c_str()); } 12 | }; 13 | -------------------------------------------------------------------------------- /m3u8.re2c.cpp: -------------------------------------------------------------------------------- 1 | /*!re2c 2 | yyyy = ('-'|[-+]?[0-9]{4}); 3 | mo = ([0][1-9] | [1][0-2]); 4 | dd = ([0][1-9] | [12][0-9] | [3][0-1]); 5 | wk = ([0][1-9] | [1-4][0-9] | [5][0-3]) ([-]? [0-7])?; 6 | // TODO Ordinal date 7 | date = yyyy '-' (wk | (mo '-' dd)); 8 | 9 | hh = ([01][0-9] | [2][0-4]); 10 | mm = ([0-5][0-9]); 11 | ss = ([0-5][0-9]); 12 | ms = ([0-9]+); 13 | time = 'T' (hh ':' mm ':' ss '.' ms | hh mm ss '.' ms | hh ':' mm ':' ss | hh mm ss | hh ':' mm | hh mm | hh); 14 | 15 | timezone = ('Z'|[-+][0-9]{2}(':'[0-9]{2})?); 16 | iso8601 = date (time? timezone)?; 17 | 18 | endl = ("\r\n" | [\r\n] | [\x00]); 19 | decimal_integer = ("0" | [1-9][0-9]*); 20 | signed_decimal_integer = [+-]? decimal_integer; 21 | floating_point = signed_decimal_integer ("." [0-9]+)?; 22 | quoted_string = ([\x22] [^\r\n\x22\x00]* [\x22]); 23 | enumerated_string = ([^\r\n\t\x20\x22\x2c\x3d\x00]+); 24 | 25 | attribute_name = ([-A-Z0-9]+); 26 | attribute_value = (quoted_string | enumerated_string); 27 | attribute_list = (#keyStart attribute_name #keyEnd "=" #valStart attribute_value #valEnd) ','?; 28 | 29 | title = [^\r\n\x00]*; 30 | uri = [^\r\n\x00]+; 31 | */ 32 | 33 | #include "m3u8.hpp" 34 | using mtag_t = std::pair; 35 | static nlohmann::json render_attribute_list(const std::vector& list, int keyStart, int keyEnd, int valStart, int valEnd) 36 | { 37 | if (0 > keyStart || 0 > valStart) { 38 | return nlohmann::json::object(); 39 | } 40 | 41 | auto json = render_attribute_list(list, list[keyStart].first, list[keyEnd].first, list[valStart].first, list[valEnd].first); 42 | auto key = std::string(list[keyStart].second, list[keyEnd].second); 43 | auto val = std::string(list[valStart].second, list[valEnd].second); 44 | json.emplace(std::move(key), std::move(val)); 45 | return json; 46 | } 47 | // https://tools.ietf.org/html/rfc8216 48 | nlohmann::json m3u8::parse(const char* YYCURSOR) 49 | { 50 | int keyindex = -1, mapindex = -1; 51 | int segment_size = 0, segment_offset = 0; 52 | auto json = nlohmann::json::object(); 53 | auto key = nlohmann::json::array(); 54 | auto map = nlohmann::json::array(); 55 | auto media = nlohmann::json::object(); 56 | auto inf = nlohmann::json::array(); 57 | auto streaminf = nlohmann::json::array(); 58 | bool discontinuity = false; 59 | const char* YYMARKER = YYCURSOR; 60 | const char *a, *b, *c, *d, *e; 61 | int keyStart, keyEnd, valStart, valEnd; 62 | std::vector attrlist; 63 | std::string program_date_time; 64 | const char* end = YYCURSOR + strlen(YYCURSOR); 65 | 66 | #define YYMTAGP(t) mtag(&t, YYCURSOR); 67 | #define YYMTAGN(t) mtag(&t, nullptr); 68 | auto mtag = [&attrlist](int* pmt, const char* YYCURSOR) { 69 | attrlist.push_back({ (*pmt), YYCURSOR }); 70 | *pmt = attrlist.size() - 1; 71 | }; 72 | 73 | /*!mtags:re2c format = "int @@;\n"; */ 74 | /*!stags:re2c format = "const char * @@;"; */ 75 | while (YYCURSOR < end) { 76 | attrlist.clear(); 77 | /*!mtags:re2c format = "@@ = -1;"; */ 78 | /*!stags:re2c format = "@@ = nullptr;"; */ 79 | /*!re2c 80 | re2c:flags:tags = 1; 81 | re2c:yyfill:enable = 0; 82 | re2c:define:YYCTYPE = char; 83 | re2c:flags:bit-vectors = 1; 84 | 85 | // Special cases: 86 | "#EXTINF:" @a floating_point (',' @b title @c)? endl @d uri @e endl { 87 | inf.push_back({ 88 | {"DURATION", std::atof(a)}, 89 | {"URI", std::string(d, e)}, 90 | }); 91 | 92 | if(c > b) { inf.back().push_back({"TITLE", std::string(b, c)}); } 93 | if(0 <= keyindex) { inf.back().push_back({"KEY", keyindex}); } 94 | if(0 <= mapindex) { inf.back().push_back({"MAP", mapindex}); } 95 | 96 | if(discontinuity) { 97 | inf.back().push_back({"DISCONTINUITY", true}); 98 | discontinuity = false; 99 | } 100 | 101 | if(!program_date_time.empty()) { 102 | inf.back().push_back({"PROGRAM-DATE-TIME", std::move(program_date_time)}); 103 | program_date_time.clear(); 104 | } 105 | 106 | if(0 < segment_size) { 107 | inf.back().push_back({"BYTERANGE", {segment_size, segment_offset}}); 108 | segment_offset += segment_size; 109 | segment_size = 0; 110 | } else { 111 | segment_offset = 0; 112 | } 113 | 114 | continue; 115 | } 116 | 117 | "#EXT-X-PREFETCH:" @a uri @b endl { 118 | inf.push_back({ 119 | {"PREFETCH", true}, 120 | {"URI", std::string(a, b)}, 121 | }); 122 | 123 | if(discontinuity) { 124 | inf.back().emplace("DISCONTINUITY", true); 125 | discontinuity = false; 126 | } 127 | 128 | continue; 129 | } 130 | 131 | "#EXT-X-MEDIA:" attribute_list+ endl { 132 | // Move the GROUP-ID to be the key for the object, to make lookup of groups eaiser 133 | auto group = render_attribute_list(attrlist, keyStart, keyEnd, valStart, valEnd); 134 | auto group_id = group.find("GROUP-ID"); 135 | if(group.end() != group_id) { 136 | auto name = (*group_id); 137 | group.erase(group_id); 138 | media.emplace(std::move(name), std::move(group)); 139 | } 140 | continue; 141 | } 142 | 143 | "#EXT-X-STREAM-INF:" attribute_list+ endl @a uri @b endl { 144 | streaminf.push_back( { 145 | { "ATTRIBUTES", render_attribute_list(attrlist, keyStart, keyEnd, valStart, valEnd) }, 146 | { "URI", std::string(a, b) } 147 | }); 148 | continue; 149 | } 150 | 151 | "#EXT-X-KEY:" attribute_list+ endl { 152 | auto attrs = render_attribute_list(attrlist, keyStart, keyEnd, valStart, valEnd); 153 | auto method = attrs.find("METHOD"); 154 | if(attrs.end() == method || "NONE" == (*method) ) { 155 | keyindex = -1; 156 | } else { 157 | key.push_back({std::move(attrs)}); 158 | keyindex = key.size() - 1; 159 | } 160 | continue; 161 | } 162 | 163 | "#EXT-X-KEY:" attribute_list+ endl { 164 | auto attrs = render_attribute_list(attrlist, keyStart, keyEnd, valStart, valEnd); 165 | map.push_back({std::move(attrs)}); 166 | index = map.size() - 1; 167 | continue; 168 | } 169 | 170 | // #EXT-X-DISCONTINUITY, #EXT-X-PREFETCH-DISCONTINUITY 171 | "#EXT-X-" "PREFETCH-"? "DISCONTINUITY" endl { discontinuity = true; continue; } 172 | "#EXT-X-PROGRAM-DATE-TIME:" @a iso8601 @b endl { program_date_time = std::string(a, b); continue; } 173 | "#EXT-X-PLAYLIST-TYPE:" @a ("EVENT" | "VOD") @b endl { json.emplace("X-PLAYLIST-TYPE", std::string(a, b)); continue; } 174 | "#EXT-X-BYTERANGE:" @a decimal_integer ("@" @b decimal_integer)? endl { segment_size = std::atoi(a); segment_offset = b ? std::atoi(b) : 0; continue; } 175 | 176 | // #EXT-X-MEDIA-SEQUENCE, #EXT-X-TARGETDURATION, #EXT-X-VERSION, #EXT-X-DISCONTINUITY-SEQUENCE 177 | "#EXT-" @a [A-Z][-A-Z]* @b ":" @c decimal_integer endl { 178 | json.emplace(std::string(a, b), std::atof(c)); 179 | continue; 180 | } 181 | 182 | // #EXT-X-START, #EXT-X-DATERANGE, #EXT-X-SESSION-DATA, #EXT-X-SESSION-KEY 183 | "#EXT-" @a [A-Z][-A-Z]* @b ":" attribute_list+ endl { 184 | json.emplace(std::string(a, b), render_attribute_list(attrlist, keyStart, keyEnd, valStart, valEnd)); 185 | continue; 186 | } 187 | 188 | // #EXT-X-ENDLIST, #EXT-X-I-FRAMES-ONLY, #EXT-X-INDEPENDENT-SEGMENTS 189 | "#EXT-" @a [A-Z][-A-Z]* @b endl { 190 | json.emplace(std::string(a, b), true); 191 | continue; 192 | } 193 | 194 | // Other 195 | "#" [^\r\n\x00]* endl { continue; } // Comment 196 | [ \t]* endl { continue; } // Whitespace 197 | [\x00] { goto done; } // eof 198 | * { goto parse_error; } // Something else 199 | */ 200 | } 201 | 202 | done: 203 | // If these values are set, they were not consumed. 204 | if (discontinuity || segment_size || !program_date_time.empty()) { 205 | return nlohmann::json {}; 206 | } 207 | 208 | // can not be both master and varient 209 | if ((streaminf.empty() && inf.empty()) || (!streaminf.empty() && !inf.empty())) { 210 | return nlohmann::json {}; 211 | } 212 | 213 | if (streaminf.empty()) { 214 | json.emplace("INF", std::move(inf)); 215 | 216 | if (!key.empty()) { 217 | json.emplace("X-KEY", std::move(key)); 218 | } 219 | 220 | if (!map.empty()) { 221 | json.emplace("X-MAP", std::move(map)); 222 | } 223 | 224 | if (json.end() == json.find("X-ENDLIST")) { 225 | json.emplace("X-ENDLIST", false); 226 | } 227 | } else { 228 | json.emplace("X-STREAM-INF", std::move(streaminf)); 229 | 230 | if (!media.empty()) { 231 | json.emplace("X-MEDIA", std::move(media)); 232 | } 233 | } 234 | 235 | return json; 236 | parse_error: 237 | return nlohmann::json {}; 238 | } 239 | 240 | std::string render_attribute_list(const nlohmann::json& list) 241 | { 242 | std::string str; 243 | for (auto pair = list.begin(); pair != list.end(); ++pair) { 244 | if (pair != list.begin()) { 245 | str += ","; 246 | } 247 | 248 | str += pair.key() + "="; 249 | if (pair.value().is_number()) { 250 | auto num = pair.value().get(); 251 | if (num == std::floor(num)) { 252 | str += std::to_string(static_cast(num)); 253 | } else { 254 | str += std::to_string(static_cast(num)); 255 | } 256 | } else if (pair.value().is_string()) { 257 | str += pair.value().get(); 258 | } 259 | } 260 | 261 | return str; 262 | } 263 | -------------------------------------------------------------------------------- /main.cpp: -------------------------------------------------------------------------------- 1 | #include "m3u8.hpp" 2 | #include "netutils.hpp" 3 | #include "tsanalyser.hpp" 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | /////////////////////////////////////////////////////////////////////////////////////////////////// 10 | void ProcessVariant(const nlohmann::json& playlist, const uri& base_uri) 11 | { 12 | transportstream ts; 13 | const auto& EXTINF = playlist.find("INF"); 14 | 15 | for (const auto& seg : *EXTINF) { 16 | // TODO check for MAP to get init (It could contain PAT/PMT) 17 | // TODO check mime type 18 | auto segment_uri = base_uri.resolve(uri(seg["URI"])); 19 | auto byterange = seg.find("BYTE-RANGE"); 20 | size_t offset = 0, size = 0; 21 | if (byterange != seg.end()) { 22 | if (2 == byterange->size()) { 23 | offset = byterange->at(0).get(); 24 | size = byterange->at(1).get(); 25 | } 26 | } 27 | 28 | net::download(segment_uri, offset, size, [&ts](const char* data, size_t size) -> bool { 29 | ts.analyse(reinterpret_cast(data), size); 30 | return true; 31 | }); 32 | } 33 | 34 | ts.printStats(); 35 | } 36 | 37 | int main(int argc, char** argv) 38 | { 39 | curl_global_init(CURL_GLOBAL_ALL); 40 | auto playlist_url = uri(argv[1]); 41 | 42 | std::string playlist; 43 | auto err = net::download(playlist_url, [&playlist](const char* data, unsigned int size) -> bool { 44 | static const size_t maxplaylistSize = 10 * 1024 * 1024; 45 | playlist.insert(playlist.end(), data, data + size); 46 | return playlist.size() < maxplaylistSize; 47 | }); 48 | 49 | if (playlist.empty()) { 50 | std::cerr << "Could not download playlist: " << err << std::endl; 51 | return EXIT_FAILURE; 52 | } 53 | 54 | auto json = m3u8::parse(playlist); 55 | const auto X_STREAM_INF = json.find("X-STREAM-INF"); 56 | 57 | if (json.end() != X_STREAM_INF) { 58 | // Master playlist. Pick a rendition 59 | for (const auto& inf : *X_STREAM_INF) { 60 | auto redition_url = playlist_url.resolve(uri(inf["URI"])); 61 | playlist.clear(); 62 | std::cout << redition_url.string() << std::endl; 63 | net::download(redition_url, [&playlist](const char* data, unsigned int size) -> bool { 64 | static const size_t maxplaylistSize = 10 * 1024 * 1024; 65 | playlist.insert(playlist.end(), data, data + size); 66 | return playlist.size() < maxplaylistSize; 67 | }); 68 | 69 | if (playlist.empty()) { 70 | std::cerr << "Could not download playlist" << std::endl; 71 | return EXIT_FAILURE; 72 | } 73 | 74 | auto rendition = m3u8::parse(playlist); 75 | if (rendition.empty()) { 76 | std::cerr << "Could not parse playlist" << std::endl; 77 | return EXIT_FAILURE; 78 | } 79 | 80 | ProcessVariant(rendition, redition_url); 81 | } 82 | } else { 83 | ProcessVariant(json, playlist_url); 84 | } 85 | 86 | return EXIT_SUCCESS; 87 | } 88 | -------------------------------------------------------------------------------- /netutils.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | class uri { 8 | private: 9 | // rfc3986 10 | std::string m_scheme; 11 | std::string m_authority; 12 | std::string m_path; 13 | std::string m_query; 14 | std::string m_fragment; 15 | std::string merge(const std::string& path) const; 16 | 17 | public: 18 | uri() = default; 19 | uri(const std::string& u); 20 | uri resolve(uri u) const; 21 | bool is_relative() const { return m_scheme.empty(); } 22 | bool is_absolute() const { return !(m_scheme.empty() || m_authority.empty()); } 23 | std::string string() const; 24 | 25 | const std::string& scheme() const { return m_scheme; } 26 | const std::string& authority() const { return m_authority; } 27 | const std::string& path() const { return m_path; } 28 | const std::string& query() const { return m_query; } 29 | const std::string& fragment() const { return m_fragment; } 30 | }; 31 | 32 | class net { 33 | public: 34 | static int download(const uri& u, unsigned int offset, unsigned int size, const std::function& callback); 35 | static int download(const uri& u, const std::function& callback) { return download(u, 0, 0, callback); } 36 | }; 37 | -------------------------------------------------------------------------------- /netutils.re2c.cpp: -------------------------------------------------------------------------------- 1 | #include "netutils.hpp" 2 | #include 3 | 4 | static std::string tos(const char* b, const char* e) 5 | { 6 | return (b && e && b < e) ? std::string(b, e) : std::string(); 7 | } 8 | 9 | uri::uri(const std::string& u) 10 | { 11 | const char* YYCURSOR = u.c_str(); 12 | const char* YYMARKER = YYCURSOR; 13 | const char *scheme_b = nullptr, *scheme_e = nullptr; 14 | const char *authority_b = nullptr, *authority_e = nullptr; 15 | const char *path_b = nullptr, *path_e = nullptr; 16 | const char *query_b = nullptr, *query_e = nullptr; 17 | const char *fragment_b = nullptr, *fragment_e = nullptr; 18 | /*!stags:re2c format = "const char * @@;"; */ 19 | /*!stags:re2c format = "@@ = nullptr;"; */ 20 | /*!re2c 21 | re2c:flags:tags = 1; 22 | re2c:yyfill:enable = 0; 23 | re2c:define:YYCTYPE = char; 24 | re2c:flags:bit-vectors = 1; 25 | * {return;} 26 | ( @scheme_b [^:/?#\x00]+ @scheme_e ":")? 27 | ("//" @authority_b [^/?#\x00]* @authority_e )? 28 | (@path_b [^?#\x00]* @path_e ) 29 | ("?" @query_b [^#\x00]* @query_e )? 30 | ("#" @fragment_b [^\x00]* @fragment_e )? { 31 | m_scheme = tos(scheme_b, scheme_e); 32 | m_authority = tos(authority_b, authority_e); 33 | m_path = tos(path_b, path_e); 34 | m_query = tos(query_b, query_e); 35 | m_fragment = tos(fragment_b, fragment_e); 36 | return; 37 | } 38 | */ 39 | } 40 | 41 | std::string uri::merge(const std::string& path) const 42 | { 43 | // If the base URI has a defined authority component and an empty 44 | // path, then return a string consisting of "/" concatenated with the 45 | // reference's path 46 | if (!m_authority.empty() && m_path.empty()) { 47 | return "/" + path; 48 | } 49 | 50 | // return a string consisting of the reference's path component 51 | // appended to all but the last segment of the base URI's path (i.e., 52 | // excluding any characters after the right-most "/" in the base URI 53 | // path, or excluding the entire base URI path if it does not contain 54 | // any "/" characters). 55 | auto end = m_path.find_last_of("/"); 56 | if (std::string::npos != end) { 57 | return m_path.substr(0, end + 1) + path; 58 | } 59 | 60 | return path; 61 | } 62 | 63 | static std::string remove_dot_segments(const std::string& path) 64 | { 65 | // While the input buffer is not empty, loop as follows: 66 | const char* YYCURSOR = path.c_str(); 67 | const char* YYMARKER = YYCURSOR; 68 | std::vector segments; 69 | const char *a = nullptr, *b = nullptr; 70 | while (true) { 71 | /*!stags:re2c format = "const char * @@;"; */ 72 | /*!stags:re2c format = "@@ = nullptr;"; */ 73 | /*!re2c 74 | re2c:flags:tags = 1; 75 | re2c:yyfill:enable = 0; 76 | re2c:define:YYCTYPE = char; 77 | re2c:flags:bit-vectors = 1; 78 | 79 | // End of string 80 | "\x00" { goto done; } 81 | "/" / "/" { continue; } // Multiple slashes, eat one 82 | * { return std::string(); } 83 | 84 | // A. If the input buffer begins with a prefix of "../" or "./", 85 | // then remove that prefix from the input buffer; otherwise, 86 | "."{1,2} / "/" { 87 | continue; 88 | } 89 | 90 | // B. if the input buffer begins with a prefix of "/./" or "/.", 91 | // where "." is a complete path segment, then replace that 92 | // prefix with "/" in the input buffer; otherwise, 93 | "/." / [/\x00] { 94 | continue; 95 | } 96 | 97 | // C. if the input buffer begins with a prefix of "/../" or "/..", 98 | // where ".." is a complete path segment, then replace that 99 | // prefix with "/" in the input buffer and remove the last 100 | // segment and its preceding "/" (if any) from the output 101 | // buffer; otherwise, 102 | "/.." / [/\x00] { 103 | if(!segments.empty()) { 104 | segments.pop_back(); 105 | } 106 | 107 | continue; 108 | } 109 | 110 | // D. if the input buffer consists only of "." or "..", then remove 111 | // that from the input buffer; otherwise, 112 | "."{1,2} "\x00" { 113 | continue; 114 | } 115 | 116 | // E. move the first path segment in the input buffer to the end of 117 | // the output buffer, including the initial "/" character (if 118 | // any) and any subsequent characters up to, but not including, 119 | // the next "/" character or the end of the input buffer. 120 | path_segment = @a [^?#\x00][^/?#\x00]* @b; 121 | path_segment / [/\x00] { 122 | segments.emplace_back( std::string(a, b) ); 123 | continue; 124 | } 125 | */ 126 | } 127 | done: 128 | return std::accumulate(segments.begin(), segments.end(), std::string("")); 129 | } 130 | 131 | uri uri::resolve(uri u) const 132 | { 133 | if (u.m_scheme.empty()) { 134 | u.m_scheme = m_scheme; 135 | if (u.m_authority.empty()) { 136 | u.m_authority = m_authority; 137 | if (u.m_path.empty()) { 138 | u.m_path = m_path; 139 | if (u.m_query.empty()) { 140 | u.m_query = m_query; 141 | } 142 | } else if ('/' != u.m_path.front()) { 143 | u.m_path = merge(u.m_path); 144 | } 145 | } 146 | 147 | u.m_path = remove_dot_segments(u.m_path); 148 | } 149 | 150 | return u; 151 | } 152 | 153 | std::string uri::string() const 154 | { 155 | std::string str; 156 | str += !m_scheme.empty() ? m_scheme + "://" : std::string(); 157 | str += m_authority; 158 | str += !m_authority.empty() && m_path.empty() ? std::string("/") : m_path; 159 | str += !m_query.empty() ? "?" + m_query : std::string(); 160 | str += !m_fragment.empty() ? "#" + m_fragment : std::string(); 161 | return str; 162 | } 163 | /////////////////////////////////////////////////////////////////////////////////////////////////// 164 | size_t _write_callback(char* ptr, size_t size, size_t nmemb, void* userdata) 165 | { 166 | size *= nmemb; 167 | auto callback = static_cast*>(userdata); 168 | if ((*callback)(ptr, size)) { 169 | return size; 170 | } 171 | 172 | return 0; 173 | } 174 | 175 | int net::download(const uri& u, unsigned int offset, unsigned int size, const std::function& callback) 176 | { 177 | long response_code = 0; 178 | CURL* curl = nullptr; 179 | CURLcode err = CURLE_OK; 180 | char range[32]; 181 | if (!(curl = curl_easy_init())) { 182 | err = CURLE_FAILED_INIT; 183 | curl_easy_reset(curl); 184 | return -err; 185 | } 186 | 187 | if (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L))) { 188 | curl_easy_reset(curl); 189 | return -err; 190 | } 191 | 192 | if (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_URL, u.string().c_str()))) { 193 | curl_easy_reset(curl); 194 | return -err; 195 | } 196 | 197 | if (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, _write_callback))) { 198 | curl_easy_reset(curl); 199 | return -err; 200 | } 201 | 202 | if (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&callback)))) { 203 | curl_easy_reset(curl); 204 | return -err; 205 | } 206 | 207 | if (0 < offset || 0 < size) { 208 | if (0 < size) { 209 | snprintf(range, sizeof(range), "%u-%u", offset, size); 210 | } else { 211 | snprintf(range, sizeof(range), "%u-", offset); 212 | } 213 | 214 | if (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_RANGE, range))) { 215 | curl_easy_reset(curl); 216 | return -err; 217 | } 218 | } 219 | 220 | if (CURLE_OK != (err = curl_easy_perform(curl))) { 221 | curl_easy_reset(curl); 222 | return -err; 223 | } 224 | 225 | if (CURLE_OK != (err = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code))) { 226 | curl_easy_reset(curl); 227 | return -err; 228 | } 229 | 230 | return response_code; 231 | } 232 | -------------------------------------------------------------------------------- /tsanalyser.cpp: -------------------------------------------------------------------------------- 1 | #include "tsanalyser.hpp" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | class safe_access { 9 | public: 10 | size_t size = 0; 11 | uint8_t* data = nullptr; 12 | safe_access& operator+=(size_t s) 13 | { 14 | assert(s <= size); 15 | s = std::min(s, size); 16 | data += s, size -= s; 17 | return *this; 18 | } 19 | 20 | uint8_t operator[](size_t i) const 21 | { 22 | assert(i < size); 23 | static const uint8_t zero = 0; 24 | return size <= i ? zero : data[i]; 25 | } 26 | 27 | safe_access(const uint8_t* data, size_t size) 28 | : data(const_cast(data)) 29 | , size(size) 30 | { 31 | } 32 | }; 33 | 34 | int64_t decode_timestap(const safe_access& d, size_t off) 35 | { 36 | int64_t t = 0; 37 | // mmmmxxx1 xxxxxxxx xxxxxxx1 xxxxxxxx xxxxxxx1 38 | t |= d[off + 0] << 29 & 0x1c0000000; 39 | t |= d[off + 1] << 22 & 0x03fc00000; 40 | t |= d[off + 2] << 14 & 0x0003f8000; 41 | t |= d[off + 3] << 7 & 0x000007f10; 42 | t |= d[off + 4] >> 1 & 0x00000007f; 43 | return t; 44 | } 45 | 46 | uint16_t transportstream::analysePacket(const uint8_t* data, size_t size) 47 | { 48 | assert(188 == size && 0x47 == data[0]); 49 | safe_access d(data, size); 50 | bool payload = d[3] & 0x10; 51 | bool adaptationfield = d[3] & 0x20; 52 | uint16_t pid = ((d[1] & 0x1F) << 8) | d[2]; // PacketId 53 | uint8_t cc = 0x0F & d[3]; // Continuity Counter 54 | stats[pid].payloadunitstart = 0x40 & d[1]; // Pyaload Unit Start Indicator 55 | ++stats[pid].packets; 56 | 57 | d += 4; 58 | if (adaptationfield) { // Adaption Field 59 | d += d[0] + 1; 60 | } 61 | 62 | if (0 == pid) { 63 | // PAT 64 | pmtpid = (d[11] << 8 & 0x1f00) | d[12]; 65 | } else if (pmtpid == pid) { 66 | bool current = d[6] & 0x01; 67 | int16_t sectionlength = (d[2] & 0x03 << 8) | d[3]; 68 | int16_t pcrpid = (d[9] & 0x1f << 8) | d[10]; 69 | sectionlength -= 13; 70 | d += 13; 71 | while (5 <= sectionlength) { 72 | uint8_t streamtype = d[0]; 73 | int16_t streampid = (d[1] << 8 & 0x1f00) | d[2]; 74 | int16_t infolength = (d[3] << 8 & 0x0300) | d[4]; 75 | sectionlength -= 5 + infolength; 76 | stats[streampid].streamtype = streamtype; 77 | d += 5 + infolength; 78 | } 79 | } else if (stats[pid].streamtype == 3 || stats[pid].streamtype == 15 || stats[pid].streamtype == 27) { 80 | // 3 = mp3, 15 = aac, 27 =avc 81 | if (stats[pid].payloadunitstart) { 82 | assert(0 == d[0] && 0 == d[1] && 1 == d[2]); 83 | stats[pid].streamid = d[3]; // StreamID 84 | uint16_t pespacketsize = (d[4] << 8) | d[5]; 85 | uint16_t flags = (d[6] << 8) | d[7]; 86 | int16_t pesheadersize = 9 + d[8]; 87 | 88 | if (flags & 0x80) { 89 | stats[pid].previousdecodetimestamp = stats[pid].decodetimestamp; 90 | stats[pid].presentationtimestamp = decode_timestap(d, 9); 91 | if (flags & 0x40) { 92 | stats[pid].decodetimestamp = decode_timestap(d, 14); 93 | } else { 94 | stats[pid].decodetimestamp = stats[pid].presentationtimestamp; 95 | } 96 | } 97 | 98 | d += pesheadersize; 99 | 100 | // If not a video stream. record payload size 101 | if (!(stats[pid].streamid >= 0xe0 && stats[pid].streamid <= 0xeF)) { 102 | stats[pid].payloadsize += pespacketsize - pesheadersize; 103 | } 104 | } 105 | 106 | // if this is a video stream, count the bytes remaining in the packet 107 | if (payload && stats[pid].streamid >= 0xe0 && stats[pid].streamid <= 0xeF) { 108 | stats[pid].payloadsize += d.size; 109 | } 110 | } 111 | 112 | return pid; 113 | } 114 | 115 | void transportstream::printStats() 116 | { 117 | size_t totalpackets = 0, totalpayloadsize = 0; 118 | for (int pid = 0; pid < stats.size(); ++pid) { 119 | if (0 < stats[pid].packets) { 120 | totalpackets += stats[pid].packets; 121 | totalpayloadsize += stats[pid].payloadsize; 122 | double overhead = 0 >= stats[pid].payloadsize ? 1.0 : 1 - ((double)stats[pid].payloadsize / (188 * stats[pid].packets)); 123 | printf("PID %d: SID %02x, size %zu, overhead %0.2f%%\n", pid, stats[pid].streamid, 188 * stats[pid].packets, 100 * overhead); 124 | } 125 | } 126 | 127 | double overhead = 1 - ((double)totalpayloadsize / (188 * totalpackets)); 128 | printf("-------------------------------------------------------------\n"); 129 | printf("TOTALS: size %zu, overhead %0.2f%%\n", 188 * totalpackets, 100 * overhead); 130 | } 131 | -------------------------------------------------------------------------------- /tsanalyser.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | class transportstream { 8 | public: 9 | uint16_t analysePacket(const uint8_t* data, size_t size); 10 | void printStats(); 11 | struct pidstats { 12 | bool payloadunitstart = false; 13 | size_t packets = 0; 14 | uint8_t streamtype = 0; 15 | int16_t streamid = 0; 16 | size_t payloadsize = 0; 17 | int16_t pespayloadtsize = 0; 18 | int64_t decodetimestamp = -1; 19 | int64_t presentationtimestamp = -1; 20 | int64_t previousdecodetimestamp = -1; 21 | }; 22 | 23 | int pmtpid; 24 | std::array stats; 25 | 26 | std::vector m_data; 27 | void analyse(const uint8_t* data, size_t size) 28 | { 29 | m_data.insert(m_data.end(), data, data + size); 30 | data = m_data.data(), size = m_data.size(); 31 | for (; 188 <= size; data += 188, size -= 188) { 32 | analysePacket(data, 188); 33 | } 34 | 35 | m_data = std::vector(data, data + size); 36 | } 37 | }; 38 | --------------------------------------------------------------------------------