├── .eslintignore ├── .eslintrc.js ├── .github └── workflows │ └── node.js.yml ├── .gitignore ├── .labrc.js ├── .npmignore ├── README.md ├── build └── transform-typescript.js ├── lib ├── index.ts ├── segment-downloader.ts ├── segment-reader.ts └── segment-streamer.ts ├── package-lock.json ├── package.json ├── test ├── _shared.js ├── fixtures │ ├── 500-00000.ts │ ├── 500-00001.ts │ ├── 500-00002.ts │ ├── 500.m3u8 │ ├── 500.ts │ ├── badtype-data.m3u8 │ ├── badtype.m3u8 │ ├── discont.m3u8 │ ├── files │ │ ├── audio.aac │ │ ├── audio.ac3 │ │ ├── audio.dts │ │ ├── audio.eac3 │ │ ├── audio.m4a │ │ ├── file.m4s │ │ ├── file.mp4 │ │ ├── text.vtt │ │ └── video.m4v │ ├── index.m3u8 │ ├── long.m3u8 │ ├── malformed.m3u8 │ ├── single.m3u8 │ └── slow.m3u8 ├── reader.js └── streamer.js └── tsconfig.json /.eslintignore: -------------------------------------------------------------------------------- 1 | /test/fixtures/** -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const EslintPluginHapi = require('@hapi/eslint-plugin'); 4 | const TypescriptRules = require('@typescript-eslint/eslint-plugin').rules; 5 | 6 | 7 | const tsifyRules = function (from) { 8 | 9 | const rules = {}; 10 | 11 | for (const rule in from) { 12 | if (TypescriptRules[rule] && rule !== 'padding-line-between-statements') { 13 | rules[rule] = 'off'; 14 | rules[`@typescript-eslint/${rule}`] = from[rule]; 15 | } 16 | } 17 | 18 | return rules; 19 | }; 20 | 21 | 22 | module.exports = { 23 | root: true, 24 | extends: [ 25 | 'plugin:@hapi/recommended', 26 | 'plugin:@typescript-eslint/eslint-recommended' 27 | ], 28 | plugins: [ 29 | '@typescript-eslint' 30 | ], 31 | parserOptions: { 32 | ecmaVersion: 2019 33 | }, 34 | ignorePatterns: ['/lib/*.js', '/lib/*.d.ts'], 35 | overrides: [{ 36 | files: ['lib/**/*.ts'], 37 | extends: [ 38 | 'plugin:@typescript-eslint/recommended' 39 | ], 40 | parser: '@typescript-eslint/parser', 41 | parserOptions: { 42 | sourceType: 'module', 43 | project: './tsconfig.json', 44 | tsconfigRootDir: __dirname 45 | }, 46 | rules: { 47 | ...tsifyRules(EslintPluginHapi.configs.recommended.rules), 48 | '@typescript-eslint/no-explicit-any': 'off', 49 | '@typescript-eslint/no-non-null-assertion': 'off', 50 | 51 | '@typescript-eslint/member-delimiter-style': 'warn', 52 | '@typescript-eslint/no-throw-literal': 'error', 53 | '@typescript-eslint/prefer-for-of': 'warn', 54 | '@typescript-eslint/type-annotation-spacing': 'warn', 55 | '@typescript-eslint/unified-signatures': 'warn' 56 | } 57 | }] 58 | }; 59 | -------------------------------------------------------------------------------- /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Node.js CI 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | strategy: 18 | matrix: 19 | node-version: [14] 20 | 21 | steps: 22 | - uses: actions/checkout@v2 23 | - name: Use Node.js ${{ matrix.node-version }} 24 | uses: actions/setup-node@v2 25 | with: 26 | node-version: ${{ matrix.node-version }} 27 | check-latest: ${{ matrix.node-version == '*' }} 28 | - run: npm ci 29 | - run: npm run test-full 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules/ 2 | /lib/*.js 3 | /lib/*.d.ts 4 | coverage.html 5 | -------------------------------------------------------------------------------- /.labrc.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | transform: require.resolve('./build/transform-typescript'), 5 | sourcemaps: true, 6 | flat: true, 7 | leaks: false 8 | }; 9 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | * 2 | !lib/**/*.js 3 | !lib/**/*.d.ts 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HlsSegmentReader for node.js 2 | 3 | Read segments from any [Apple HLS](http://tools.ietf.org/html/draft-pantos-http-live-streaming) source in an object-mode `Readable`. 4 | 5 | ![Node.js CI](https://github.com/kanongil/node-hls-segment-reader/workflows/Node.js%20CI/badge.svg?event=push) 6 | 7 | ## Installation 8 | 9 | ```sh 10 | $ npm install hls-segment-reader 11 | ``` 12 | 13 | # License 14 | 15 | (BSD 2-Clause License) 16 | 17 | Copyright (c) 2014-2020, Gil Pedersen <gpdev@gpost.dk> 18 | All rights reserved. 19 | 20 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 21 | 22 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 23 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 24 | 25 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /build/transform-typescript.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Ts = require('typescript'); 4 | 5 | const cache = new Map(); 6 | 7 | const tsify = function (content, fileName) { 8 | 9 | const searchPath = Ts.normalizePath(Ts.sys.getCurrentDirectory()); 10 | const configFileName = process.env.TSCONFIG || Ts.findConfigFile(searchPath, Ts.sys.fileExists); 11 | 12 | const compilerOptions = getCompilerOptionsViaCache(configFileName); 13 | compilerOptions.sourceMap = false; 14 | compilerOptions.inlineSourceMap = true; 15 | 16 | const { outputText/*, diagnostics*/ } = Ts.transpileModule(content, { 17 | fileName, 18 | compilerOptions, 19 | reportDiagnostics: false 20 | }); 21 | 22 | const splicePoint = outputText.indexOf('Object.defineProperty(exports, "__esModule", { value: true })'); 23 | if (splicePoint !== -1) { 24 | return '/* $lab:coverage:off$ */' + outputText.slice(0, splicePoint) + '/* $lab:coverage:on$ */' + outputText.slice(splicePoint); 25 | } 26 | 27 | return outputText; 28 | }; 29 | 30 | const getCompilerOptionsViaCache = function (configFileName) { 31 | 32 | let options; 33 | 34 | if (!(options = cache[configFileName])) { 35 | options = cache[configFileName] = getCompilerOptions(configFileName); 36 | } 37 | 38 | return options; 39 | }; 40 | 41 | const getCompilerOptions = function (configFileName) { 42 | 43 | const { config, error } = Ts.readConfigFile(configFileName, Ts.sys.readFile); 44 | if (error) { 45 | throw new Error(`TS config error in ${configFileName}: ${error.messageText}`); 46 | } 47 | 48 | const { options } = Ts.parseJsonConfigFileContent( 49 | config, 50 | Ts.sys, 51 | Ts.getDirectoryPath(configFileName), 52 | {}, 53 | configFileName); 54 | 55 | return options; 56 | }; 57 | 58 | module.exports = [{ 59 | ext: '.ts', 60 | transform: tsify 61 | }]; 62 | -------------------------------------------------------------------------------- /lib/index.ts: -------------------------------------------------------------------------------- 1 | import { HlsPlaylistReader, HlsPlaylistReaderOptions } from 'hls-playlist-reader'; 2 | import { HlsSegmentReader, HlsSegmentReaderOptions } from './segment-reader'; 3 | import { HlsSegmentStreamer, HlsSegmentStreamerOptions } from './segment-streamer'; 4 | 5 | export { HlsReaderObject } from './segment-reader'; 6 | export type { HlsIndexMeta } from 'hls-playlist-reader'; 7 | export { HlsStreamerObject } from './segment-streamer'; 8 | 9 | const createSimpleReader = function (uri: string, options: HlsSegmentReaderOptions & HlsSegmentStreamerOptions = {}): HlsSegmentStreamer { 10 | 11 | const reader = new HlsSegmentReader(uri, options); 12 | 13 | options.withData ?? (options.withData = false); 14 | 15 | const streamer = new HlsSegmentStreamer(reader, options); 16 | 17 | reader.on('problem', (err) => streamer.emit('problem', err)); 18 | 19 | return streamer; 20 | }; 21 | 22 | export { createSimpleReader, HlsPlaylistReader, HlsSegmentReader, HlsSegmentStreamer }; 23 | export type { HlsPlaylistReaderOptions, HlsSegmentReaderOptions, HlsSegmentStreamerOptions }; 24 | 25 | export default createSimpleReader; 26 | -------------------------------------------------------------------------------- /lib/segment-downloader.ts: -------------------------------------------------------------------------------- 1 | import type { URL } from 'url'; 2 | import type { Byterange } from 'hls-playlist-reader/lib/helpers'; 3 | 4 | import { finished } from 'stream'; 5 | import { promisify } from 'util'; 6 | 7 | import { applyToDefaults, assert } from '@hapi/hoek'; 8 | 9 | import { performFetch } from 'hls-playlist-reader/lib/helpers'; 10 | 11 | 12 | const internals = { 13 | defaults: { 14 | probe: false 15 | }, 16 | streamFinished: promisify(finished) 17 | }; 18 | 19 | 20 | // eslint-disable-next-line @typescript-eslint/ban-types 21 | type FetchToken = object | string | number; 22 | 23 | export class SegmentDownloader { 24 | 25 | probe: boolean; 26 | 27 | #fetches = new Map>(); 28 | 29 | constructor(options: { probe?: boolean }) { 30 | 31 | options = applyToDefaults(internals.defaults, options); 32 | 33 | this.probe = !!options.probe; 34 | } 35 | 36 | fetchSegment(token: FetchToken, uri: URL, byterange?: Required, { tries = 3 } = {}): ReturnType { 37 | 38 | const promise = performFetch(uri, { byterange, probe: this.probe, retries: tries - 1 }); 39 | this._startTracking(token, promise); 40 | return promise; 41 | } 42 | 43 | /** 44 | * Stops any fetch not in token list 45 | * 46 | * @param {Set} tokens 47 | */ 48 | setValid(tokens = new Set()): void { 49 | 50 | for (const [token, fetch] of this.#fetches) { 51 | 52 | if (!tokens.has(token)) { 53 | this._stopTracking(token); 54 | fetch.abort(); 55 | } 56 | } 57 | } 58 | 59 | private _startTracking(token: FetchToken, promise: ReturnType) { 60 | 61 | assert(!this.#fetches.has(token), 'A token can only be tracked once'); 62 | 63 | // Setup auto-untracking 64 | 65 | promise.then(({ stream }) => { 66 | 67 | if (!stream) { 68 | return this._stopTracking(token); 69 | } 70 | 71 | if (!this.#fetches.has(token)) { 72 | return; // It has already been aborted 73 | } 74 | 75 | finished(stream, () => this._stopTracking(token)); 76 | }).catch((/*err*/) => { 77 | 78 | this._stopTracking(token); 79 | }); 80 | 81 | this.#fetches.set(token, promise); 82 | } 83 | 84 | private _stopTracking(token: FetchToken) { 85 | 86 | this.#fetches.delete(token); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /lib/segment-reader.ts: -------------------------------------------------------------------------------- 1 | import { assert as hoekAssert } from '@hapi/hoek'; 2 | import { MediaPlaylist, MasterPlaylist, MediaSegment, IndependentSegment, AttrList } from 'm3u8parse'; 3 | 4 | import { Deferred } from 'hls-playlist-reader/lib/helpers'; 5 | import { DuplexEvents, TypedDuplex, TypedEmitter } from 'hls-playlist-reader/lib/raw/typed-readable'; 6 | import { HlsIndexMeta, HlsPlaylistReader, HlsPlaylistReaderOptions } from 'hls-playlist-reader'; 7 | import type { ParsedPlaylist, PartData, PlaylistReaderObject, PreloadHints } from 'hls-playlist-reader/lib/playlist-reader'; 8 | 9 | 10 | // eslint-disable-next-line func-style 11 | function assert(condition: any, ...args: any[]): asserts condition { 12 | 13 | hoekAssert(condition, ...args); 14 | } 15 | 16 | 17 | class SegmentPointer { 18 | 19 | readonly msn: number; 20 | readonly part?: number; 21 | 22 | constructor(msn = -1, part?: number) { 23 | 24 | this.msn = +msn; 25 | this.part = part; 26 | } 27 | 28 | next(isPartial = false): SegmentPointer { 29 | 30 | if (isPartial) { 31 | return new SegmentPointer(this.msn, undefined); 32 | } 33 | 34 | return new SegmentPointer(this.msn + 1, this.part === undefined ? undefined : 0); 35 | } 36 | 37 | isEmpty(): boolean { 38 | 39 | return this.msn < 0; 40 | } 41 | } 42 | 43 | class UnendingPlaylistReader extends HlsPlaylistReader { 44 | 45 | protected preprocessIndex(index: T): T | undefined { 46 | 47 | if (!index.master) { 48 | MediaPlaylist.cast(index).ended = false; 49 | } 50 | 51 | return super.preprocessIndex(index); 52 | } 53 | } 54 | 55 | export class HlsReaderObject { 56 | 57 | readonly msn: number; 58 | readonly isClosed: boolean; 59 | 60 | onUpdate?: ((entry: IndependentSegment, old?: IndependentSegment) => void) = undefined; 61 | 62 | private _entry: IndependentSegment; 63 | #closed?: Deferred; 64 | 65 | constructor(msn: number, segment: IndependentSegment) { 66 | 67 | this.msn = msn; 68 | this._entry = new MediaSegment(segment) as IndependentSegment; 69 | this.isClosed = !segment.isPartial(); 70 | } 71 | 72 | get entry(): IndependentSegment { 73 | 74 | return this._entry; 75 | } 76 | 77 | set entry(entry: IndependentSegment) { 78 | 79 | assert(!this.isClosed); 80 | 81 | const old = this._entry; 82 | this._entry = new MediaSegment(entry) as IndependentSegment; 83 | 84 | this._entry.discontinuity = !!(+entry.discontinuity | +old.discontinuity); 85 | 86 | this._update(!entry.isPartial(), old); 87 | } 88 | 89 | closed(): PromiseLike | true { 90 | 91 | if (this.isClosed) { 92 | return true; 93 | } 94 | 95 | if (!this.#closed) { 96 | this.#closed = new Deferred(); 97 | } 98 | 99 | return this.#closed.promise; 100 | } 101 | 102 | abandon(): void { 103 | 104 | if (!this.isClosed) { 105 | return this._update(true); 106 | } 107 | } 108 | 109 | private _update(closed: boolean, old?: IndependentSegment): void { 110 | 111 | if (closed) { 112 | (<{ isClosed: boolean }> this).isClosed = true; 113 | if (this.#closed) { 114 | this.#closed.resolve(true); 115 | } 116 | } 117 | 118 | if (this.onUpdate) { 119 | process.nextTick(this.onUpdate.bind(this, this._entry, old)); 120 | } 121 | } 122 | } 123 | 124 | export type HlsSegmentReaderOptions = { 125 | /** Start from first segment, or use stream start */ 126 | fullStream?: boolean; 127 | 128 | /** Initial segment ends after this date */ 129 | startDate?: Date | string | number; 130 | 131 | /** End when segment start after this date */ 132 | stopDate?: Date | string | number; 133 | } & HlsPlaylistReaderOptions; 134 | 135 | 136 | const HlsSegmentReaderEvents = >(null as any); 137 | interface IHlsSegmentReaderEvents { 138 | index(index: Readonly, meta: Readonly): void; 139 | hints(part?: PartData, map?: PartData): void; 140 | problem(err: Error): void; 141 | } 142 | 143 | /** 144 | * Reads an HLS media playlist, and output segments in order. 145 | * Live & Event playlists are refreshed as needed, and expired segments are dropped when backpressure is applied. 146 | */ 147 | export class HlsSegmentReader extends TypedEmitter(HlsSegmentReaderEvents, TypedDuplex, HlsReaderObject>()) { 148 | 149 | readonly fullStream: boolean; 150 | startDate?: Date; 151 | stopDate?: Date; 152 | 153 | readonly feeder: HlsPlaylistReader; 154 | 155 | #next = new SegmentPointer(); 156 | #current: HlsReaderObject | null = null; 157 | #playlist?: ParsedPlaylist; 158 | #nextPlaylist = new Deferred(true); 159 | #needRead = new Deferred(); 160 | #lastHints: PreloadHints; 161 | #readActive = false; 162 | 163 | constructor(src: string, options: HlsSegmentReaderOptions = {}) { 164 | 165 | super({ objectMode: true, highWaterMark: 0, autoDestroy: true, emitClose: true, allowHalfOpen: false }); 166 | 167 | this.fullStream = !!options.fullStream; 168 | 169 | // Dates are inclusive 170 | 171 | this.startDate = options.startDate ? new Date(options.startDate) : undefined; 172 | this.stopDate = options.stopDate ? new Date(options.stopDate) : undefined; 173 | 174 | this.feeder = new (this.stopDate ? UnendingPlaylistReader : HlsPlaylistReader)(src, options); 175 | this.#lastHints = this.feeder.hints; 176 | 177 | this.feeder.on<'problem'>('problem', (err) => !this.destroyed && this.emit<'problem'>('problem', err)); 178 | this.feeder.on<'error'>('error', (err) => { 179 | 180 | // Must defer to NT since this.readableEnded might already have been scheduled 181 | 182 | process.nextTick(() => this.destroy(this.readableEnded ? undefined : err)); 183 | }); 184 | 185 | this.feeder.pipe(this, { end: false }); 186 | } 187 | 188 | get index(): Readonly | undefined { 189 | 190 | return this.feeder.index; // TODO: use _transform input 191 | } 192 | 193 | get hints(): ParsedPlaylist['preloadHints'] { 194 | 195 | return this.feeder.hints; 196 | } 197 | 198 | async _write(input: Readonly, _: unknown, done: (err?: Error) => void): Promise { 199 | 200 | try { 201 | const { index, playlist, meta } = input; 202 | 203 | if (index.master) { 204 | process.nextTick(this.emit.bind(this, 'index', index, meta)); 205 | process.nextTick(this.#nextPlaylist.reject, new Error('master playlist')); 206 | return; 207 | } 208 | 209 | assert(input.playlist); 210 | 211 | // Update current entry with latest data 212 | 213 | if (this.#current && !this.#current.isClosed) { 214 | const currentSegment = index.getSegment(this.#current.msn, true); 215 | if (currentSegment && (currentSegment.isPartial() || currentSegment.parts)) { 216 | this.#current.entry = currentSegment; 217 | } 218 | } 219 | 220 | // Emit updates 221 | 222 | process.nextTick(this.emit.bind(this, 'index', index, meta)); 223 | 224 | if (this.hints !== this.#lastHints) { 225 | this.#lastHints = this.hints; 226 | process.nextTick(this.emit.bind(this, 'hints'), this.#lastHints.part, this.#lastHints.map); 227 | } 228 | 229 | // Signal new playlist is ready 230 | 231 | this.#playlist = playlist; 232 | process.nextTick(this.#nextPlaylist.resolve, playlist); 233 | this.#nextPlaylist = new Deferred(true); 234 | 235 | // Wait until output side needs more segments 236 | 237 | if (index.isLive()) { 238 | await this.#needRead.promise; 239 | } 240 | } 241 | catch (err: any) { 242 | //this.#nextPlaylist.reject(err); 243 | return done(err); 244 | } 245 | 246 | return done(); 247 | } 248 | 249 | /** 250 | * Called to push the next segment. 251 | */ 252 | /*protected*/ async _read(): Promise { 253 | 254 | if (this.#readActive) { 255 | return; 256 | } 257 | 258 | this.#readActive = true; 259 | try { 260 | const deferred = this.#needRead; 261 | this.#needRead = new Deferred(); 262 | deferred.resolve(); 263 | 264 | let more = true; 265 | while (more) { 266 | try { 267 | const result = await this._getNextSegment(this.#next); 268 | 269 | this.#current?.abandon(); 270 | 271 | if (!result) { 272 | this.#current = null; 273 | this.push(null); 274 | this.feeder.destroy(new Error('aborted')); 275 | return; 276 | } 277 | 278 | // Apply cross playlist discontinuity 279 | 280 | if (result.discont) { 281 | result.segment.discontinuity = true; 282 | } 283 | 284 | this.#current = new HlsReaderObject(result.ptr.msn, result.segment); 285 | this.#next = result.ptr.next(); 286 | 287 | this.#readActive = more = this.push(this.#current); 288 | 289 | if (result.segment.isPartial()) { 290 | more || (more = !await this._getNextSegment(new SegmentPointer(result.ptr.msn))); // fetch until we have the full segment 291 | } 292 | } 293 | catch (err: any) { 294 | if (this.index) { 295 | if (this.index.master) { 296 | this.push(null); // Just ignore any error 297 | this.feeder.destroy(new Error('aborted')); 298 | return; 299 | } 300 | 301 | if (this.feeder.isRecoverableUpdateError(err)) { 302 | continue; 303 | } 304 | } 305 | 306 | throw err; 307 | } 308 | } 309 | } 310 | catch (err: any) { 311 | this.destroy(err); 312 | } 313 | finally { 314 | this.#readActive = false; 315 | } 316 | } 317 | 318 | _destroy(err: Error | null, cb: unknown): void { 319 | 320 | this.#current?.abandon(); 321 | this.feeder.destroy(); 322 | 323 | super._destroy(err, cb as any); 324 | } 325 | 326 | // Private methods 327 | 328 | private async _waitForUpdate(from?: Readonly): Promise { 329 | 330 | if (this.index?.master) { 331 | throw new Error('Master playlist cannot be updated'); 332 | } 333 | 334 | let playlist = this.#playlist; 335 | while (!this.destroyed) { 336 | if (playlist) { 337 | const updated = !from || !playlist.index.isLive() || !playlist.isSameHead(from); 338 | if (updated) { 339 | return playlist; 340 | } 341 | } 342 | 343 | // Signal stall 344 | 345 | const deferred = this.#needRead; 346 | this.#needRead = new Deferred(); 347 | deferred.resolve(); 348 | 349 | // Wait for new playlist 350 | 351 | playlist = await this.#nextPlaylist.promise; 352 | } 353 | 354 | throw new Error('Stream was destroyed'); 355 | } 356 | 357 | private async _getNextSegment(ptr: SegmentPointer): Promise<{ ptr: SegmentPointer; discont: boolean; segment: IndependentSegment } | null> { 358 | 359 | let playlist: ParsedPlaylist | undefined; 360 | let discont = false; 361 | while (playlist = await this._waitForUpdate(playlist?.index)) { 362 | if (ptr.isEmpty()) { 363 | ptr = this._initialSegmentPointer(playlist); 364 | } 365 | else if ((ptr.msn < playlist.index.startMsn(true)) || 366 | (ptr.msn > (playlist.index.lastMsn(this.feeder.lowLatency) + 1))) { 367 | 368 | // Playlist jump 369 | 370 | if (playlist.index.type /* VOD or event */) { 371 | throw new Error('Fatal playlist inconsistency'); 372 | } 373 | 374 | this.emit<'problem'>('problem', new Error('Playlist jump')); 375 | 376 | ptr = new SegmentPointer(playlist.index.startMsn(true), this.feeder.lowLatency ? 0 : undefined); 377 | discont = true; 378 | } 379 | 380 | const segment = playlist.index.getSegment(ptr.msn, true); 381 | if (!segment || 382 | (ptr.part === undefined && segment.isPartial()) || 383 | (ptr.part && ptr.part >= (segment?.parts?.length || 0))) { 384 | 385 | if (!playlist.index.isLive()) { 386 | return null; // Done - nothing more to do 387 | } 388 | 389 | continue; // Try again 390 | } 391 | 392 | // Check if we need to stop 393 | 394 | if (this.stopDate && (segment.program_time || 0) > this.stopDate) { 395 | return null; 396 | } 397 | 398 | return { ptr, discont, segment }; 399 | } 400 | 401 | return null; 402 | } 403 | 404 | private _initialSegmentPointer(playlist: ParsedPlaylist): SegmentPointer { 405 | 406 | if (!this.fullStream && this.startDate) { 407 | const msn = playlist.index.msnForDate(this.startDate, true); 408 | if (msn >= 0) { 409 | return new SegmentPointer(msn, this.feeder.lowLatency ? 0 : undefined); 410 | } 411 | 412 | // no date information in index 413 | } 414 | 415 | if (this.feeder.lowLatency && playlist.serverControl.partHoldBack) { 416 | let partHoldBack = playlist.serverControl.partHoldBack; 417 | let msn = playlist.index.lastMsn(true); 418 | let segment; 419 | while (segment = playlist.index.getSegment(msn)) { 420 | // TODO: use INDEPENDENT=YES information for better start point 421 | 422 | if (segment.uri) { 423 | partHoldBack -= segment.duration || 0; 424 | } 425 | else { 426 | assert(segment.parts); 427 | partHoldBack -= segment.parts.reduce((duration, part) => duration + part.get('duration', AttrList.Types.Float), 0); 428 | } 429 | 430 | if (partHoldBack <= 0) { 431 | break; 432 | } 433 | 434 | msn--; 435 | } 436 | 437 | return new SegmentPointer(msn, 0); 438 | } 439 | 440 | // TODO: use start offset, when present 441 | 442 | return new SegmentPointer(playlist.index.startMsn(this.fullStream), this.feeder.lowLatency ? 0 : undefined); 443 | } 444 | } 445 | -------------------------------------------------------------------------------- /lib/segment-streamer.ts: -------------------------------------------------------------------------------- 1 | 2 | import type { Readable } from 'stream'; 3 | import type { MasterPlaylist, MediaPlaylist } from 'm3u8parse'; 4 | import type { HlsSegmentReader, HlsReaderObject } from './segment-reader'; 5 | import type { FetchResult, Byterange } from 'hls-playlist-reader/lib/helpers'; 6 | 7 | import { Stream, finished } from 'stream'; 8 | import { URL } from 'url'; 9 | 10 | import { assert as hoekAssert } from '@hapi/hoek'; 11 | import { AttrList } from 'm3u8parse'; 12 | 13 | import { DuplexEvents, TypedEmitter, TypedTransform } from 'hls-playlist-reader/lib/raw/typed-readable'; 14 | import { SegmentDownloader } from './segment-downloader'; 15 | 16 | import { types as MimeTypes } from 'mime-types'; 17 | 18 | /* eslint-disable @typescript-eslint/dot-notation */ 19 | MimeTypes['ac3'] = 'audio/ac3'; 20 | MimeTypes['eac3'] = 'audio/eac3'; 21 | MimeTypes['m4s'] = 'video/iso.segment'; 22 | /* eslint-enable @typescript-eslint/dot-notation */ 23 | 24 | 25 | // eslint-disable-next-line func-style 26 | function assert(condition: any, ...args: any[]): asserts condition { 27 | 28 | hoekAssert(condition, ...args); 29 | } 30 | 31 | // eslint-disable-next-line func-style 32 | function assertReaderObject(obj: any, message: string): asserts obj is HlsReaderObject { 33 | 34 | assert(typeof obj.msn === 'number' && obj.entry, message); 35 | } 36 | 37 | 38 | const internals = { 39 | segmentMimeTypes: new Set([ 40 | 'video/mp2t', 41 | 'video/mpeg', 42 | 'video/mp4', 43 | 'video/iso.segment', 44 | 'video/x-m4v', 45 | 'audio/aac', 46 | 'audio/x-aac', 47 | 'audio/ac3', 48 | 'audio/vnd.dolby.dd-raw', 49 | 'audio/x-ac3', 50 | 'audio/eac3', 51 | 'audio/mp4', 52 | 'text/vtt', 53 | 'application/mp4' 54 | ]), 55 | 56 | isSameMap(m1?: AttrList, m2?: AttrList) { 57 | 58 | return m1 === m2 || (m1 && m2 && m1.get('uri') === m2.get('uri') && m1.get('byterange') === m2.get('byterange')); 59 | }, 60 | 61 | isAbortedError(err: Error) { 62 | 63 | return err.message === 'Aborted'; 64 | } 65 | }; 66 | 67 | 68 | export class HlsStreamerObject { 69 | 70 | type: 'segment' | 'map'; 71 | file: FetchResult['meta']; 72 | stream?: Readable; 73 | segment?: HlsReaderObject; 74 | attrs?: AttrList; 75 | 76 | constructor(fileMeta: FetchResult['meta'], stream: Readable | undefined, type: 'map', details: AttrList); 77 | constructor(fileMeta: FetchResult['meta'], stream: Readable | undefined, type: 'segment', details: HlsReaderObject); 78 | 79 | constructor(fileMeta: FetchResult['meta'], stream: Readable | undefined, type: 'segment' | 'map', details: HlsReaderObject | AttrList) { 80 | 81 | const isSegment = type === 'segment'; 82 | 83 | this.type = type; 84 | this.file = fileMeta; 85 | this.stream = stream; 86 | 87 | if (isSegment) { 88 | this.segment = details as HlsReaderObject; 89 | } 90 | else { 91 | this.attrs = details as AttrList; 92 | } 93 | } 94 | } 95 | 96 | export type HlsSegmentStreamerOptions = { 97 | withData?: boolean; // default true 98 | highWaterMark?: number; 99 | }; 100 | 101 | const HlsSegmentStreamerEvents = >(null as any); 102 | interface IHlsSegmentStreamerEvents { 103 | problem(err: Error): void; 104 | } 105 | 106 | export class HlsSegmentStreamer extends TypedEmitter(HlsSegmentStreamerEvents, TypedTransform()) { 107 | 108 | baseUrl = 'unknown:'; 109 | readonly withData: boolean; 110 | 111 | #readState = new (class ReadState { 112 | indexTokens = new Set(); 113 | activeTokens = new Set(); 114 | 115 | //mapSeq: -1, 116 | map?: AttrList; 117 | fetching: Promise | null = null; 118 | //active: false; 119 | //discont: false; 120 | })(); 121 | 122 | #active = new Map(); // used to stop buffering on expired segments 123 | #downloader: SegmentDownloader; 124 | #reader?: HlsSegmentReader; 125 | #started = false; 126 | 127 | #onReaderIndex = this._onReaderIndex.bind(this); 128 | #onReaderProblem = this._onReaderProblem.bind(this); 129 | 130 | constructor(reader?: HlsSegmentReader, options: HlsSegmentStreamerOptions = {}) { 131 | 132 | super({ objectMode: true, allowHalfOpen: false, autoDestroy: false, writableHighWaterMark: 0, readableHighWaterMark: (reader as any)?.highWaterMark ?? options.highWaterMark ?? 0 }); 133 | 134 | // autoDestroy is broken for transform streams on node 14, so we need to manually emit 'close' after 'end' 135 | // Don't actually call destroy(), since it will trigger an abort() that aborts all tracked segment fetches 136 | 137 | this.on('end', () => process.nextTick(() => this.emit('close'))); 138 | 139 | if (typeof reader === 'object' && !(reader instanceof Stream)) { 140 | options = reader; 141 | reader = undefined; 142 | } 143 | 144 | this.withData = options.withData ?? true; 145 | 146 | this.#downloader = new SegmentDownloader({ probe: !this.withData }); 147 | 148 | this.on('pipe', (src: HlsSegmentReader) => { 149 | 150 | assert(!this.#reader, 'Only one piped source is supported'); 151 | assert(!src.feeder.index?.master, 'Source cannot be based on a master playlist'); 152 | 153 | this.#reader = src; 154 | src.on<'index'>('index', this.#onReaderIndex); 155 | src.on<'problem'>('problem', this.#onReaderProblem); 156 | 157 | if (src.index) { 158 | process.nextTick(this._onReaderIndex.bind(this, src.index, { url: src.feeder.baseUrl })); 159 | } 160 | 161 | this.baseUrl = src.feeder.baseUrl; 162 | }); 163 | 164 | this.on('unpipe', () => { 165 | 166 | this.#reader?.off<'index'>('index', this.#onReaderIndex); 167 | this.#reader?.off<'problem'>('problem', this.#onReaderProblem); 168 | this.#reader = undefined; 169 | }); 170 | 171 | // Pipe to self 172 | 173 | if (reader) { 174 | reader.on('error', (err) => { 175 | 176 | if (!this.destroyed) { 177 | this.destroy(err); 178 | } 179 | }).pipe(this); 180 | } 181 | } 182 | 183 | abort(graceful = false): void { 184 | 185 | if (!graceful) { 186 | this.#downloader.setValid(); 187 | } 188 | 189 | if (!this.readable) { 190 | return; 191 | } 192 | 193 | this.push(null); 194 | } 195 | 196 | _destroy(err: Error | null, cb: unknown): void { 197 | 198 | if (this.#reader && !this.#reader.destroyed) { 199 | this.#reader.destroy(err || undefined); 200 | } 201 | 202 | super._destroy(err, cb as any); 203 | 204 | this.abort(!!err); 205 | } 206 | 207 | get segmentMimeTypes(): Set { 208 | 209 | return internals.segmentMimeTypes; 210 | } 211 | 212 | validateSegmentMeta(meta: FetchResult['meta']): void | never { 213 | 214 | // Check for valid mime type 215 | 216 | if (!this.segmentMimeTypes.has(meta.mime.toLowerCase())) { 217 | throw new Error(`Unsupported segment MIME type: ${meta.mime}`); 218 | } 219 | } 220 | 221 | _transform(segment: HlsReaderObject | unknown, _: unknown, done: (err?: Error) => void): void { 222 | 223 | assertReaderObject(segment, 'Only segment-reader segments are supported'); 224 | 225 | this._process(segment).then(() => done(), (err) => { 226 | 227 | done(internals.isAbortedError(err) ? undefined : err); 228 | }); 229 | } 230 | 231 | // Private methods 232 | 233 | protected _onReaderIndex(index: Readonly, { url }: { url: string }): void { 234 | 235 | this.baseUrl = url; 236 | 237 | if (index.master) { 238 | this.destroy(new Error('The reader source is a master playlist')); 239 | return; 240 | } 241 | 242 | // Update active token list 243 | 244 | this._updateTokens(index); 245 | this.#downloader.setValid(this.#readState.activeTokens); 246 | } 247 | 248 | protected _onReaderProblem(err: Error): void { 249 | 250 | this.emit<'problem'>('problem', err); 251 | } 252 | 253 | private async _process(segment: HlsReaderObject): Promise { 254 | 255 | // Check for new map entry 256 | 257 | if (!internals.isSameMap(segment.entry.map, this.#readState.map)) { 258 | this.#readState.map = segment.entry.map; 259 | 260 | // Fetch init segment 261 | 262 | if (segment.entry.map) { 263 | const uri = segment.entry.map.get('uri', AttrList.Types.String); 264 | assert(uri, 'EXT-X-MAP must have URI attribute'); 265 | let byterange: Required | undefined; 266 | if (segment.entry.map.has('byterange')) { 267 | byterange = Object.assign({ offset: 0 }, segment.entry.map.get('byterange', AttrList.Types.Byterange)!); 268 | } 269 | 270 | // Fetching the map is essential to the processing 271 | 272 | let fetch: FetchResult | undefined; 273 | let tries = 0; 274 | do { 275 | try { 276 | tries++; 277 | fetch = await this._fetchFrom(this._tokenForMsn(segment.msn, segment.entry.map), { uri, byterange }); 278 | } 279 | catch (err: any) { 280 | if (tries >= 4) { 281 | throw err; 282 | } 283 | 284 | this.emit('problem', new Error('Failed to fetch map: ' + err.message)); 285 | 286 | // delay and retry 287 | 288 | await new Promise((resolve) => setTimeout(resolve, 200 * (segment.entry.duration || 4))); 289 | assert(!this.destroyed, 'destroyed'); 290 | } 291 | } while (!fetch); 292 | 293 | assert(!this.destroyed, 'destroyed'); 294 | this.push(new HlsStreamerObject(fetch.meta, fetch.stream, 'map', segment.entry.map)); 295 | 296 | // It is a fatal inconsistency error, if the map stream fails to download 297 | 298 | if (fetch.stream) { 299 | fetch.stream.on('error', (err) => { 300 | 301 | this.destroy(new Error('Failed to download map data: ' + err.message)); 302 | }); 303 | } 304 | } 305 | } 306 | 307 | // Fetch the segment 308 | 309 | await segment.closed(); 310 | if (segment.entry.isPartial()) { 311 | return; 312 | } 313 | 314 | const fetch = await this._fetchFrom(this._tokenForMsn(segment.msn), { uri: segment.entry.uri!, byterange: segment.entry.byterange }); 315 | assert(!this.destroyed, 'destroyed'); 316 | 317 | // At this point object.stream has only been readied / opened 318 | 319 | let stream = fetch.stream; 320 | try { 321 | // Check meta 322 | 323 | if (this.#reader && fetch.meta.modified) { 324 | const segmentTime = segment.entry.program_time || new Date(+fetch.meta.modified - (segment.entry.duration || 0) * 1000); 325 | 326 | if (!this.#started && this.#reader.startDate && 327 | segmentTime < this.#reader.startDate) { 328 | 329 | return; // Too early - ignore segment 330 | } 331 | 332 | if (this.#reader.stopDate && 333 | segmentTime > this.#reader.stopDate) { 334 | 335 | this.push(null); 336 | return; 337 | } 338 | } 339 | 340 | // Track embedded stream to append more parts later 341 | 342 | if (stream) { 343 | this.#active.set(segment.msn, fetch); 344 | finished(stream, () => this.#active.delete(segment.msn)); 345 | } 346 | 347 | this.push(new HlsStreamerObject(fetch.meta, stream, 'segment', segment)); 348 | stream = undefined; // Don't destroy 349 | 350 | this.#started = true; 351 | } 352 | finally { 353 | stream?.destroy(); 354 | } 355 | } 356 | 357 | private async _fetchFrom(token: number | string, part: { uri: string; byterange?: Required }) { 358 | 359 | const { uri, byterange } = part; 360 | const fetch = await this.#downloader.fetchSegment(token, new URL(uri, this.baseUrl), byterange); 361 | 362 | try { 363 | this.validateSegmentMeta(fetch.meta); 364 | 365 | return fetch; 366 | } 367 | catch (err) { 368 | if (fetch.stream && !fetch.stream.destroyed) { 369 | fetch.stream.destroy(); 370 | } 371 | 372 | throw err; 373 | } 374 | } 375 | 376 | private _tokenForMsn(msn: number, map?: AttrList): number | string { 377 | 378 | if (map) { 379 | return map.toString(); 380 | } 381 | 382 | return msn; // TODO: handle start over – add generation 383 | } 384 | 385 | private _updateTokens(index: Readonly) { 386 | 387 | const old = this.#readState.indexTokens; 388 | 389 | const current = new Set(); 390 | for (let i = index.startMsn(true); i <= index.lastMsn(true); ++i) { 391 | const segment = index.getSegment(i); 392 | if (segment) { 393 | const token = this._tokenForMsn(i); 394 | current.add(token); 395 | old.delete(token); 396 | 397 | const map = segment.map; 398 | if (map) { 399 | const mapToken = this._tokenForMsn(i, map); 400 | current.add(mapToken); 401 | old.delete(mapToken); 402 | } 403 | } 404 | } 405 | 406 | this.#readState.indexTokens = current; 407 | this.#readState.activeTokens = new Set([...old, ...current]); 408 | } 409 | } 410 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hls-segment-reader", 3 | "version": "8.0.1", 4 | "description": "Read segments from HLS streams using a Readable", 5 | "main": "lib", 6 | "scripts": { 7 | "clean": "rm -f lib/*.{js,d.ts}", 8 | "postpack": "npm run clean", 9 | "prepack": "tsc", 10 | "test": "lab -t 90", 11 | "test-full": "npm test && npm run test-js", 12 | "test-cov-html": "lab -c -r html -o coverage.html", 13 | "test-js": "tsc && lab --transform '' --t 0 && npm run clean" 14 | }, 15 | "keywords": [ 16 | "streaming", 17 | "live", 18 | "video", 19 | "audio", 20 | "m3u8" 21 | ], 22 | "author": "Gil Pedersen ", 23 | "license": "BSD-2-Clause", 24 | "engines": { 25 | "node": "^14.17.0" 26 | }, 27 | "dependencies": { 28 | "@hapi/hoek": "^10.0.0", 29 | "hls-playlist-reader": "^2.0.0", 30 | "m3u8parse": "^3.2.0" 31 | }, 32 | "devDependencies": { 33 | "@hapi/boom": "^10.0.0", 34 | "@hapi/code": "^9.0.1", 35 | "@hapi/eslint-plugin": "^6.0.0", 36 | "@hapi/hapi": "^20.0.0", 37 | "@hapi/inert": "^6.0.1", 38 | "@hapi/lab": "^25.0.1", 39 | "@types/mime-types": "^2.1.0", 40 | "@typescript-eslint/eslint-plugin": "^5.29.0", 41 | "@typescript-eslint/parser": "^5.29.0", 42 | "eslint": "^8.18.0", 43 | "joi": "^17.2.0", 44 | "typescript": "^4.4.3", 45 | "uristream": "^6.3.0" 46 | }, 47 | "repository": { 48 | "type": "git", 49 | "url": "git+https://github.com/kanongil/node-hls-segment-reader.git" 50 | }, 51 | "bugs": { 52 | "url": "https://github.com/kanongil/node-hls-segment-reader/issues" 53 | }, 54 | "homepage": "https://github.com/kanongil/node-hls-segment-reader" 55 | } 56 | -------------------------------------------------------------------------------- /test/_shared.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Fs = require('fs'); 4 | const Path = require('path'); 5 | const Readable = require('stream').Readable; 6 | 7 | const Hapi = require('@hapi/hapi'); 8 | const Hoek = require('@hapi/hoek'); 9 | const Inert = require('@hapi/inert'); 10 | const Joi = require('joi'); 11 | const M3U8Parse = require('m3u8parse'); 12 | 13 | 14 | exports.provisionServer = () => { 15 | 16 | const server = new Hapi.Server({ 17 | routes: { files: { relativeTo: Path.join(__dirname, 'fixtures') } } 18 | }); 19 | 20 | server.register(Inert); 21 | 22 | const delay = async (request, h) => { 23 | 24 | await Hoek.wait(200); 25 | 26 | return 200; 27 | }; 28 | 29 | const slowServe = (request, h) => { 30 | 31 | const slowStream = new Readable(); 32 | slowStream._read = () => { }; 33 | 34 | const path = Path.join(__dirname, 'fixtures', request.params.path); 35 | const buffer = Fs.readFileSync(path); 36 | slowStream.push(buffer.slice(0, 5000)); 37 | setTimeout(() => { 38 | 39 | slowStream.push(buffer.slice(5000)); 40 | slowStream.push(null); 41 | }, 200); 42 | 43 | return h.response(slowStream).type('video/mp2t').header('content-length', buffer.byteLength); 44 | }; 45 | 46 | server.route({ method: 'GET', path: '/simple/{path*}', handler: { directory: { path: '.' } } }); 47 | server.route({ method: 'GET', path: '/slow/{path*}', handler: { directory: { path: '.' } }, config: { pre: [{ method: delay, assign: 'delay' }] } }); 48 | server.route({ method: 'GET', path: '/slow-data/{path*}', handler: slowServe }); 49 | server.route({ 50 | method: 'GET', path: '/error', handler(request, h) { 51 | 52 | throw new Error('!!!'); 53 | } 54 | }); 55 | 56 | return server; 57 | }; 58 | 59 | exports.provisionLiveServer = function (shared) { 60 | 61 | const server = new Hapi.Server({ 62 | routes: { 63 | files: { relativeTo: Path.join(__dirname, 'fixtures') } 64 | } 65 | }); 66 | 67 | const serveLiveIndex = async (request, h) => { 68 | 69 | let index; 70 | if (shared.state.index) { 71 | index = await shared.state.index(request.query); 72 | } 73 | else { 74 | index = exports.genIndex(shared.state); 75 | } 76 | 77 | return h.response(index.toString()).type('application/vnd.apple.mpegURL'); 78 | }; 79 | 80 | const serveSegment = (request, h) => { 81 | 82 | if (shared.state.slow) { 83 | const slowStream = new Readable({ read: Hoek.ignore }); 84 | 85 | slowStream.push(Buffer.alloc(5000)); 86 | 87 | return h.response(slowStream).type('video/mp2t').bytes(30000); 88 | } 89 | 90 | const size = ~~(5000 / (request.params.part === undefined ? 1 : shared.state.partCount)) + parseInt(request.params.msn) + 100 * parseInt(request.params.part || 0); 91 | 92 | if (shared.state.unstable) { 93 | --shared.state.unstable; 94 | 95 | const unstableStream = new Readable({ read: Hoek.ignore }); 96 | 97 | unstableStream.push(Buffer.alloc(50 - shared.state.unstable)); 98 | unstableStream.push(null); 99 | 100 | unstableStream.once('end', () => { 101 | 102 | // Manually destroy socket in case it is a keep-alive connection 103 | // otherwise the receiver will never know that the request is done 104 | 105 | request.raw.req.destroy(); 106 | }); 107 | 108 | return h.response(unstableStream).type('video/mp2t').bytes(size); 109 | } 110 | 111 | return h.response(Buffer.alloc(size)).type('video/mp2t').bytes(size); 112 | }; 113 | 114 | server.route({ 115 | method: 'GET', 116 | path: '/live/live.m3u8', 117 | handler: serveLiveIndex, 118 | options: { 119 | validate: { 120 | query: Joi.object({ 121 | '_HLS_msn': Joi.number().integer().min(0).optional(), 122 | '_HLS_part': Joi.number().min(0).optional() 123 | }).with('_HLS_part', '_HLS_msn') 124 | } 125 | } 126 | }); 127 | server.route({ method: 'GET', path: '/live/{msn}.ts', handler: serveSegment }); 128 | server.route({ method: 'GET', path: '/live/{msn}-part{part}.ts', handler: serveSegment }); 129 | 130 | return server; 131 | }; 132 | 133 | 134 | exports.readSegments = (Class, ...args) => { 135 | 136 | let r; 137 | const promise = new Promise((resolve, reject) => { 138 | 139 | r = new Class(...args); 140 | r.on('error', reject); 141 | 142 | const segments = []; 143 | r.on('data', (segment) => { 144 | 145 | segments.push(segment); 146 | }); 147 | 148 | r.on('end', () => { 149 | 150 | resolve(segments); 151 | }); 152 | }); 153 | 154 | promise.reader = r; 155 | 156 | return promise; 157 | }; 158 | 159 | 160 | exports.genIndex = function ({ targetDuration, segmentCount, firstMsn, partCount, partIndex, ended }) { 161 | 162 | const partDuration = targetDuration / partCount; 163 | 164 | const segments = []; 165 | const meta = {}; 166 | 167 | for (let i = 0; i < segmentCount; ++i) { 168 | const parts = []; 169 | if (i >= segmentCount - 2) { 170 | for (let j = 0; j < partCount; ++j) { 171 | parts.push(new M3U8Parse.AttrList({ 172 | duration: partDuration, 173 | uri: `"${firstMsn + i}-part${j}.ts"` 174 | })); 175 | } 176 | } 177 | 178 | segments.push({ 179 | duration: targetDuration || 2, 180 | uri: `${firstMsn + i}.ts`, 181 | title: '', 182 | parts: parts.length ? parts : undefined 183 | }); 184 | } 185 | 186 | if (partIndex !== undefined) { 187 | if (partIndex > 0) { 188 | const parts = []; 189 | for (let i = 0; i < partIndex; ++i) { 190 | parts.push(new M3U8Parse.AttrList({ 191 | duration: partDuration, 192 | uri: `"${firstMsn + segmentCount}-part${i}.ts"` 193 | })); 194 | } 195 | 196 | segments.push({ parts }); 197 | } 198 | 199 | // Add hint 200 | 201 | if (!ended) { 202 | meta.preload_hints = [new M3U8Parse.AttrList({ 203 | type: 'part', 204 | uri: `"${firstMsn + segmentCount}-part${partIndex}.ts"` 205 | })]; 206 | } 207 | } 208 | 209 | const index = new M3U8Parse.MediaPlaylist({ 210 | media_sequence: firstMsn, 211 | target_duration: targetDuration, 212 | part_info: partCount ? new M3U8Parse.AttrList({ 'part-target': partDuration }) : undefined, 213 | segments, 214 | meta, 215 | ended 216 | }); 217 | 218 | //console.log('GEN', index.startMsn(true), index.lastMsn(true), index.meta.preload_hints, index.ended); 219 | 220 | return index; 221 | }; 222 | 223 | 224 | exports.genLlIndex = function (query, state) { 225 | 226 | // Return playlist with exactly the next part 227 | 228 | if (!state.ended && query._HLS_msn !== undefined) { 229 | let msn = query._HLS_msn; 230 | let part = query._HLS_part === undefined ? state.partCount : query._HLS_part + 1; 231 | 232 | if (part >= state.partCount) { 233 | msn++; 234 | part = 0; 235 | } 236 | 237 | state.firstMsn = msn - state.segmentCount; 238 | state.partIndex = part; 239 | } 240 | 241 | const index = exports.genIndex(state); 242 | 243 | index.server_control = new M3U8Parse.AttrList({ 244 | 'can-block-reload': 'YES', 245 | 'part-hold-back': 3 * state.targetDuration / state.partCount 246 | }); 247 | 248 | state.genCount = (state.genCount || 0) + 1; 249 | 250 | if (!state.ended) { 251 | if (state.end && 252 | index.lastMsn() > state.end.msn || (index.lastMsn() === state.end.msn && state.end.part === index.getSegment(index.lastMsn()).parts.length)) { 253 | 254 | index.ended = state.ended = true; 255 | delete index.meta.preload_hints; 256 | return index; 257 | } 258 | 259 | state.partIndex = ~~state.partIndex + 1; 260 | if (state.partIndex >= state.partCount) { 261 | state.partIndex = 0; 262 | state.firstMsn++; 263 | } 264 | } 265 | 266 | return index; 267 | }; 268 | -------------------------------------------------------------------------------- /test/fixtures/500-00000.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/500-00000.ts -------------------------------------------------------------------------------- /test/fixtures/500-00001.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/500-00001.ts -------------------------------------------------------------------------------- /test/fixtures/500-00002.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/500-00002.ts -------------------------------------------------------------------------------- /test/fixtures/500.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-MY-HEADER:hello 3 | #EXT-X-TARGETDURATION:4 4 | #EXT-X-VERSION:3 5 | #EXT-X-MEDIA-SEQUENCE:0 6 | #EXT-X-PLAYLIST-TYPE:EVENT 7 | #EXT-X-START:TIME-OFFSET=0 8 | #EXT-X-PROGRAM-DATE-TIME:2000-01-07T07:03:05+0100 9 | #EXTINF:4.000, 10 | 500-00000.ts 11 | #EXT-X-PROGRAM-DATE-TIME:2000-01-07T07:03:09+0100 12 | #EXTINF:4.000, 13 | #EXT-MY-SEGMENT-OK 14 | 500-00001.ts 15 | #EXT-X-PROGRAM-DATE-TIME:2000-01-07T07:03:12+0100 16 | #EXTINF:0.760, 17 | 500-00002.ts 18 | #EXT-X-ENDLIST 19 | -------------------------------------------------------------------------------- /test/fixtures/500.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/500.ts -------------------------------------------------------------------------------- /test/fixtures/badtype-data.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-TARGETDURATION:4 3 | #EXTINF:4.000, 4 | data:video/x-unknown;charset=utf-8,hello 5 | #EXT-X-ENDLIST -------------------------------------------------------------------------------- /test/fixtures/badtype.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-TARGETDURATION:4 3 | #EXTINF:4.000, 4 | 500.m3u8 5 | #EXT-X-ENDLIST -------------------------------------------------------------------------------- /test/fixtures/discont.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-TARGETDURATION:4 3 | #EXT-X-MEDIA-SEQUENCE:0 4 | #EXT-X-PLAYLIST-TYPE:EVENT 5 | #EXTINF:4.000, 6 | 500-00000.ts 7 | #EXT-X-DISCONTINUITY 8 | #EXTINF:4.000, 9 | 500-00001.ts 10 | #EXTINF:0.760, 11 | 500-00002.ts 12 | #EXT-X-ENDLIST 13 | -------------------------------------------------------------------------------- /test/fixtures/files/audio.aac: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/files/audio.aac -------------------------------------------------------------------------------- /test/fixtures/files/audio.ac3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/files/audio.ac3 -------------------------------------------------------------------------------- /test/fixtures/files/audio.dts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/files/audio.dts -------------------------------------------------------------------------------- /test/fixtures/files/audio.eac3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/files/audio.eac3 -------------------------------------------------------------------------------- /test/fixtures/files/audio.m4a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/files/audio.m4a -------------------------------------------------------------------------------- /test/fixtures/files/file.m4s: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/files/file.m4s -------------------------------------------------------------------------------- /test/fixtures/files/file.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/files/file.mp4 -------------------------------------------------------------------------------- /test/fixtures/files/text.vtt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/files/text.vtt -------------------------------------------------------------------------------- /test/fixtures/files/video.m4v: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kanongil/hls-segment-reader/e45c80f17982a3b29545d9c01e4685f54a638b69/test/fixtures/files/video.m4v -------------------------------------------------------------------------------- /test/fixtures/index.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=2704940,AVERAGE-BANDWIDTH=2147880,CODECS="avc1.64001e,mp4a.40.2",RESOLUTION=704x576,CLOSED-CAPTIONS=NONE 3 | 500.m3u8 4 | -------------------------------------------------------------------------------- /test/fixtures/long.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-TARGETDURATION:4 3 | #EXT-X-MEDIA-SEQUENCE:0 4 | #EXTINF:4.000, 5 | 500-00000.ts 6 | #EXTINF:4.000, 7 | #EXT-MY-SEGMENT-OK 8 | 500-00001.ts 9 | #EXTINF:0.760, 10 | 500-00002.ts 11 | #EXT-X-DISCONTINUITY 12 | #EXTINF:4.000, 13 | 500-00000.ts 14 | #EXTINF:4.000, 15 | #EXT-MY-SEGMENT-OK 16 | 500-00001.ts 17 | #EXTINF:0.760, 18 | 500-00002.ts 19 | #EXT-X-ENDLIST 20 | -------------------------------------------------------------------------------- /test/fixtures/malformed.m3u8: -------------------------------------------------------------------------------- 1 | hello! -------------------------------------------------------------------------------- /test/fixtures/single.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-TARGETDURATION:4 3 | #EXT-X-VERSION:5 4 | #EXT-X-MEDIA-SEQUENCE:0 5 | #EXT-X-PLAYLIST-TYPE:EVENT 6 | #EXT-X-START:TIME-OFFSET=0 7 | #EXT-X-MAP:URI="500.ts",BYTERANGE="376@0" 8 | #EXT-X-BYTERANGE:748428@0 9 | #EXTINF:4.000, 10 | 500.ts 11 | #EXT-X-PROGRAM-DATE-TIME:2000-01-07T07:03:09+0100 12 | #EXT-X-BYTERANGE:721168 13 | #EXTINF:4.000, 14 | 500.ts 15 | #EXT-X-PROGRAM-DATE-TIME:2000-01-07T07:03:12+0100 16 | #EXT-X-BYTERANGE:295724 17 | #EXTINF:0.760, 18 | 500.ts 19 | #EXT-X-ENDLIST 20 | -------------------------------------------------------------------------------- /test/fixtures/slow.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-TARGETDURATION:4 3 | #EXT-X-VERSION:3 4 | #EXT-X-MEDIA-SEQUENCE:0 5 | #EXT-X-PLAYLIST-TYPE:EVENT 6 | #EXT-X-START:TIME-OFFSET=0 7 | #EXT-X-PROGRAM-DATE-TIME:2000-01-07T07:03:05+0100 8 | #EXTINF:4.000, 9 | 500-00000.ts 10 | #EXT-X-PROGRAM-DATE-TIME:2000-01-07T07:03:09+0100 11 | #EXTINF:4.000, 12 | /slow-data/500-00001.ts 13 | #EXT-X-PROGRAM-DATE-TIME:2000-01-07T07:03:12+0100 14 | #EXTINF:0.760, 15 | /slow/500-00002.ts 16 | #EXT-X-ENDLIST 17 | -------------------------------------------------------------------------------- /test/reader.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Events = require('events'); 4 | const Fs = require('fs'); 5 | const Os = require('os'); 6 | const Path = require('path'); 7 | const Url = require('url'); 8 | 9 | const Boom = require('@hapi/boom'); 10 | const Code = require('@hapi/code'); 11 | const Hoek = require('@hapi/hoek'); 12 | const Lab = require('@hapi/lab'); 13 | const M3U8Parse = require('m3u8parse'); 14 | 15 | const Shared = require('./_shared'); 16 | const { HlsSegmentReader, HlsPlaylistReader } = require('..'); 17 | 18 | 19 | // Test shortcuts 20 | 21 | const lab = exports.lab = Lab.script(); 22 | const { after, before, describe, it } = lab; 23 | const { expect } = Code; 24 | 25 | 26 | describe('HlsSegmentReader()', () => { 27 | 28 | const readSegments = Shared.readSegments.bind(null, HlsSegmentReader); 29 | let server; 30 | 31 | before(async () => { 32 | 33 | server = await Shared.provisionServer(); 34 | return server.start(); 35 | }); 36 | 37 | after(() => { 38 | 39 | return server.stop(); 40 | }); 41 | 42 | describe('constructor', () => { 43 | 44 | it('creates a valid object', async () => { 45 | 46 | const r = new HlsSegmentReader('http://localhost:' + server.info.port + '/simple/500.m3u8'); 47 | const closed = Events.once(r, 'close'); 48 | 49 | expect(r).to.be.instanceOf(HlsSegmentReader); 50 | 51 | await Hoek.wait(10); 52 | 53 | r.destroy(); 54 | 55 | await closed; 56 | }); 57 | 58 | it('throws on missing uri option', () => { 59 | 60 | const createObject = () => { 61 | 62 | return new HlsSegmentReader(); 63 | }; 64 | 65 | expect(createObject).to.throw(); 66 | }); 67 | 68 | it('throws on invalid uri option', () => { 69 | 70 | const createObject = () => { 71 | 72 | return new HlsSegmentReader('asdf://test'); 73 | }; 74 | 75 | expect(createObject).to.throw(); 76 | }); 77 | }); 78 | 79 | describe('master index', () => { 80 | 81 | it('does not output any segments', async () => { 82 | 83 | const segments = await readSegments(`http://localhost:${server.info.port}/simple/index.m3u8`); 84 | expect(segments.length).to.equal(0); 85 | }); 86 | 87 | it('emits "index" event', async () => { 88 | 89 | const promise = readSegments(`http://localhost:${server.info.port}/simple/index.m3u8`); 90 | 91 | let remoteIndex; 92 | promise.reader.on('index', (index) => { 93 | 94 | remoteIndex = index; 95 | }); 96 | 97 | await promise; 98 | 99 | expect(remoteIndex).to.exist(); 100 | expect(remoteIndex.master).to.be.true(); 101 | expect(remoteIndex.variants[0].uri).to.exist(); 102 | }); 103 | }); 104 | 105 | describe('on-demand index', () => { 106 | 107 | it('outputs all segments', async () => { 108 | 109 | const segments = await readSegments(`http://localhost:${server.info.port}/simple/500.m3u8`); 110 | 111 | expect(segments.length).to.equal(3); 112 | for (let i = 0; i < segments.length; ++i) { 113 | expect(segments[i].msn).to.equal(i); 114 | } 115 | }); 116 | 117 | it('emits the "index" event before starting', async () => { 118 | 119 | const promise = readSegments(`http://localhost:${server.info.port}/simple/500.m3u8`); 120 | 121 | let hasSegment = false; 122 | promise.reader.on('data', () => { 123 | 124 | hasSegment = true; 125 | }); 126 | 127 | const index = await new Promise((resolve) => { 128 | 129 | promise.reader.on('index', resolve); 130 | }); 131 | 132 | expect(index).to.exist(); 133 | expect(hasSegment).to.be.false(); 134 | 135 | await promise; 136 | 137 | expect(hasSegment).to.be.true(); 138 | }); 139 | 140 | it('supports the startDate option', async () => { 141 | 142 | const r = new HlsSegmentReader(`http://localhost:${server.info.port}/simple/500.m3u8`, { startDate: new Date('Fri Jan 07 2000 07:03:09 GMT+0100 (CET)') }); 143 | const segments = []; 144 | 145 | for await (const segment of r) { 146 | expect(segment.msn).to.equal(segments.length + 2); 147 | segments.push(segment); 148 | } 149 | 150 | expect(segments).to.have.length(1); 151 | }); 152 | 153 | it('supports the stopDate option', async () => { 154 | 155 | const r = new HlsSegmentReader(`http://localhost:${server.info.port}/simple/500.m3u8`, { stopDate: new Date('Fri Jan 07 2000 07:03:09 GMT+0100 (CET)') }); 156 | const segments = []; 157 | 158 | for await (const segment of r) { 159 | expect(segment.msn).to.equal(segments.length); 160 | segments.push(segment); 161 | } 162 | 163 | expect(segments).to.have.length(2); 164 | }); 165 | 166 | it('applies the extensions option', async () => { 167 | 168 | const extensions = { 169 | '#EXT-MY-HEADER': false, 170 | '#EXT-MY-SEGMENT-OK': true 171 | }; 172 | 173 | const r = new HlsSegmentReader('file://' + Path.join(__dirname, 'fixtures', '500.m3u8'), { extensions }); 174 | const segments = []; 175 | 176 | for await (const obj of r) { 177 | segments.push(obj); 178 | } 179 | 180 | expect(r.index).to.exist(); 181 | expect(r.index.vendor[0]).to.equal(['#EXT-MY-HEADER', 'hello']); 182 | expect(r.index.segments[1].vendor[0]).to.equal(['#EXT-MY-SEGMENT-OK', null]); 183 | expect(segments).to.have.length(3); 184 | expect(segments[1].entry.vendor[0]).to.equal(['#EXT-MY-SEGMENT-OK', null]); 185 | }); 186 | 187 | it('does not internally buffer', async () => { 188 | 189 | const r = new HlsSegmentReader('file://' + Path.join(__dirname, 'fixtures', 'long.m3u8')); 190 | 191 | for await (const obj of r) { 192 | expect(obj).to.exist(); 193 | await Hoek.wait(20); 194 | expect(r._readableState.buffer).to.have.length(0); 195 | } 196 | }); 197 | 198 | /*it('supports the highWaterMark option', async () => { 199 | 200 | const r = new HlsSegmentReader('file://' + Path.join(__dirname, 'fixtures', 'long.m3u8'), { highWaterMark: 2 }); 201 | const buffered = []; 202 | 203 | for await (const obj of r) { 204 | expect(obj).to.exist(); 205 | await Hoek.wait(20); 206 | buffered.push(r._readableState.buffer.length); 207 | } 208 | 209 | expect(buffered).to.equal([2, 2, 2, 2, 1, 0]); 210 | });*/ 211 | 212 | it('can be destroyed', async () => { 213 | 214 | const r = new HlsSegmentReader('file://' + Path.join(__dirname, 'fixtures', '500.m3u8')); 215 | const segments = []; 216 | 217 | for await (const obj of r) { 218 | segments.push(obj); 219 | r.destroy(); 220 | } 221 | 222 | expect(segments).to.have.length(1); 223 | }); 224 | 225 | // handles all kinds of segment reference url 226 | // handles .m3u files 227 | }); 228 | 229 | describe('live index', { parallel: false }, () => { 230 | 231 | const serverState = { state: {} }; 232 | let liveServer; 233 | 234 | const prepareLiveReader = function (readerOptions = {}, state = {}) { 235 | 236 | const reader = new HlsSegmentReader(`http://localhost:${liveServer.info.port}/live/live.m3u8`, { fullStream: true, ...readerOptions }); 237 | reader.feeder._intervals = []; 238 | reader.feeder.getUpdateInterval = function (...args) { 239 | 240 | this._intervals.push(HlsPlaylistReader.prototype.getUpdateInterval.call(this, ...args)); 241 | return undefined; 242 | }; 243 | 244 | serverState.state = { firstMsn: 0, segmentCount: 10, targetDuration: 2, ...state }; 245 | 246 | return { reader, state: serverState.state }; 247 | }; 248 | 249 | before(() => { 250 | 251 | liveServer = Shared.provisionLiveServer(serverState); 252 | return liveServer.start(); 253 | }); 254 | 255 | after(() => { 256 | 257 | return liveServer.stop(); 258 | }); 259 | 260 | it('handles a basic stream (http)', async () => { 261 | 262 | const { reader, state } = prepareLiveReader(); 263 | const segments = []; 264 | 265 | for await (const obj of reader) { 266 | expect(obj.msn).to.equal(segments.length); 267 | segments.push(obj); 268 | 269 | if (obj.msn > 5) { 270 | state.firstMsn++; 271 | if (state.firstMsn >= 5) { 272 | state.firstMsn = 5; 273 | state.ended = true; 274 | } 275 | } 276 | } 277 | 278 | expect(segments).to.have.length(15); 279 | }); 280 | 281 | it('handles a basic stream (file)', async () => { 282 | 283 | const state = serverState.state = { firstMsn: 0, segmentCount: 10, targetDuration: 10 }; 284 | 285 | const tmpDir = await Fs.promises.mkdtemp(await Fs.promises.realpath(Os.tmpdir()) + Path.sep); 286 | try { 287 | const tmpUrl = new URL('next.m3u8', Url.pathToFileURL(tmpDir + Path.sep)); 288 | const indexUrl = new URL('index.m3u8', Url.pathToFileURL(tmpDir + Path.sep)); 289 | await Fs.promises.writeFile(indexUrl, Shared.genIndex(state).toString(), 'utf-8'); 290 | 291 | const reader = new HlsSegmentReader(indexUrl.href, { fullStream: true }); 292 | const segments = []; 293 | 294 | (async () => { 295 | 296 | while (!state.ended) { 297 | await Hoek.wait(50); 298 | 299 | state.firstMsn++; 300 | if (state.firstMsn === 5) { 301 | state.ended = true; 302 | } 303 | 304 | // Atomic write 305 | 306 | await Fs.promises.writeFile(tmpUrl, Shared.genIndex(state).toString(), 'utf-8'); 307 | await Fs.promises.rename(tmpUrl, indexUrl); 308 | } 309 | })(); 310 | 311 | for await (const obj of reader) { 312 | expect(obj.msn).to.equal(segments.length); 313 | segments.push(obj); 314 | } 315 | 316 | expect(segments).to.have.length(15); 317 | } 318 | finally { 319 | await Fs.promises.rm(tmpDir, { recursive: true }); 320 | } 321 | }); 322 | 323 | it('can start with 0 segments', async () => { 324 | 325 | const { reader, state } = prepareLiveReader({}, { segmentCount: 0, index() { 326 | 327 | const index = Shared.genIndex(state); 328 | index.type = 'EVENT'; 329 | 330 | if (state.segmentCount === 5) { 331 | state.ended = true; 332 | } 333 | else { 334 | state.segmentCount++; 335 | } 336 | 337 | return index; 338 | } }); 339 | const segments = []; 340 | 341 | for await (const obj of reader) { 342 | expect(obj.msn).to.equal(segments.length); 343 | segments.push(obj); 344 | } 345 | 346 | expect(segments).to.have.length(5); 347 | }); 348 | 349 | it('emits "close" event when destroyed without consuming', async () => { 350 | 351 | const { reader } = prepareLiveReader(); 352 | 353 | const closeEvent = Events.once(reader, 'close'); 354 | 355 | const playlist = await reader._waitForUpdate(); 356 | expect(playlist).to.exist(); 357 | 358 | reader.destroy(); 359 | await closeEvent; 360 | 361 | expect(reader.destroyed).to.be.true(); 362 | }); 363 | 364 | it('handles sequence number resets', async () => { 365 | 366 | let reset = false; 367 | const { reader, state } = prepareLiveReader({}, { 368 | firstMsn: 9, 369 | segmentCount: 5, 370 | index() { 371 | 372 | if (!state.ended) { 373 | if (!reset) { 374 | state.firstMsn++; 375 | 376 | if (state.firstMsn === 13) { 377 | state.firstMsn = 0; 378 | state.segmentCount = 1; 379 | reset = true; 380 | } 381 | } 382 | else { 383 | state.segmentCount++; 384 | if (state.segmentCount === 5) { 385 | state.ended = true; 386 | } 387 | } 388 | } 389 | 390 | return Shared.genIndex(state); 391 | } 392 | }); 393 | 394 | const segments = []; 395 | for await (const obj of reader) { 396 | segments.push(obj); 397 | } 398 | 399 | expect(segments).to.have.length(12); 400 | expect(segments[7].msn).to.equal(0); 401 | expect(segments[6].entry.discontinuity).to.be.false(); 402 | expect(segments[7].entry.discontinuity).to.be.true(); 403 | expect(segments[8].entry.discontinuity).to.be.false(); 404 | }); 405 | 406 | it('handles sequence number jumps', async () => { 407 | 408 | let skipped = false; 409 | const { reader, state } = prepareLiveReader({}, { 410 | index() { 411 | 412 | const index = Shared.genIndex(state); 413 | 414 | if (!skipped) { 415 | ++state.firstMsn; 416 | if (state.firstMsn === 5) { 417 | state.firstMsn = 50; 418 | skipped = true; 419 | } 420 | } 421 | else if (skipped) { 422 | ++state.firstMsn; 423 | if (state.firstMsn === 55) { 424 | state.ended = true; 425 | } 426 | } 427 | 428 | return index; 429 | } 430 | }); 431 | 432 | const segments = []; 433 | for await (const obj of reader) { 434 | segments.push(obj); 435 | } 436 | 437 | expect(segments).to.have.length(29); 438 | expect(segments[13].msn).to.equal(13); 439 | expect(segments[13].entry.discontinuity).to.be.false(); 440 | expect(segments[14].msn).to.equal(50); 441 | expect(segments[14].entry.discontinuity).to.be.true(); 442 | expect(segments[15].entry.discontinuity).to.be.false(); 443 | }); 444 | 445 | it('handles a temporary server outage', async () => { 446 | 447 | const { reader, state } = prepareLiveReader({}, { 448 | index() { 449 | 450 | if (state.error === undefined && state.firstMsn === 5) { 451 | state.error = 6; 452 | } 453 | 454 | if (state.error) { 455 | --state.error; 456 | ++state.firstMsn; 457 | throw new Error('fail'); 458 | } 459 | 460 | if (state.firstMsn === 20) { 461 | state.ended = true; 462 | } 463 | 464 | const index = Shared.genIndex(state); 465 | 466 | ++state.firstMsn; 467 | 468 | return index; 469 | } 470 | }); 471 | 472 | const errors = []; 473 | reader.on('problem', errors.push.bind(errors)); 474 | 475 | const segments = []; 476 | for await (const obj of reader) { 477 | expect(obj.msn).to.equal(segments.length); 478 | segments.push(obj); 479 | } 480 | 481 | expect(segments).to.have.length(30); 482 | expect(errors.length).to.be.greaterThan(0); 483 | expect(errors[0]).to.be.an.error('Internal Server Error'); 484 | }); 485 | 486 | it('drops segments when reader is slow', async () => { 487 | 488 | const { reader, state } = prepareLiveReader({ fullStream: false }, { 489 | index() { 490 | 491 | if (state.firstMsn === 50) { 492 | state.ended = true; 493 | } 494 | 495 | const index = Shared.genIndex(state); 496 | 497 | return index; 498 | } 499 | }); 500 | 501 | const segments = []; 502 | for await (const obj of reader) { 503 | segments.push(obj); 504 | 505 | state.firstMsn += 5; 506 | 507 | await Hoek.wait(20); 508 | } 509 | 510 | expect(segments).to.have.length(20); 511 | expect(segments.map((s) => s.msn)).to.equal([ 512 | 6, 7, 10, 15, 20, 25, 30, 513 | 35, 40, 45, 50, 51, 52, 53, 514 | 54, 55, 56, 57, 58, 59 515 | ]); 516 | expect(segments.map((s) => s.entry.discontinuity)).to.equal([ 517 | false, false, true, true, true, true, true, 518 | true, true, true, true, false, false, false, 519 | false, false, false, false, false, false 520 | ]); 521 | }); 522 | 523 | it('respects the maxStallTime option', async () => { 524 | 525 | const { reader } = prepareLiveReader({ maxStallTime: 50 }, { segmentCount: 1 }); 526 | 527 | await expect((async () => { 528 | 529 | for await (const obj of reader) { 530 | 531 | expect(obj).to.exist(); 532 | } 533 | })()).to.reject(Error, /Index update stalled/); 534 | }); 535 | 536 | describe('destroy()', () => { 537 | 538 | it('works when called while waiting for a segment', async () => { 539 | 540 | const { reader, state } = prepareLiveReader({ fullStream: false }, { 541 | async index() { 542 | 543 | if (state.firstMsn > 0) { 544 | await Hoek.wait(100); 545 | } 546 | 547 | return Shared.genIndex(state); 548 | } 549 | }); 550 | 551 | setTimeout(() => reader.destroy(), 50); 552 | 553 | const segments = []; 554 | for await (const obj of reader) { 555 | segments.push(obj); 556 | 557 | state.firstMsn++; 558 | } 559 | 560 | expect(segments).to.have.length(4); 561 | }); 562 | 563 | it('emits passed error', async () => { 564 | 565 | const { reader, state } = prepareLiveReader({ fullStream: false }, { 566 | async index() { 567 | 568 | if (state.firstMsn > 0) { 569 | await Hoek.wait(10); 570 | } 571 | 572 | return Shared.genIndex(state); 573 | } 574 | }); 575 | 576 | setTimeout(() => reader.destroy(new Error('destroyed')), 50); 577 | 578 | await expect((async () => { 579 | 580 | for await (const {} of reader) { 581 | state.firstMsn++; 582 | } 583 | })()).to.reject('destroyed'); 584 | }); 585 | }); 586 | 587 | // TODO: move 588 | describe('isRecoverableUpdateError()', () => { 589 | 590 | it('is called on index update errors', async () => { 591 | 592 | const { reader, state } = prepareLiveReader({}, { 593 | index() { 594 | 595 | const { error } = state; 596 | if (error) { 597 | state.error++; 598 | switch (error) { 599 | case 1: 600 | case 2: 601 | case 3: 602 | throw Boom.notFound(); 603 | case 4: 604 | throw Boom.serverUnavailable(); 605 | case 5: 606 | throw Boom.unauthorized(); 607 | } 608 | } 609 | else if (state.firstMsn === 5) { 610 | state.error = 1; 611 | return ''; 612 | } 613 | 614 | const index = Shared.genIndex(state); 615 | 616 | ++state.firstMsn; 617 | 618 | return index; 619 | } 620 | }); 621 | 622 | const errors = []; 623 | reader.feeder.isRecoverableUpdateError = function (err) { 624 | 625 | errors.push(err); 626 | return HlsPlaylistReader.prototype.isRecoverableUpdateError.call(reader, err); 627 | }; 628 | 629 | const segments = []; 630 | const err = await expect((async () => { 631 | 632 | for await (const obj of reader) { 633 | segments.push(obj); 634 | } 635 | })()).to.reject('Unauthorized'); 636 | 637 | expect(segments.length).to.equal(14); 638 | expect(errors).to.have.length(4); 639 | expect(errors[0]).to.have.error(M3U8Parse.ParserError, 'Missing required #EXTM3U header'); 640 | expect(errors[1]).to.have.error(Boom.Boom, 'Not Found'); 641 | expect(errors[2]).to.have.error(Boom.Boom, 'Service Unavailable'); 642 | expect(errors[3]).to.shallow.equal(err); 643 | }); 644 | }); 645 | 646 | describe('with LL-HLS', () => { 647 | 648 | const prepareLlReader = function (readerOptions = {}, state = {}, indexGen) { 649 | 650 | return prepareLiveReader({ 651 | lowLatency: true, 652 | fullStream: false, 653 | ...readerOptions 654 | }, { 655 | partIndex: 0, 656 | partCount: 5, 657 | index: indexGen, 658 | ...state 659 | }); 660 | }; 661 | 662 | const { genLlIndex } = Shared; 663 | 664 | it('handles a basic stream', async () => { 665 | 666 | const { reader, state } = prepareLlReader({}, { partIndex: 4, end: { msn: 20, part: 3 } }, (query) => genLlIndex(query, state)); 667 | 668 | let updates = 0; 669 | const incrUpdates = () => updates++; 670 | 671 | const segments = []; 672 | const expected = { parts: state.partIndex, gens: 1 }; 673 | for await (const obj of reader) { 674 | switch (obj.msn) { 675 | case 10: 676 | expected.parts = 4; 677 | obj.onUpdate = incrUpdates; 678 | break; 679 | case 11: 680 | expected.parts = 1; 681 | expected.gens = 3; 682 | break; 683 | } 684 | 685 | expect(obj.msn).to.equal(segments.length + 10); 686 | expect(obj.entry.parts).to.have.length(expected.parts); 687 | expect(obj.entry.parts[0].has('byterange')).to.be.false(); 688 | expect(state.genCount).to.equal(expected.gens); 689 | expect(reader.hints.part).to.exist(); 690 | segments.push(obj); 691 | 692 | expected.gens += 5; 693 | } 694 | 695 | expect(segments.length).to.equal(11); 696 | expect(segments[0].entry.parts).to.have.length(5); 697 | expect(segments[10].entry.parts).to.have.length(3); 698 | expect(updates).to.equal(1); 699 | expect(reader.hints.part).to.not.exist(); 700 | }); 701 | 702 | it('finishes partial segments (without another read())', async () => { 703 | 704 | const { reader, state } = prepareLlReader({}, { partIndex: 4, end: { msn: 20, part: 3 } }, (query) => genLlIndex(query, state)); 705 | 706 | let updates = 0; 707 | const incrUpdates = () => updates++; 708 | 709 | const segments = []; 710 | const expected = { parts: state.partIndex, gens: 1 }; 711 | for await (const obj of reader) { 712 | switch (obj.msn) { 713 | case 10: 714 | expected.parts = 4; 715 | obj.onUpdate = incrUpdates; 716 | break; 717 | case 11: 718 | expected.parts = 1; 719 | expected.gens = 3; 720 | break; 721 | } 722 | 723 | expect(obj.msn).to.equal(segments.length + 10); 724 | expect(obj.entry.parts).to.have.length(expected.parts); 725 | expect(obj.entry.parts[0].has('byterange')).to.be.false(); 726 | expect(state.genCount).to.equal(expected.gens); 727 | expect(reader.hints.part).to.exist(); 728 | segments.push(obj); 729 | 730 | expected.gens += 5; 731 | 732 | await obj.closed(); 733 | } 734 | 735 | expect(segments.length).to.equal(11); 736 | expect(segments[0].entry.parts).to.have.length(5); 737 | expect(segments[10].entry.parts).to.have.length(3); 738 | expect(updates).to.equal(1); 739 | expect(reader.hints.part).to.not.exist(); 740 | }); 741 | 742 | it('ignores LL parts when lowLatency=false', async () => { 743 | 744 | const { reader, state } = prepareLlReader({ lowLatency: false }, { partIndex: 4, end: { msn: 20, part: 3 } }, (query) => genLlIndex(query, state)); 745 | 746 | const segments = []; 747 | let expectedGens = 1; 748 | for await (const obj of reader) { 749 | expect(obj.msn).to.equal(segments.length + 6); 750 | expect(obj.entry.isPartial()).to.be.false(); 751 | expect(state.genCount).to.equal(expectedGens); 752 | segments.push(obj); 753 | 754 | if (obj.msn > 8) { 755 | expectedGens++; 756 | } 757 | } 758 | 759 | expect(segments.length).to.equal(16); 760 | }); 761 | 762 | it('handles a basic stream with initial full segment', async () => { 763 | 764 | const { reader, state } = prepareLlReader({}, { partIndex: 2, end: { msn: 20, part: 3 } }, (query) => genLlIndex(query, state)); 765 | 766 | const segments = []; 767 | const expected = { parts: 5, gens: 1 }; 768 | for await (const obj of reader) { 769 | switch (obj.msn) { 770 | case 9: 771 | expected.parts = 5; 772 | break; 773 | case 10: 774 | expected.parts = 2; 775 | expected.gens = 1; 776 | break; 777 | case 11: 778 | expected.parts = 1; 779 | expected.gens = 5; 780 | break; 781 | } 782 | 783 | expect(obj.msn).to.equal(segments.length + 9); 784 | expect(obj.entry.parts).to.have.length(expected.parts); 785 | expect(state.genCount).to.equal(expected.gens); 786 | segments.push(obj); 787 | 788 | expected.gens += 5; 789 | } 790 | 791 | expect(segments.length).to.equal(12); 792 | expect(segments[0].entry.parts).to.have.length(5); 793 | expect(segments[11].isClosed).to.be.true(); 794 | expect(segments[11].entry.parts).to.have.length(3); 795 | }); 796 | 797 | it('handles a basic stream using byteranges', async () => { 798 | 799 | const { reader, state } = prepareLlReader({}, { partIndex: 4, end: { msn: 20, part: 3 } }, (query) => { 800 | 801 | const index = genLlIndex(query, state); 802 | const firstMsn = index.media_sequence; 803 | let segment; 804 | let offset; 805 | for (let msn = firstMsn; msn <= index.lastMsn(); ++msn) { // eslint-disable-line @hapi/for-loop 806 | segment = index.getSegment(msn); 807 | offset = 0; 808 | if (segment.parts) { 809 | for (let j = 0; j < segment.parts.length; ++j) { 810 | const part = segment.parts[j]; 811 | part.set('uri', `${msn}.ts`, 'string'); 812 | part.set('byterange', { length: 800 + j, offset: j === 0 ? 0 : undefined }, 'byterange'); 813 | offset += 800 + j; 814 | } 815 | } 816 | } 817 | 818 | if (index.meta.preload_hints) { 819 | const hint = index.meta.preload_hints[0]; 820 | hint.set('uri', `${index.lastMsn() + +!segment.isPartial()}.ts`, 'string'); 821 | hint.set('byterange-start', segment.isPartial() ? offset : 0, 'int'); 822 | } 823 | 824 | return index; 825 | }); 826 | 827 | const segments = []; 828 | let expectedParts = 4; 829 | for await (const obj of reader) { 830 | expect(obj.msn).to.equal(segments.length + 10); 831 | expect(obj.entry.parts).to.have.length(expectedParts); 832 | expect(obj.entry.parts[0].get('byterange')).to.include('@'); 833 | segments.push(obj); 834 | 835 | expectedParts = 1; 836 | } 837 | 838 | expect(segments).to.have.length(11); 839 | expect(segments[0].entry.parts).to.have.length(5); 840 | expect(segments[10].entry.parts).to.have.length(3); 841 | }); 842 | 843 | it('handles active parts being evicted from index', async () => { 844 | 845 | const { reader, state } = prepareLlReader({}, { partIndex: 4, end: { msn: 20, part: 3 } }, (query) => { 846 | 847 | // Jump during active part 848 | 849 | if (query._HLS_msn === 13 && query._HLS_part === 2) { 850 | state.firstMsn += 5; 851 | state.partIndex = 4; 852 | query = {}; 853 | } 854 | 855 | return genLlIndex(query, state); 856 | }); 857 | 858 | const segments = []; 859 | const expected = { parts: 5, gens: 1, incr: 5 }; 860 | for await (const obj of reader) { 861 | switch (obj.msn) { 862 | case 10: 863 | expected.parts = 4; 864 | break; 865 | case 11: 866 | expected.parts = 1; 867 | expected.gens = 3; 868 | break; 869 | case 14: 870 | expected.parts = undefined; 871 | expected.gens = 15; 872 | expected.incr = 0; 873 | break; 874 | case 16: 875 | expected.parts = 5; 876 | break; 877 | case 18: 878 | expected.parts = 4; 879 | break; 880 | case 19: 881 | expected.parts = 1; 882 | expected.gens = 17; 883 | expected.incr = 5; 884 | break; 885 | } 886 | 887 | expect(obj.msn).to.equal(segments.length + 10); 888 | if (expected.parts === undefined) { 889 | expect(obj.entry.parts).to.not.exist(); 890 | } 891 | else { 892 | expect(obj.entry.parts).to.have.length(expected.parts); 893 | } 894 | 895 | expect(state.genCount).to.equal(expected.gens); 896 | segments.push(obj); 897 | 898 | expected.gens += expected.incr; 899 | } 900 | 901 | expect(segments.length).to.equal(11); 902 | expect(segments[2].entry.parts).to.have.length(5); 903 | expect(segments[3].entry.parts).to.have.length(2); 904 | expect(segments[4].entry.parts).to.not.exist(); 905 | expect(segments[5].entry.parts).to.not.exist(); 906 | expect(segments[6].entry.parts).to.have.length(5); 907 | }); 908 | 909 | // TODO: mp4 with initial map 910 | // TODO: segment jumps 911 | // TODO: out of index stall 912 | }); 913 | 914 | // handles fullStream option 915 | // emits index updates 916 | // TODO: resilience?? 917 | }); 918 | }); 919 | -------------------------------------------------------------------------------- /test/streamer.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Crypto = require('crypto'); 4 | const Path = require('path'); 5 | 6 | const Code = require('@hapi/code'); 7 | const Hoek = require('@hapi/hoek'); 8 | const Lab = require('@hapi/lab'); 9 | const { MediaSegment, AttrList } = require('m3u8parse'); 10 | const Uristream = require('uristream'); 11 | 12 | const Shared = require('./_shared'); 13 | 14 | // eslint-disable-next-line @hapi/capitalize-modules 15 | const { createSimpleReader, HlsSegmentReader, HlsReaderObject, HlsSegmentStreamer, HlsPlaylistReader } = require('..'); 16 | 17 | 18 | // Declare internals 19 | 20 | const internals = { 21 | checksums: [ 22 | 'a6b0e0ce44f29e965e751113b39fdf4a47787cab', 23 | 'c38d0718851a20be2edba13fc1643c1076826c62', 24 | '612991f34ae7cc19df5d595a2a4249b8f5d2d3f0', 25 | 'bc600f4039aae412c4d978b3fd4d608ce4dec59a' 26 | ] 27 | }; 28 | 29 | 30 | internals.nextValue = async function (iter, expectDone = false) { 31 | 32 | const { value, done } = await iter.next(); 33 | 34 | expect(done).to.equal(expectDone); 35 | 36 | return value; 37 | }; 38 | 39 | 40 | // Test shortcuts 41 | 42 | const lab = exports.lab = Lab.script(); 43 | const { after, before, describe, it } = lab; 44 | const { expect } = Code; 45 | 46 | 47 | describe('HlsSegmentStreamer()', () => { 48 | 49 | const readSegments = Shared.readSegments.bind(null, HlsSegmentStreamer); 50 | let server; 51 | 52 | before(async () => { 53 | 54 | server = await Shared.provisionServer(); 55 | return server.start(); 56 | }); 57 | 58 | after(() => { 59 | 60 | return server.stop(); 61 | }); 62 | 63 | describe('constructor', () => { 64 | 65 | it('creates a valid object', () => { 66 | 67 | const r = new HlsSegmentStreamer(); 68 | 69 | expect(r).to.be.instanceOf(HlsSegmentStreamer); 70 | 71 | r.destroy(); 72 | }); 73 | 74 | it('handles a reader', () => { 75 | 76 | const r = new HlsSegmentStreamer(new HlsSegmentReader('http://localhost:' + server.info.port + '/simple/500.m3u8')); 77 | 78 | expect(r).to.be.instanceOf(HlsSegmentStreamer); 79 | 80 | r.destroy(); 81 | }); 82 | }); 83 | 84 | it('emits error for missing data', async () => { 85 | 86 | const promise = readSegments(new HlsSegmentReader(`http://localhost:${server.info.port}/notfound`)); 87 | await expect(promise).to.reject(Error, /Not Found/); 88 | }); 89 | 90 | // TODO: move?? 91 | it('Uristream maps file extensions to suitable mime types', async () => { 92 | 93 | const map = new Map([ 94 | ['audio.aac', 'audio/x-aac'], 95 | ['audio.ac3', 'audio/ac3'], 96 | ['audio.dts', 'audio/vnd.dts'], 97 | ['audio.eac3', 'audio/eac3'], 98 | ['audio.m4a', 'audio/mp4'], 99 | ['file.m4s', 'video/iso.segment'], 100 | ['file.mp4', 'video/mp4'], 101 | ['text.vtt', 'text/vtt'], 102 | ['video.m4v', 'video/x-m4v'] 103 | ]); 104 | 105 | for (const [file, mime] of map) { 106 | const meta = await new Promise((resolve, reject) => { 107 | 108 | const uristream = new Uristream(`file://${Path.resolve(__dirname, 'fixtures/files', file)}`); 109 | uristream.on('error', reject); 110 | uristream.on('meta', resolve); 111 | }); 112 | 113 | expect(meta.mime).to.equal(mime); 114 | } 115 | }); 116 | 117 | it('emits error on unknown segment mime type', async () => { 118 | 119 | await expect((async () => { 120 | 121 | const r = createSimpleReader('file://' + Path.join(__dirname, 'fixtures', 'badtype.m3u8'), { withData: false }); 122 | 123 | for await (const obj of r) { 124 | 125 | expect(obj).to.exist(); 126 | } 127 | })()).to.reject(Error, /Unsupported segment MIME type/); 128 | 129 | await expect((async () => { 130 | 131 | const r = createSimpleReader('file://' + Path.join(__dirname, 'fixtures', 'badtype-data.m3u8'), { withData: true }); 132 | 133 | for await (const obj of r) { 134 | 135 | expect(obj).to.exist(); 136 | } 137 | })()).to.reject(Error, /Unsupported segment MIME type/); 138 | }); 139 | 140 | describe('writing HlsReaderObjects', () => { 141 | 142 | it('works', async () => { 143 | 144 | const streamer = new HlsSegmentStreamer({ highWaterMark: 0 }); 145 | const iter = streamer[Symbol.asyncIterator](); 146 | 147 | streamer.write(new HlsReaderObject(0, new MediaSegment({ 148 | uri: 'data:video/mp2t,TS', 149 | duration: 2 150 | }))); 151 | 152 | const obj = await internals.nextValue(iter); 153 | expect(obj.type).to.equal('segment'); 154 | expect(obj.segment.msn).to.equal(0); 155 | 156 | streamer.end(); 157 | await internals.nextValue(iter, true); 158 | }); 159 | 160 | it('returns map objects', async () => { 161 | 162 | const streamer = new HlsSegmentStreamer({}); 163 | 164 | const segment = new MediaSegment({ 165 | uri: 'data:video/mp2t,DATA', 166 | duration: 2, 167 | map: new AttrList({ uri: '"data:video/mp2t,MAP"', value: 'OK' }) 168 | }); 169 | 170 | streamer.write(new HlsReaderObject(0, segment)); 171 | streamer.write(new HlsReaderObject(1, segment)); 172 | streamer.end(); 173 | 174 | const segments = []; 175 | for await (const obj of streamer) { 176 | segments.push(obj); 177 | } 178 | 179 | expect(segments).to.have.length(3); 180 | expect(segments[0].type).to.equal('map'); 181 | expect(segments[0].attrs).to.equal(segment.map); 182 | expect(segments[1].type).to.equal('segment'); 183 | expect(segments[1].segment.msn).to.equal(0); 184 | expect(segments[2].type).to.equal('segment'); 185 | expect(segments[2].segment.msn).to.equal(1); 186 | }); 187 | 188 | it('returns updated map objects', async () => { 189 | 190 | const streamer = new HlsSegmentStreamer({}); 191 | 192 | const segment = new MediaSegment({ 193 | uri: 'data:video/mp2t,DATA', 194 | duration: 2 195 | }); 196 | 197 | streamer.write(new HlsReaderObject(0, segment)); 198 | streamer.write(new HlsReaderObject(1, new MediaSegment({ ...segment, map: new AttrList({ uri: '"data:video/mp2t,MAP1"' }) }))); 199 | streamer.write(new HlsReaderObject(2, new MediaSegment({ ...segment, map: new AttrList({ uri: '"data:video/mp2t,MAP2"' }) }))); 200 | streamer.write(new HlsReaderObject(3, new MediaSegment({ ...segment, map: new AttrList({ uri: '"data:video/mp2t,MAP3"', byterange: '2@0' }) }))); 201 | streamer.write(new HlsReaderObject(4, new MediaSegment({ ...segment, map: new AttrList({ uri: '"data:video/mp2t,MAP3"', byterange: '3@1' }) }))); 202 | streamer.write(new HlsReaderObject(5, segment)); 203 | streamer.end(); 204 | 205 | const segments = []; 206 | for await (const obj of streamer) { 207 | segments.push(obj); 208 | } 209 | 210 | expect(segments).to.have.length(10); 211 | 212 | expect(segments[0].type).to.equal('segment'); 213 | expect(segments[0].segment.msn).to.equal(0); 214 | expect(segments[1].type).to.equal('map'); 215 | expect(segments[2].segment.msn).to.equal(1); 216 | expect(segments[3].type).to.equal('map'); 217 | expect(segments[4].segment.msn).to.equal(2); 218 | expect(segments[5].type).to.equal('map'); 219 | expect(segments[6].segment.msn).to.equal(3); 220 | expect(segments[7].type).to.equal('map'); 221 | expect(segments[8].segment.msn).to.equal(4); 222 | expect(segments[9].segment.msn).to.equal(5); 223 | }); 224 | 225 | it('handles partial segments, where part completes', async () => { 226 | 227 | const streamer = new HlsSegmentStreamer(); 228 | const iter = streamer[Symbol.asyncIterator](); 229 | 230 | const segment = new HlsReaderObject(0, new MediaSegment({ 231 | parts: [new AttrList(), new AttrList()] 232 | })); 233 | 234 | const waitingForClosed = new Promise((resolve) => { 235 | 236 | segment.closed = function () { 237 | 238 | process.nextTick(resolve, 'closed'); 239 | return HlsReaderObject.prototype.closed.call(this); 240 | }; 241 | }); 242 | 243 | streamer.write(segment); 244 | 245 | const promise = internals.nextValue(iter); 246 | expect(await Promise.race([waitingForClosed, promise])).to.equal('closed'); 247 | 248 | segment.entry = new MediaSegment({ 249 | ...segment.entry, 250 | uri: 'data:video/mp2t,TS', 251 | duration: 2 252 | }); 253 | 254 | const obj = await promise; 255 | expect(obj.segment).to.equal(segment); 256 | 257 | streamer.end(); 258 | await internals.nextValue(iter, true); 259 | }); 260 | 261 | it('handles partial segments, where part is dropped', async () => { 262 | 263 | const streamer = new HlsSegmentStreamer(); 264 | const iter = streamer[Symbol.asyncIterator](); 265 | 266 | const segment = new HlsReaderObject(0, new MediaSegment({ 267 | parts: [new AttrList()] 268 | })); 269 | 270 | const waitingForClosed = new Promise((resolve) => { 271 | 272 | segment.closed = function () { 273 | 274 | process.nextTick(resolve, 'closed'); 275 | return HlsReaderObject.prototype.closed.call(this); 276 | }; 277 | }); 278 | 279 | streamer.write(segment); 280 | 281 | const promise = internals.nextValue(iter); 282 | expect(await Promise.race([waitingForClosed, promise])).to.equal('closed'); 283 | 284 | segment.entry = new MediaSegment({ 285 | uri: 'data:video/mp2t,TS', 286 | duration: 2 287 | }); 288 | 289 | const obj = await promise; 290 | expect(obj.segment).to.equal(segment); 291 | 292 | streamer.end(); 293 | await internals.nextValue(iter, true); 294 | }); 295 | 296 | it('drops partial segments that are abandoned', async () => { 297 | 298 | const streamer = new HlsSegmentStreamer(); 299 | const iter = streamer[Symbol.asyncIterator](); 300 | 301 | const segment = new HlsReaderObject(0, new MediaSegment({ 302 | parts: [new AttrList()] 303 | })); 304 | 305 | const waitingForClosed = new Promise((resolve) => { 306 | 307 | segment.closed = function () { 308 | 309 | process.nextTick(resolve, 'closed'); 310 | return HlsReaderObject.prototype.closed.call(this); 311 | }; 312 | }); 313 | 314 | streamer.write(segment); 315 | streamer.write(new HlsReaderObject(1, new MediaSegment({ 316 | uri: 'data:video/mp2t,TS', 317 | duration: 2 318 | }))); 319 | 320 | const promise = internals.nextValue(iter); 321 | expect(await Promise.race([waitingForClosed, promise])).to.equal('closed'); 322 | 323 | segment.abandon(); 324 | 325 | const obj = await promise; 326 | expect(obj.type).to.equal('segment'); 327 | expect(obj.segment.msn).to.equal(1); 328 | 329 | streamer.end(); 330 | await internals.nextValue(iter, true); 331 | }); 332 | }); 333 | 334 | describe('master index', () => { 335 | 336 | it('does not output any segments', async () => { 337 | 338 | const reader = new HlsSegmentReader(`http://localhost:${server.info.port}/simple/index.m3u8`); 339 | await expect(readSegments(reader)).to.reject('The reader source is a master playlist'); 340 | expect(reader.index.master).to.be.true(); 341 | }); 342 | }); 343 | 344 | describe('on-demand index', () => { 345 | 346 | it('outputs all segments', async () => { 347 | 348 | const segments = await readSegments(new HlsSegmentReader(`http://localhost:${server.info.port}/simple/500.m3u8`)); 349 | 350 | expect(segments).to.have.length(3); 351 | for (let i = 0; i < segments.length; ++i) { 352 | expect(segments[i].segment.msn).to.equal(i); 353 | } 354 | }); 355 | 356 | it('handles byte-range (file)', async () => { 357 | 358 | const r = createSimpleReader('file://' + Path.join(__dirname, 'fixtures', 'single.m3u8'), { withData: true }); 359 | const checksums = []; 360 | 361 | for await (const obj of r) { 362 | const hasher = Crypto.createHash('sha1'); 363 | hasher.setEncoding('hex'); 364 | 365 | obj.stream.pipe(hasher); 366 | 367 | const hash = await new Promise((resolve, reject) => { 368 | 369 | obj.stream.on('error', reject); 370 | obj.stream.on('end', () => resolve(hasher.read())); 371 | }); 372 | 373 | checksums.push(hash); 374 | } 375 | 376 | expect(checksums).to.equal(internals.checksums); 377 | }); 378 | 379 | it('handles byte-range (http)', async () => { 380 | 381 | const r = createSimpleReader(`http://localhost:${server.info.port}/simple/single.m3u8`, { withData: true }); 382 | const checksums = []; 383 | 384 | for await (const obj of r) { 385 | const hasher = Crypto.createHash('sha1'); 386 | hasher.setEncoding('hex'); 387 | 388 | obj.stream.pipe(hasher); 389 | 390 | const hash = await new Promise((resolve, reject) => { 391 | 392 | obj.stream.on('error', reject); 393 | obj.stream.on('end', () => resolve(hasher.read())); 394 | }); 395 | 396 | checksums.push(hash); 397 | } 398 | 399 | expect(checksums).to.equal(internals.checksums); 400 | }); 401 | 402 | it('does not internally buffer (highWaterMark=0)', async () => { 403 | 404 | const reader = new HlsSegmentReader('file://' + Path.join(__dirname, 'fixtures', 'long.m3u8')); 405 | const streamer = new HlsSegmentStreamer(reader, { withData: false, highWaterMark: 0 }); 406 | 407 | for await (const obj of streamer) { 408 | expect(obj).to.exist(); 409 | await Hoek.wait(20); 410 | expect(streamer.readableLength).to.equal(0); 411 | } 412 | }); 413 | 414 | it('supports the highWaterMark option', async () => { 415 | 416 | const r = createSimpleReader('file://' + Path.join(__dirname, 'fixtures', 'long.m3u8'), { highWaterMark: 3 }); 417 | const buffered = []; 418 | 419 | for await (const obj of r) { 420 | expect(obj).to.exist(); 421 | await Hoek.wait(100); 422 | buffered.push(r.readableLength); 423 | } 424 | 425 | expect(buffered).to.equal([3, 3, 3, 2, 1, 0]); 426 | }); 427 | 428 | it('abort() also aborts active streams when withData is set', async () => { 429 | 430 | const r = createSimpleReader(`http://localhost:${server.info.port}/simple/slow.m3u8`, { withData: true, highWaterMark: 2 }); 431 | const segments = []; 432 | 433 | setTimeout(() => r.abort(), 50); 434 | 435 | for await (const obj of r) { 436 | expect(obj.segment.msn).to.equal(segments.length); 437 | segments.push(obj); 438 | } 439 | 440 | expect(segments.length).to.equal(2); 441 | }); 442 | 443 | it('abort() graceful is respected', async () => { 444 | 445 | const r = createSimpleReader(`http://localhost:${server.info.port}/simple/slow.m3u8`, { withData: true, stopDate: new Date('Fri Jan 07 2000 07:03:09 GMT+0100 (CET)') }); 446 | const checksums = []; 447 | 448 | for await (const obj of r) { 449 | const hasher = Crypto.createHash('sha1'); 450 | hasher.setEncoding('hex'); 451 | 452 | obj.stream.pipe(hasher); 453 | const hash = await new Promise((resolve, reject) => { 454 | 455 | obj.stream.on('error', reject); 456 | obj.stream.on('end', () => resolve(hasher.read())); 457 | }); 458 | 459 | checksums.push(hash); 460 | 461 | if (obj.segment.msn === 1) { 462 | r.abort(true); 463 | } 464 | } 465 | 466 | expect(checksums).to.equal(internals.checksums.slice(1, 3)); 467 | }); 468 | 469 | it('can be destroyed', async () => { 470 | 471 | const r = createSimpleReader('file://' + Path.join(__dirname, 'fixtures', '500.m3u8')); 472 | const segments = []; 473 | 474 | for await (const obj of r) { 475 | segments.push(obj); 476 | r.destroy(); 477 | } 478 | 479 | expect(segments.length).to.equal(1); 480 | }); 481 | 482 | // handles all kinds of segment reference url 483 | // handles .m3u files 484 | }); 485 | 486 | describe('live index', { parallel: false }, () => { 487 | 488 | const serverState = { state: {} }; 489 | let liveServer; 490 | 491 | const prepareLiveReader = function (readerOptions = {}, state = {}) { 492 | 493 | const reader = new HlsSegmentReader(`http://localhost:${liveServer.info.port}/live/live.m3u8`, readerOptions); 494 | const streamer = new HlsSegmentStreamer(reader, { fullStream: false, withData: true, ...readerOptions }); 495 | 496 | reader.feeder._intervals = []; 497 | reader.feeder.getUpdateInterval = function (...args) { 498 | 499 | this._intervals.push(HlsPlaylistReader.prototype.getUpdateInterval.call(this, ...args)); 500 | return undefined; 501 | }; 502 | 503 | serverState.state = { firstMsn: 0, segmentCount: 10, targetDuration: 2, ...state }; 504 | 505 | return { reader: streamer, state: serverState.state }; 506 | }; 507 | 508 | before(() => { 509 | 510 | liveServer = Shared.provisionLiveServer(serverState); 511 | return liveServer.start(); 512 | }); 513 | 514 | after(() => { 515 | 516 | return liveServer.stop(); 517 | }); 518 | 519 | it('handles a basic stream', async () => { 520 | 521 | const { reader, state } = prepareLiveReader({ fullStream: true }); 522 | const segments = []; 523 | 524 | for await (const obj of reader) { 525 | expect(obj.segment.msn).to.equal(segments.length); 526 | segments.push(obj); 527 | 528 | if (obj.segment.msn > 5) { 529 | state.firstMsn++; 530 | if (state.firstMsn >= 5) { 531 | state.firstMsn = 5; 532 | state.ended = true; 533 | } 534 | } 535 | } 536 | 537 | expect(segments.length).to.equal(15); 538 | }); 539 | 540 | it('handles sequence number resets', async () => { 541 | 542 | let reset = false; 543 | const { reader, state } = prepareLiveReader({ fullStream: false }, { firstMsn: 10, async index() { 544 | 545 | const index = Shared.genIndex(state); 546 | 547 | if (!reset) { 548 | state.firstMsn++; 549 | 550 | if (state.firstMsn === 16) { 551 | state.firstMsn = 0; 552 | state.segmentCount = 1; 553 | reset = true; 554 | } 555 | 556 | await Hoek.wait(20); // give the reader a chance to catch up 557 | } 558 | else { 559 | state.segmentCount++; 560 | if (state.segmentCount === 5) { 561 | state.ended = true; 562 | } 563 | } 564 | 565 | return index; 566 | } }); 567 | 568 | const segments = []; 569 | for await (const obj of reader) { 570 | segments.push(obj); 571 | } 572 | 573 | expect(segments.length).to.equal(14); 574 | expect(segments[0].segment.msn).to.equal(16); 575 | expect(segments[9].segment.msn).to.equal(0); 576 | expect(segments[8].segment.entry.discontinuity).to.be.false(); 577 | expect(segments[9].segment.entry.discontinuity).to.be.true(); 578 | expect(segments[10].segment.entry.discontinuity).to.be.false(); 579 | }); 580 | 581 | it('handles sequence number jumps', async () => { 582 | 583 | let skipped = false; 584 | const { reader, state } = prepareLiveReader({}, { 585 | index() { 586 | 587 | const index = Shared.genIndex(state); 588 | 589 | if (!skipped) { 590 | ++state.firstMsn; 591 | if (state.firstMsn === 5) { 592 | state.firstMsn = 50; 593 | skipped = true; 594 | } 595 | } 596 | else if (skipped) { 597 | ++state.firstMsn; 598 | if (state.firstMsn === 55) { 599 | state.ended = true; 600 | } 601 | } 602 | 603 | return index; 604 | } 605 | }); 606 | 607 | const segments = []; 608 | for await (const obj of reader) { 609 | segments.push(obj); 610 | } 611 | 612 | expect(segments.length).to.equal(23); 613 | expect(segments[7].segment.msn).to.equal(13); 614 | expect(segments[7].segment.entry.discontinuity).to.be.false(); 615 | expect(segments[8].segment.msn).to.equal(50); 616 | expect(segments[8].segment.entry.discontinuity).to.be.true(); 617 | expect(segments[9].segment.entry.discontinuity).to.be.false(); 618 | }); 619 | 620 | // TODO: test problem emit & data outage 621 | /*it('handles a temporary server outage', async () => { 622 | 623 | const { reader, state } = prepareLiveReader({}, { 624 | index() { 625 | 626 | if (state.error === undefined && state.firstMsn === 5) { 627 | state.error = 6; 628 | } 629 | 630 | if (state.error) { 631 | --state.error; 632 | ++state.firstMsn; 633 | throw new Error('fail'); 634 | } 635 | 636 | if (state.firstMsn === 20) { 637 | state.ended = true; 638 | } 639 | 640 | const index = Shared.genIndex(state); 641 | 642 | ++state.firstMsn; 643 | 644 | return index; 645 | } 646 | }); 647 | 648 | const errors = []; 649 | reader.on('problem', errors.push.bind(errors)); 650 | 651 | const segments = []; 652 | for await (const obj of reader) { 653 | expect(obj.msn).to.equal(segments.length); 654 | segments.push(obj); 655 | } 656 | 657 | expect(segments.length).to.equal(30); 658 | expect(errors.length).to.be.greaterThan(0); 659 | expect(errors[0]).to.be.an.error('Internal Server Error'); 660 | });*/ 661 | 662 | it('aborts downloads that have been evicted from index', async () => { 663 | 664 | // Note: the eviction logic works on index updates, with a delay to allow an initial segment load some time to complete - otherwise it could be scheduled, have an immediate update, and be aborted before being given a chance 665 | 666 | const { reader, state } = prepareLiveReader({ fullStream: true }, { segmentCount: 3, slow: true }); 667 | const segments = []; 668 | 669 | for await (const obj of reader) { 670 | segments.push(obj); 671 | 672 | state.firstMsn++; 673 | if (state.firstMsn >= 3) { 674 | state.firstMsn = 3; 675 | state.ended = true; 676 | } 677 | } 678 | 679 | expect(segments.length).to.equal(6); 680 | expect(segments[0].stream.destroyed).to.be.true(); // Test 681 | 682 | reader.abort(); 683 | }); 684 | 685 | it('completes unstable downloads', async () => { 686 | 687 | const { reader, state } = prepareLiveReader({}, { unstable: true }); 688 | const segments = []; 689 | 690 | for await (const obj of reader) { 691 | expect(obj.segment.msn).to.equal(segments.length + 6); 692 | segments.push(obj); 693 | 694 | let bytes = 0; 695 | for await (const chunk of obj.stream) { 696 | bytes += chunk.length; 697 | } 698 | 699 | expect(bytes).to.equal(5000 + obj.segment.msn); 700 | 701 | if (obj.segment.msn > 5) { 702 | state.firstMsn++; 703 | if (state.firstMsn >= 5) { 704 | state.firstMsn = 5; 705 | state.ended = true; 706 | } 707 | } 708 | } 709 | 710 | expect(segments.length).to.equal(9); 711 | }); 712 | }); 713 | }); 714 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "lib": [ "ES2020" ], 5 | "module": "commonjs", 6 | "isolatedModules": true, 7 | "declaration": true, 8 | "strict": true, 9 | "strictNullChecks": true, 10 | "esModuleInterop": false, 11 | "types": ["node"] 12 | }, 13 | "include": [ 14 | "lib" 15 | ] 16 | } 17 | --------------------------------------------------------------------------------