├── .dockerignore ├── .gitignore ├── .jshintrc ├── .nvmrc ├── LICENSE ├── LiveController_deployment.md ├── README.md ├── build_scripts ├── build_addon.sh ├── build_binaries.sh ├── build_ffmpeg.sh ├── build_ffmpeg4.sh └── build_ts2mp4_convertor.sh ├── common ├── Configuration.js ├── PersistenceFormat.js ├── config │ ├── config.json.template │ └── configMapping.json.template ├── logger.js └── utils │ └── hostname.js ├── deployment ├── addServerNode.js ├── docker.md ├── docker │ ├── .env.template │ ├── build_images.sh │ ├── docker-compose-full.yml │ ├── docker-compose.yml │ ├── initScript.sh │ ├── liveController │ │ ├── Dockerfile │ │ └── entryPoint.sh │ ├── liveJobs │ │ └── Dockerfile │ ├── livePackager │ │ ├── Dockerfile │ │ ├── deploy.sh │ │ └── entryPoint.sh │ └── liveRecorder │ │ ├── Dockerfile │ │ └── entryPoint.sh ├── get_bins ├── kubernetes │ ├── config.template.yaml │ ├── live-front.yaml │ ├── live-publish.yaml │ └── readme.md ├── runtimeScripts │ ├── cleaner.sh │ ├── liveCleaner.sh │ └── recordingCleaner.sh └── upgradeLive ├── development.md ├── grunt-config.js ├── gruntfile.js ├── lib ├── Adapters │ ├── APIQueryAdapter.js │ ├── AdapterFactory.js │ ├── BaseAdapter.js │ ├── CompositeWowzaAdapter.js │ ├── EntryCache.js │ ├── RegressionAdapter │ │ ├── RegressionAdapter.js │ │ ├── RegressionConfig.js │ │ ├── RegressionEngine.js │ │ ├── RegressionTestStaticDiagram.puml │ │ ├── RegressionValidatorBase.js │ │ ├── RegressionValidatorFactory.js │ │ ├── hlsAnalysisRegressionValidaotor.js │ │ ├── hlsChecksumRegressionValidator.js │ │ ├── regression_utility.js │ │ ├── scripts │ │ │ └── validate_hls_data_warehouse.sh │ │ └── validatorUtility.js │ ├── TestAdapter.js │ ├── WowzaAdapter.js │ └── WowzaStreamInfo.js ├── App.js ├── BackendClient.js ├── BackendClientFactory.js ├── Controller.js ├── Diagnostics │ ├── Diagnostics.js │ ├── DiagnosticsAlerts.js │ ├── DiagnosticsAnalyzer.js │ └── InvalidClipError.js ├── MP4WriteStream.js ├── MonitorServer.js ├── NetworkClient.js ├── NetworkClientFactory.js ├── PushManager.js ├── SessionManager.js ├── TaskManager.js ├── entry │ ├── FlavorDownloader.js │ ├── LiveEntry.js │ ├── StateManager.js │ ├── StateManagerUMLState.puml │ └── crossDCsStreamPersistence.js ├── grunt │ ├── config │ │ ├── component-tests-config.js │ │ ├── coverage-config.js │ │ ├── jshint-config.js │ │ ├── recording-config.js │ │ ├── regression-tests-config.js │ │ └── unit-tests-config.js │ └── tasks │ │ ├── component-test-task.js │ │ ├── recording-task.js │ │ ├── regression-tests-task.js │ │ └── unit-tests-task.js ├── kaltura-client-lib │ ├── KalturaClient.js │ ├── KalturaClientBase.js │ ├── KalturaServices.js │ ├── KalturaTypes.js │ └── KalturaVO.js ├── liveDNSupdate.js ├── m3u8 │ ├── LICENSE.txt │ ├── Makefile │ ├── README.md │ ├── m3u.js │ ├── m3u │ │ ├── AttributeList.js │ │ ├── IframeStreamItem.js │ │ ├── Item.js │ │ ├── MediaItem.js │ │ ├── PlaylistItem.js │ │ └── StreamItem.js │ ├── package.json │ ├── parser.js │ └── test │ │ ├── acceptance │ │ ├── parse-iframe.js │ │ ├── parse-playlist.js │ │ └── parse-variant.js │ │ ├── attributelist.test.js │ │ ├── fixtures │ │ ├── iframe.m3u8 │ │ ├── playlist.m3u8 │ │ └── variant.m3u8 │ │ ├── item.test.js │ │ ├── m3u.test.js │ │ └── parser.test.js ├── manifest │ └── promise-m3u8.js ├── mocks │ ├── BackendClientFileBasedMock.js │ ├── BackendClientMock.js │ ├── NetworkClientMock.js │ └── mockBackendResults.json ├── playlistGenerator │ ├── BroadcastEventEmitter.js │ ├── ConcatSource.js │ ├── GapPatcher.js │ ├── MixFilterClip.js │ ├── Playlist.js │ ├── PlaylistGenerator.js │ ├── PlaylistItem.js │ ├── SearchIndex.js │ ├── Sequence.js │ ├── TimestampList.js │ ├── ValueHolder.js │ └── playlistGen-utils.js ├── recording │ ├── RecordingEntrySession.js │ ├── RecordingManager.js │ ├── RestoreRecordingFromBackup.js │ └── RestoreRecordingScript.js └── utils │ ├── error-utils.js │ ├── fs-utils.js │ ├── fsm.js │ ├── http-utils.js │ ├── math-utils.js │ ├── mp4-utils.js │ └── promise-utils.js ├── liveRecorder ├── BackendClient.py ├── Config │ ├── __init__.py │ ├── config.ini │ ├── config.py │ └── configMapping.ini.template ├── DailyReport.py ├── KalturaClient │ ├── Base.py │ ├── Client.py │ ├── Plugins │ │ ├── Core.py │ │ └── __init__.py │ ├── __init__.py │ └── poster │ │ ├── __init__.py │ │ ├── encode.py │ │ └── streaminghttp.py ├── Logger │ ├── Logger.py │ ├── LoggerDecorator.py │ └── __init__.py ├── README.md ├── RecordingException.py ├── RecoverRecording.py ├── Tasks │ ├── ConcatinationTask.py │ ├── Iso639Wrapper.py │ ├── KalturaUploadSession.py │ ├── MockFileObject.py │ ├── TaskBase.py │ ├── TaskRunner.py │ ├── ThreadWorkers.py │ ├── UploadChunkJob.py │ ├── UploadTask.py │ ├── __init__.py │ └── database │ │ └── iso639-3.json ├── ZombieEntryCatchers.py ├── install.sh ├── installPython.sh ├── liveRecorder_deployment.md ├── main.py ├── scripts │ ├── filter_log_errors.sh │ └── token_generator ├── serviceWrappers │ ├── defaults │ │ ├── liveRecorder │ │ └── liveRecorder.template │ └── linux │ │ ├── getRecordingBaseDir.py │ │ └── liveRecorder └── ts_to_mp4_convertor │ ├── Makefile │ ├── audio_filler.h │ ├── audio_fillter.c │ ├── downloader.py │ ├── main.c │ ├── test.py │ ├── tests.h │ └── ts_to_mp4_convertor.xcodeproj │ ├── project.pbxproj │ └── project.xcworkspace │ └── contents.xcworkspacedata ├── node_addons └── FormatConverter │ ├── binding.gyp │ ├── include │ ├── AVFormat.h │ ├── Converter.h │ ├── FileStream.h │ ├── MemoryStream.h │ ├── NodeFormatConverter.h │ ├── NodeStream.h │ ├── Stream.h │ └── Utils.h │ ├── readme.md │ ├── src │ ├── AVFormat.cpp │ ├── Converter.cpp │ ├── NodeFormatConverter.cpp │ ├── NodeStream.cpp │ ├── Stream.cpp │ └── main.cpp │ └── test │ ├── FormatConvertTest.js │ ├── mp4writerTest.js │ └── playlistTest.js ├── package-lock.json ├── package.json ├── packager ├── bin │ ├── anal_nginx_log_packager.awk │ ├── build_nginx.sh │ ├── monitorKeyFrameAlignment.sh │ ├── monitorKeyFrames.awk │ ├── monitor_packager.sh │ ├── packager_check_output_ranges.sh │ ├── replayLiveSession.sh │ ├── run_nginx.sh │ └── test.sh ├── config │ ├── cors.common.conf │ ├── nginx.conf.live.bootstrap.template │ ├── nginx.conf.live.conf.template │ ├── nginx.conf.live.protocols.template │ └── nginx.conf.template └── www │ ├── clientaccesspolicy.xml │ └── crossdomain.xml ├── serviceWrappers ├── defaults │ ├── kLiveController │ └── kLiveController.template └── linux │ └── kLiveController └── tests ├── component └── workerComponent-spec.js ├── mocks └── wowzaMock.js ├── recording ├── recording-spec.js └── test.py ├── regression ├── ControllerWrapper.js └── regression-spec.js ├── resources ├── append │ └── 7 │ │ └── 0_dv6q5l87 │ │ └── 0_uy8h6jmv │ │ └── playlist.json.template ├── crash.ts ├── crash2.ts ├── decreasing_pts.ts ├── downloadChunks ├── erronousManifest.m3u8 ├── flavor-downloader-data │ └── simpleManifest.m3u8 ├── getMetaData.json ├── manifestWithDiscontinuity.m3u8 ├── media-u4cc7m30h_b1496000_3383.ts.mp4 ├── playlist.json ├── simpleManifest.m3u8 ├── simpleManifestWithEndList.m3u8 ├── ufhwdejgz-1.ts ├── updatedManifest1.m3u8 └── updatedManifest1_withoutDiscontinuity.m3u8 ├── streaming_client ├── mux │ └── main.cc └── streaming_client.xcodeproj │ ├── project.pbxproj │ └── project.xcworkspace │ └── contents.xcworkspacedata └── unit ├── BackendClientFactory-spec.js ├── EntryDownloader-spec.js ├── MasterManifestGenerator-spec.js ├── NetworkClientFactory-spec.js ├── PersistenceFormat-spec.js ├── PlaylistGenerator-spec.js ├── Recording-spec.js ├── SessionManager-spec.js ├── flavor-downloader-spec.js ├── http-utils-spec.js ├── promise-m3u8-spec.js └── ts2mp4-spec.js /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | bin 4 | tests 5 | .git 6 | build_scripts/ffmpeg 7 | build_scripts/n3.0 8 | liveRecorder/ts_to_mp4_convertor/obj 9 | liveRecorder/ts_to_mp4_convertor/ffmpeg 10 | streaming_client 11 | packager/build -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | *.pyc 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 18 | .grunt 19 | 20 | # node-waf configuration 21 | .lock-wscript 22 | 23 | # Compiled binary addons (http://nodejs.org/api/addons.html) 24 | build/Release 25 | 26 | # Dependency directory 27 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 28 | node_modules 29 | .idea 30 | 31 | # Build files created by node-gyp 32 | node_addons/FormatConverter/build 33 | 34 | # packager ignore directories 35 | common/config/config.json 36 | packager/build 37 | packager/bin/build_log 38 | 39 | # ignore regression (ci) reports 40 | reports 41 | 42 | # ignor local congif 43 | common/config/configMapping.json 44 | 45 | # ignore local liveRecorder config 46 | liveRecorder/Config/configMapping.ini 47 | 48 | #ignore ts_to_mp4_convertor obj dir 49 | liveRecorder/ts_to_mp4_convertor/obj 50 | 51 | # ignore ts_to_mp4_convertor’s xcode project dir 52 | liveRecorder/ts_to_mp4_convertor/ts_to_mp4_convertor.xcodeproj/ 53 | 54 | # ignore streaming_client’s Xcode project dir 55 | tests/streaming_client/streaming_client.xcodeproj/ 56 | build/config.gypi 57 | 58 | # ignore liveRecorder executables under bin 59 | liveRecorder/bin 60 | 61 | # ignore liveController executables under bin 62 | bin 63 | 64 | deployment/docker/.env 65 | deployment/kubernetes/config.yaml 66 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "boss": true, // true: Tolerate assignments where comparisons would be expected 3 | "curly": true, // true: Require {} for every new block or scope 4 | "eqeqeq": true, // true: Require triple equals (===) for comparison 5 | "eqnull": true, // true: Tolerate use of `== null` 6 | "immed": true, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());` 7 | "latedef": true, // true: Require variables/functions to be defined before being used 8 | "newcap": false, // true: Require capitalization of all constructor functions e.g. `new F()` 9 | "noarg": true, // true: Prohibit use of `arguments.caller` and `arguments.callee` 10 | "sub": true, // true: Tolerate using `[]` notation when it can still be expressed in dot notation 11 | "undef": true, // true: Require all non-global variables to be declared (prevents global leaks) 12 | "unused": true, 13 | "node": true, // Node.js 14 | "-W117": true // true: Ignore `not defined` errors as an example of using a rule (W117) by code. 15 | } -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 6.3.0 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## liveController & LiveRecorder 2 | The liveDVR repository contains to products: liveController and liveRecorder. 3 | Kaltura liveController controls the live stream including recording during live. 4 | The liveController handles live entries state and stream content integrity throughout live session. It integrates with the Wowza to get list of live streams and download the content. 5 | It integrates with the BE for live state update and live entry info get/set. 6 | The liveController supports DC failover. 7 | Kaltura liveRecorder is responsible for VOD preparation from recorded live content and upload to destination storage. 8 | 9 | ### Deployment 10 | 11 | #### LiveController Deployment 12 | Please refer to [liveController deployment doc](LiveController_deployment.md) 13 | 14 | #### liveRecorder Deployment 15 | Please refer to [liveRecorder deployment doc](liveRecorder/liveRecorder_deployment.md) 16 | 17 | #### meida_server Deployment 18 | Please refer to [Media Server deployment doc](https://github.com/kaltura/media-server/blob/4.5.14/deployment.md) 19 | 20 | ### Docker: 21 | Build: 22 | ``` 23 | ./deployment/docker/build_images.sh live-controller 24 | ``` 25 | 26 | 27 | ``` 28 | docker-compose -f ./deployment/docker/docker-compose.yml --project-directory ./deployment/docker/ up 29 | ``` 30 | ### Copyright & License 31 | 32 | All code in this project is released under the [AGPLv3 license](http://www.gnu.org/licenses/agpl-3.0.html) unless a different license for a particular library is specified in the applicable library path. 33 | 34 | Copyright © Kaltura Inc. All rights reserved. 35 | -------------------------------------------------------------------------------- /build_scripts/build_addon.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #=============================================================================== 4 | # FILE: build_addon.sh 5 | # USAGE: ./deploy_liveRecorder.sh 6 | # DESCRIPTION: 7 | # OPTIONS: --- 8 | # REQUIREMENTS: --- 9 | # BUGS: --- 10 | # NOTES: --- 11 | # AUTHOR: (), Lilach Maliniak 12 | # ORGANIZATION: Kaltura, inc. 13 | # CREATED: June 25 2017 14 | # REVISION: --- 15 | #=============================================================================== 16 | set -e 17 | if [ "$#" -lt 2 ]; then 18 | echo "usage : $0 [Release/Debug]" 19 | echo "example: $0 /opt/kaltura/liveController/v1.14.5 /opt/kaltura/liveController/v1.14.5/bin/ffmpeg/ffmpeg-3.0 Release" 20 | exit 1 21 | fi 22 | 23 | PRODUCT_ROOT_PATH=$1 24 | FFMPEG_PATH=$2 25 | BUILD_CONF=Release 26 | ADDON_PATH=${PRODUCT_ROOT_PATH}/node_addons/FormatConverter/ 27 | FFMPEG_SYMLINK=${ADDON_PATH}/build/FFmpeg 28 | FORMAT_CONVERTER_BIN=FormatConverter.so 29 | OS=`uname` 30 | RES=0 31 | 32 | echo "OS=$OS" 33 | 34 | [ "$3" = "Debug" ] && BUILD_CONF=Debug 35 | 36 | echo "PRODUCT_ROOT=${PRODUCT_ROOT}" 37 | echo "FFMPEG_PATH=${FFMPEG_PATH}" 38 | echo "BUILD_CONF=${BUILD_CONF}" 39 | 40 | mkdir -p ${PRODUCT_ROOT_PATH}/bin 41 | mkdir -p ${ADDON_PATH}/build 42 | 43 | # note: if the second argument already exists and is a directory, 44 | # ln will create a symlink to the target inside that directory. 45 | 46 | if [ -L ${FFMPEG_SYMLINK} ]; then 47 | echo "unlink ${FFMPEG_SYMLINK}" 48 | unlink ${FFMPEG_SYMLINK} 49 | fi 50 | 51 | if [ ! -r ${FFMPEG_SYMLINK} ]; then 52 | echo "ln -s ${FFMPEG_PATH} ${FFMPEG_SYMLINK}" 53 | ln -s ${FFMPEG_PATH} ${FFMPEG_SYMLINK} 54 | fi 55 | 56 | pushd ${ADDON_PATH} 57 | 58 | mkdir -p ${BUILD_CONF} 59 | 60 | `which node-gyp` || npm install node-gyp -g 61 | 62 | case ${OS} in 63 | 'Darwin') 64 | echo "Mac OS" 65 | GYP_ARGS='-- -f xcode' 66 | echo "${GYP_ARGS}" 67 | node-gyp configure ${GYP_ARGS} 68 | FORMAT_CONVERTER_BIN=FormatConverter.dylib 69 | ;; 70 | *) ;; 71 | esac 72 | 73 | echo "Start node-gyp configure" 74 | node-gyp configure 75 | 76 | if [ "${BUILD_CONF}" = "Debug" ]; then 77 | GYP_DEBUG="--debug" 78 | DEBUG_EXT=".debug" 79 | fi 80 | echo "Start node-gyp build. ${GYP_DEBUG}" 81 | node-gyp build ${GYP_DEBUG} -v 82 | 83 | if [ -r "build/${BUILD_CONF}/${FORMAT_CONVERTER_BIN}" ]; then 84 | echo "cp build/${BUILD_CONF}/${FORMAT_CONVERTER_BIN} ${PRODUCT_ROOT_PATH}/bin/FormatConverter.node${DEBUG_EXT}" 85 | cp "build/${BUILD_CONF}/${FORMAT_CONVERTER_BIN}" "${PRODUCT_ROOT_PATH}/bin/FormatConverter.node${DEBUG_EXT}" 86 | echo "### build finished successfully" 87 | else 88 | echo "### build failed, could not access build/${BUILD_CONF}/${FORMAT_CONVERTER_BIN}, check if file exists" 89 | RES=1 90 | fi 91 | 92 | popd 93 | 94 | exit ${RES} 95 | -------------------------------------------------------------------------------- /build_scripts/build_ffmpeg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #=============================================================================== 4 | # FILE: build_ffmpeg.sh 5 | # USAGE: ./deploy_liveRecorder.sh 6 | # DESCRIPTION: 7 | # OPTIONS: --- 8 | # REQUIREMENTS: --- 9 | # BUGS: --- 10 | # NOTES: --- 11 | # AUTHOR: (), Lilach Maliniak 12 | # ORGANIZATION: Kaltura, inc. 13 | # CREATED: June 25 2017 14 | # REVISION: --- 15 | #=============================================================================== 16 | set -e 17 | if [ "$#" -lt 2 ]; then 18 | echo "usage : $0 [Release/Debug]" 19 | echo "example: $0 /opt/kaltura/liveController/v1.14.5 /opt/kaltura/liveController/v1.14.5/bin/ffmpeg 3.0" 20 | exit 1 21 | fi 22 | 23 | FFMPEG_BUILD_PATH=$1 24 | FFMPEG_VERSION=$2 25 | BUILD_CONF=Release 26 | TMP_PATH=/var/tmp 27 | OS=`uname` 28 | 29 | [ "${OS}" = "Darwin" ] && TMP_PATH=. 30 | [ "$3" = "Debug" ] && BUILD_CONF=Debug 31 | 32 | echo "build mode ${BUILD_CONF}" 33 | echo "TMP_PATH=${TMP_PATH}" 34 | echo "FFMPEG_BUILD_PATH=${FFMPEG_BUILD_PATH}" 35 | 36 | mkdir -p ${FFMPEG_BUILD_PATH} 37 | mkdir -p ${TMP_PATH} 38 | 39 | echo "Fetching tar from https://github.com/FFmpeg/FFmpeg/releases/download/n${FFMPEG_VERSION}/ffmpeg-${FFMPEG_VERSION}.tar.gz" 40 | curl -sL https://github.com/FFmpeg/FFmpeg/releases/download/n${FFMPEG_VERSION}/ffmpeg-${FFMPEG_VERSION}.tar.gz -o ${TMP_PATH}/ffmpeg-${FFMPEG_VERSION}.tar.gz 41 | 42 | echo "opening tarball ${TMP_PATH}/ffmpeg-${FFMPEG_VERSION}.tar.gz" 43 | tar -xzf ${TMP_PATH}/ffmpeg-${FFMPEG_VERSION}.tar.gz -C ${FFMPEG_BUILD_PATH} 44 | 45 | cd ${FFMPEG_BUILD_PATH}/ffmpeg-${FFMPEG_VERSION} 46 | 47 | [ "${BUILD_CONF}" = "Debug" ] && DEBUG_SPECIFICS='--enable-debug --disable-optimizations' 48 | 49 | CONFIG_FILENAME=./lastConfigure 50 | 51 | CONF_CMD="./configure --disable-everything --disable-doc --enable-protocol=file --enable-encoder=movtext --enable-demuxer=mpegts --enable-muxer=rtp_mpegts --enable-parser=h264 --enable-parser=aac --enable-muxer=mp4 --enable-zlib --enable-bsf=aac_adtstoasc --enable-decoder=aac --enable-encoder=aac --enable-decoder=h264 --enable-muxer=flv --enable-protocol=rtmp --enable-encoder=libmp3lame ${DEBUG_SPECIFICS}" 52 | 53 | [ "${OS}" = "Linux" ] && CONF_CMD="${CONF_CMD} --enable-pic" 54 | [ "${OS}" = "Darwin" ] && CONF_CMD="${CONF_CMD} --disable-static --enable-shared --enable-hwaccels " 55 | 56 | echo "configuring ffmpeg..." 57 | eval "${CONF_CMD}" 58 | 59 | echo "Saving configs to ${CONFIG_FILENAME} for traceability" 60 | echo "version=${FFMPEG_VERSION} ${CONF_CMD}" > ${CONFIG_FILENAME} 61 | 62 | echo "### starting ffmpeg build..." 63 | 64 | make 65 | 66 | echo "### ffmpeg build finished successfully. ffmpeg path: ${FFMPEG_BUILD_PATH}" -------------------------------------------------------------------------------- /build_scripts/build_ffmpeg4.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # install_ffmpeg.sh 4 | # livetranscoder 5 | # 6 | # Created by Guy.Jacubovski on 30/12/2018. 7 | # Copyright © 2018 Kaltura. All rights reserved. 8 | set -ex 9 | 10 | 11 | #export PATH="$HOME/compiled/bin":$PATH 12 | #export PKG_CONFIG_PATH=$HOME/compiled/lib/pkgconfig 13 | 14 | if [ "$#" -lt 1 ]; then 15 | echo "usage : $0 " 16 | echo "example: $0 /opt/kaltura/liveController/v1.14.5/bin/ffmpeg/ffmpeg-4.1" 17 | exit 1 18 | fi 19 | 20 | FFMPEG_BUILD_PATH=$1 21 | 22 | rm -rf "FFMPEG_BUILD_PATH" 23 | git clone -b n4.1 https://git.ffmpeg.org/ffmpeg.git "$FFMPEG_BUILD_PATH" || echo "FFmpeg dir already exists" 24 | cd "$FFMPEG_BUILD_PATH" 25 | ./configure --disable-everything --disable-doc --enable-protocol=file --enable-encoder=movtext --enable-demuxer=mpegts --enable-muxer=rtp_mpegts --enable-parser=h264 --enable-parser=aac --enable-muxer=mp4 --enable-zlib --enable-bsfs --enable-decoder=aac --enable-encoder=aac --enable-decoder=h264 --enable-muxer=flv --enable-protocol=rtmp --enable-encoder=libmp3lame 26 | make 27 | -------------------------------------------------------------------------------- /build_scripts/build_ts2mp4_convertor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #=============================================================================== 4 | # FILE: build_ts2mp4_convertor.sh 5 | # USAGE: ./deploy_liveRecorder.sh 6 | # DESCRIPTION: 7 | # OPTIONS: --- 8 | # REQUIREMENTS: --- 9 | # BUGS: --- 10 | # NOTES: --- 11 | # AUTHOR: (), Lilach Maliniak 12 | # ORGANIZATION: Kaltura, inc. 13 | # CREATED: June 25, 2017 14 | # REVISION: --- 15 | #=============================================================================== 16 | set -e 17 | if [ "$#" -lt 2 ]; then 18 | echo "usage build_ts2mp4_convertor " 19 | echo "example: $0 /opt/kaltura/liveController/v1.14.5/liveRecorder /opt/kaltura/liveController/v1.14.5/bin/ffmpeg/ffmpeg-4.1" 20 | exit 1 21 | fi 22 | 23 | PRODUCT_ROOT_PATH=$1 24 | FFMPEG_BUILD_PATH=$2 25 | TARGET=ts_to_mp4_convertor 26 | CONVERTOR_DIR=${PRODUCT_ROOT_PATH}/${TARGET} 27 | RES=0 28 | 29 | export FFMPEG_BUILD_PATH 30 | 31 | 32 | if [ -w ${CONVERTOR_DIR} ]; then 33 | pushd ${CONVERTOR_DIR} 34 | mkdir -p obj 35 | 36 | echo "starting to build ${TARGET}" 37 | 38 | make install 39 | 40 | if [ $? -eq 0 ] ; then 41 | echo "**************************************************************************************" 42 | echo "${TARGET} was built successfully, copying to bin folder" 43 | echo "**************************************************************************************" 44 | else 45 | echo "**************************************************************************************" 46 | echo "Something went wrong, failed to build ts_to_mp4_convertor!!!, please check build results" 47 | echo "**************************************************************************************" 48 | RES=1 49 | fi 50 | popd 51 | else 52 | echo "***********************************************************************************************************************************" 53 | echo "failed to build ${TARGET}, check that ${CONVERTOR_DIR} exist and has write permission" 54 | echo "***********************************************************************************************************************************" 55 | RES=1 56 | fi 57 | 58 | exit ${RES} 59 | -------------------------------------------------------------------------------- /common/Configuration.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/24/2015. 3 | */ 4 | 5 | var path = require('path'); 6 | var fs = require('fs'); 7 | var _ = require('underscore') 8 | var hostname = require('./utils/hostname'); 9 | 10 | module.exports = (function(){ 11 | 12 | let machineName = hostname.getLocalMachineHostname(); 13 | let machineShortName = hostname.getLocalMachineHostname(false); 14 | var configTemplateContent = fs.readFileSync(path.join(__dirname, './config/config.json.template'), 'utf8'); 15 | var configObj = JSON.parse(configTemplateContent); 16 | 17 | var mappingFilePath = path.join(__dirname, 'config', 'configMapping.json'); 18 | if (fs.existsSync(mappingFilePath)) 19 | { 20 | var mappingContent = fs.readFileSync(mappingFilePath, 'utf8'); 21 | var mappingObj=JSON.parse(mappingContent); 22 | _.each(mappingObj, function(value, key) { 23 | console.log("Matching configurations arguments. Key: [%s] => Match: [%s]", key, machineName.match(key)); 24 | if (machineName.match(key)) { 25 | assignValues(value, configObj); 26 | } 27 | }); 28 | } 29 | 30 | 31 | function assignValues(configPropertiesObj, configOutputObj) { 32 | for (var p in configPropertiesObj) { 33 | if (configPropertiesObj.hasOwnProperty(p)) { 34 | if (!_.isArray(configPropertiesObj[p]) && 35 | _.isObject(configPropertiesObj[p]) && 36 | _.has(configOutputObj,p)) { 37 | assignValues(configPropertiesObj[p], configOutputObj[p]); 38 | } 39 | else { 40 | configOutputObj[p] = configPropertiesObj[p]; 41 | } 42 | } 43 | } 44 | } 45 | 46 | let confString = JSON.stringify(configObj, null, 2); 47 | confString = confString.replace(/~/g,hostname.homedir()); 48 | confString = confString.replace(/@HOSTNAME@/g, machineName); 49 | confString = confString.replace(/@HOSTNAME_SHORT@/g, machineShortName); 50 | fs.writeFileSync(path.join(__dirname, './config/config.json'), confString); 51 | 52 | var nconf = require('nconf'); 53 | 54 | // Setup nconf to use (in-order): 55 | // 1. Command-line arguments 56 | // 2. Environment variables 57 | // 3. A file located at 'path/to/config.json' 58 | // 59 | nconf.argv() 60 | .env() 61 | .file({ file: path.join(__dirname, './config/config.json') }); 62 | 63 | return nconf; 64 | })(); 65 | -------------------------------------------------------------------------------- /common/config/configMapping.json.template: -------------------------------------------------------------------------------- 1 | { 2 | ".*": { 3 | "rootFolderPath" : "@LIVE_CONTENT_PATH@", 4 | "oldContentFolderPath" : "@LIVE_ARCHIVE_CONTENT_PATH@", 5 | "logFileName" : "@LOG_FILE_NAME@", 6 | "backendClient" : { 7 | "serviceUrl" :"@KALTURA_SERVICE_URL@", 8 | "adminSecret" :"@KALTURA_PARTNER_ADMIN_SECRET@", 9 | "partnerId" : "@KALTURA_PARTNER_ID@" 10 | }, 11 | "mediaServer" : { 12 | "hostname": "@HOSTNAME@", 13 | "wowzaServer": "localhost", 14 | "user" : "@WOWZA_ADMIN_USER@", 15 | "originMode": true, 16 | "password" : "@WOWZA_ADMIN_PASSWORD@" 17 | }, 18 | "recording" : { 19 | "enable" : true, 20 | "recordingFolderPath" : "@RECORDING_FOLDER@/recordings", 21 | "completedRecordingFolderPath" : "@RECORDING_FOLDER@/incoming" 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /common/logger.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by ron.yadgar on 18/04/2016. 3 | */ 4 | 5 | 6 | var config = require('./Configuration'); 7 | var path = require('path'); 8 | var mkdirp = require('mkdirp'); 9 | var util = require('util'); 10 | var hostname = require('./utils/hostname'); 11 | var log4js = require( "log4js" ); 12 | 13 | 14 | var logFullPath = path.resolve(config.get('logFileName')); 15 | logFullPath= logFullPath.replace(/~/g,hostname.homedir()); 16 | mkdirp.sync(path.dirname(logFullPath)); 17 | 18 | var appenders = { 19 | file: { 20 | "type": "dateFile", 21 | "filename": logFullPath, 22 | "pattern": ".yyyy-MM-dd", 23 | "alwaysIncludePattern": false, 24 | "timezoneOffset": config.get('logTimeZoneOffset') // NYC timezone offset relative to UTC (5 * 60) 25 | } 26 | } 27 | 28 | 29 | if (config.get('logToConsole')) 30 | { 31 | appenders.out={ 32 | "type": "console", 33 | "layout": { 34 | "type": "pattern", 35 | pattern: "%d{ABSOLUTE} %[%-5p%] %c %m" 36 | } 37 | }; 38 | } 39 | 40 | var log4jsConfiguration = { 41 | appenders: appenders, 42 | categories: { 43 | default: { appenders: Object.keys(appenders) , level: config.get('logLevel') } 44 | } 45 | 46 | }; 47 | 48 | log4js.configure(log4jsConfiguration); 49 | 50 | // Support log rotate - this is the signal that is used 51 | process.on('SIGUSR1', function() { 52 | log4js.clearAppenders(); 53 | log4js.configure(log4jsConfiguration); 54 | }); 55 | 56 | 57 | function decorate(logger, id) { 58 | 59 | if (id) { 60 | var loggerEx = {}; 61 | function modify(func) { 62 | 63 | loggerEx[func] = function () { 64 | if (arguments && arguments.length > 0) { 65 | arguments[0] = id + arguments[0]; 66 | } 67 | return logger[func].apply(logger, arguments); 68 | } 69 | } 70 | 71 | modify("debug"); 72 | modify("warn"); 73 | modify("info"); 74 | modify("trace"); 75 | modify("error"); 76 | modify("fatal"); 77 | 78 | loggerEx.logger = logger; 79 | 80 | return loggerEx; 81 | } else { 82 | return logger; 83 | } 84 | } 85 | 86 | function getLogger(module, id) { 87 | var loggerName=module; 88 | var type=typeof(module); 89 | if (type==='object') 90 | loggerName=path.basename(module.filename); 91 | 92 | 93 | var logger=log4js.getLogger("["+loggerName+"]"); 94 | 95 | return decorate(logger,id); 96 | } 97 | 98 | module.exports = { 99 | decorate: decorate, 100 | getLogger: getLogger 101 | }; -------------------------------------------------------------------------------- /common/utils/hostname.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 9/24/2015. 3 | */ 4 | 5 | const os = require('os'); 6 | const child_process = require('child_process'); 7 | 8 | 9 | function homedir() { 10 | var env = process.env; 11 | var home = env.HOME; 12 | var user = env.LOGNAME || env.USER || env.LNAME || env.USERNAME; 13 | 14 | if (process.platform === 'win32') { 15 | return env.USERPROFILE || env.HOMEDRIVE + env.HOMEPATH || home || null; 16 | } 17 | 18 | if (process.platform === 'darwin') { 19 | return home || (user ? '/Users/' + user : null); 20 | } 21 | 22 | if (process.platform === 'linux') { 23 | return home || (process.getuid() === 0 ? '/root' : (user ? '/home/' + user : null)); 24 | } 25 | 26 | return home || null; 27 | } 28 | 29 | function getLocalMachineHostname(full = true) { 30 | let res = "HOSTNAME_PLACEHOLDER"; 31 | if (os.platform() == 'win32' || os.platform() == 'win64') 32 | { 33 | // On Windows 34 | res = process.env.COMPUTERNAME; 35 | if (full && process.env.USERDNSDOMAIN && process.env.USERDOMAIN !== "") 36 | { 37 | res += "." + process.env.USERDNSDOMAIN; 38 | } 39 | } 40 | else 41 | { 42 | // On Linux 43 | let cmd = "hostname"; 44 | cmd += full ? " -f" : " -s"; 45 | res = child_process.execSync(cmd).toString().trim() 46 | } 47 | return res; 48 | } 49 | 50 | module.exports = { 51 | homedir: homedir, 52 | getLocalMachineHostname: getLocalMachineHostname 53 | }; 54 | -------------------------------------------------------------------------------- /deployment/addServerNode.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by user on 01/06/2017. 3 | */ 4 | const BackendClient = require('../lib/BackendClient'); 5 | const ErrorUtils = require('../lib/utils/error-utils'); 6 | const Hostname = require('../common/utils/hostname'); 7 | 8 | // If hostname is transferred as a parameter take its value, otherwise get hostname of local machine. 9 | let hostname = (process.argv[2]) ? process.argv[2] : Hostname.getLocalMachineHostname(); 10 | console.log("Adding server node [" + hostname + "]"); 11 | return BackendClient.addServerNode(hostname) 12 | .then((apiResult) => { 13 | console.log("API call headers: " + apiResult.headers); 14 | console.log("API call result: " + apiResult.result); 15 | if (apiResult.err || apiResult.result.objectType === 'KalturaAPIException') { 16 | throw new Error("Failed to add serverNode. " + ErrorUtils.error2string(apiResult.err)); 17 | } 18 | console.log("ServerNode [" + hostname + "] was successfully added to server"); 19 | return (apiResult.result); 20 | }) 21 | .then((result) => { 22 | return BackendClient.enableServerNode(result.id); 23 | }) 24 | .then((apiResult) => { 25 | if (apiResult.err || apiResult.result.objectType === 'KalturaAPIException') { 26 | throw new Error("Failed to enable serverNode. " + ErrorUtils.error2string(apiResult.err)) 27 | } 28 | console.log("Server node [" + apiResult.result.name + "] was successfully enabled"); 29 | }) 30 | .catch((error) => { 31 | console.log("Error while adding new ServerNode. Error: " + ErrorUtils.error2string(error)); 32 | }); -------------------------------------------------------------------------------- /deployment/docker.md: -------------------------------------------------------------------------------- 1 | to build dockers run from root: 2 | `docker build -t kaltura/live-controller -f ./deployment/docker/liveController/Dockerfile .` 3 | `docker build -t kaltura/live-packager -f ./deployment/docker/packager/Dockerfile .` 4 | 5 | on wowza: 6 | 7 | `docker build -t kaltura/media-server .` 8 | 9 | stop all: 10 | docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q) 11 | 12 | run 13 | `docker-compose -f ./deployment/docker/docker-compose.yml up 14 | ` 15 | 16 | 17 | 18 | #option1: 19 | #docker run --add-host pa-udrm:127.0.0.1 -v /Users/guyjacubovski/dev/nginx-vod-module-saas:/usr/local/nginx/externalConf -p 80:80 --name packager -t live-packager 20 | #or docker run -p 80:80 --name packager -t kaltura/live-packager 21 | 22 | 23 | #docker exec -it `docker ps | grep "live-packager" | awk '{print $1}'` bash 24 | 25 | 26 | -------------------------------------------------------------------------------- /deployment/docker/.env.template: -------------------------------------------------------------------------------- 1 | VERSION= 2 | SERVICE_URL= 3 | PARTNER_ID=-5 4 | PARTNER_ADMIN_SECRET= 5 | WSE_LIC= 6 | CONTENT_DIR= 7 | LOGS_DIR= 8 | CONFIG_DIR= 9 | PACKAGER_SECURE_TOKEN= 10 | PACKAGER_PORT= -------------------------------------------------------------------------------- /deployment/docker/build_images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd "$(dirname "$0")" 3 | lastdir=`pwd` 4 | tag=test 5 | 6 | if [[ $@ == *'push'* ]]; then 7 | echo "ecs login" 8 | eval `aws ecr get-login --no-include-email --region eu-west-1` 9 | fi 10 | 11 | if [[ $@ == *'media-server'* ]] || [[ $@ == *'all'* ]] ; then 12 | echo "Build media-server" 13 | docker build -t kaltura/media-server -f ../../../media-server/Dockerfile ../../../media-server/ 14 | echo "tag media-server:$tag" 15 | docker tag kaltura/media-server 983882572364.dkr.ecr.eu-west-1.amazonaws.com/media-server:$tag 16 | if [[ $@ == *'push'* ]]; then 17 | echo "push media-server" 18 | docker push 983882572364.dkr.ecr.eu-west-1.amazonaws.com/media-server:$tag 19 | fi 20 | fi 21 | 22 | if [[ $@ == *'live-controller'* ]] || [[ $@ == *'all'* ]] ; then 23 | echo "Build live-controller" 24 | docker build -t kaltura/live-controller -f ./liveController/Dockerfile ../../ 25 | echo "tag live-controller:$tag" 26 | docker tag kaltura/live-controller 983882572364.dkr.ecr.eu-west-1.amazonaws.com/live-controller:$tag 27 | if [[ $@ == *'push'* ]]; then 28 | echo "push live-controller" 29 | docker push 983882572364.dkr.ecr.eu-west-1.amazonaws.com/live-controller:$tag 30 | fi 31 | fi 32 | 33 | if [[ $@ == *'live-packager'* ]] || [[ $@ == *'all'* ]] ; then 34 | echo "Build live-packager" 35 | docker build -t kaltura/live-packager -f ./livePackager/Dockerfile ../../ 36 | echo "tag live-packager:$tag" 37 | docker tag kaltura/live-packager 983882572364.dkr.ecr.eu-west-1.amazonaws.com/live-packager:$tag 38 | if [[ $@ == *'push'* ]]; then 39 | echo "push live-packager" 40 | docker push 983882572364.dkr.ecr.eu-west-1.amazonaws.com/live-packager:$tag 41 | fi 42 | fi 43 | 44 | if [[ $@ == *'live-recorder'* ]] || [[ $@ == *'all'* ]] ; then 45 | echo "Build live-recorder" 46 | docker build -t kaltura/live-recorder -f ./liveRecorder/Dockerfile ../../ 47 | echo "tag live-recorder:$tag" 48 | docker tag kaltura/live-recorder 983882572364.dkr.ecr.eu-west-1.amazonaws.com/live-recorder:$tag 49 | if [[ $@ == *'push'* ]]; then 50 | echo "push live-recorder" 51 | docker push 983882572364.dkr.ecr.eu-west-1.amazonaws.com/live-recorder:$tag 52 | fi 53 | fi 54 | 55 | if [[ $@ == *'live-jobs'* ]] || [[ $@ == *'all'* ]] ; then 56 | echo "Build live-jobs" 57 | docker build -t kaltura/live-jobs -f ./liveJobs/Dockerfile ../../ 58 | echo "tag live-jobs:$tag" 59 | docker tag kaltura/live-jobs 983882572364.dkr.ecr.eu-west-1.amazonaws.com/live-jobs:$tag 60 | if [[ $@ == *'push'* ]]; then 61 | echo "push live-jobs" 62 | docker push 983882572364.dkr.ecr.eu-west-1.amazonaws.com/live-jobs:$tag 63 | fi 64 | fi 65 | 66 | 67 | 68 | cd $lastdir -------------------------------------------------------------------------------- /deployment/docker/docker-compose-full.yml: -------------------------------------------------------------------------------- 1 | 2 | #We still need to add more configuration to .env and change the configMapping accordingly, then make sure it's in /myconfig/ 3 | version: '3.1' 4 | services: 5 | media-server: 6 | env_file: ./.env 7 | image: kaltura/media-server:${VERSION} 8 | hostname: ${SERVER_NODE_HOST_NAME} 9 | networks: 10 | internal_net: 11 | aliases: 12 | - ${SERVER_NODE_HOST_NAME} 13 | volumes: 14 | - "${LOGS_DIR}/wowza/:/var/log/wowza/" 15 | container_name: ms 16 | ports: 17 | - "1935:1935" 18 | liveController: 19 | env_file: .env 20 | image: kaltura/live-controller:${VERSION} 21 | networks: ['internal_net'] 22 | hostname: ${SERVER_NODE_HOST_NAME} 23 | volumes: 24 | - "${LOGS_DIR}/liveController/:/var/log/liveController/" 25 | - "${CONTENT_DIR}:/web/content/kLive:" 26 | container_name: lc 27 | depends_on: 28 | - media-server 29 | liveRecorder: 30 | env_file: .env 31 | image: kaltura/live-recorder:${VERSION} 32 | hostname: ${SERVER_NODE_HOST_NAME} 33 | volumes: 34 | - "${LOGS_DIR}/liveRecorder:/var/log/liveRecorder" 35 | - "${CONTENT_DIR}:/web/content/kLive" 36 | container_name: lr 37 | livePackager: 38 | env_file: .env 39 | image: kaltura/live-packager:${VERSION} 40 | networks: ['internal_net'] 41 | extra_hosts: 42 | - "pa-udrm:127.0.0.1" 43 | ports: 44 | - "8080:8080" 45 | volumes: 46 | - "${LOGS_DIR}/livePackager/:/usr/local/nginx/logs/" 47 | - "${CONTENT_DIR}:/web/content/kLive" 48 | container_name: lp 49 | depends_on: 50 | - media-server 51 | - liveController 52 | 53 | liveJobs: 54 | env_file: .env 55 | image: kaltura/live-jobs:${VERSION} 56 | networks: ['internal_net'] 57 | volumes: 58 | - "${CONTENT_DIR}:/web/content/kLive" 59 | container_name: lj 60 | 61 | networks: {internal_net: {}} -------------------------------------------------------------------------------- /deployment/docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | #We still need to add more configuration to .env and change the configMapping accordingly, then make sure it's in /myconfig/ 2 | version: '3.1' 3 | services: 4 | liveController: 5 | env_file: .env 6 | image: kaltura/live-controller:${VERSION} 7 | hostname: ${SERVER_NODE_HOST_NAME} 8 | volumes: 9 | - "${LOGS_DIR}/liveController/:/var/log/liveController/" 10 | - "content:/web5/content/kLive/" 11 | container_name: liveController 12 | volumes: 13 | content: 14 | driver: local 15 | driver_opts: 16 | type: "nfs" 17 | o: "addr=pa-isilon2-front-api,nolock,soft,rw" 18 | device: ":/ifs/web5/content/kLive" -------------------------------------------------------------------------------- /deployment/docker/initScript.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -z "$SERVER_NODE_HOST_NAME" ]]; then 4 | SERVER_NODE_HOST_NAME=`hostname -f` 5 | fi 6 | 7 | if [[ -n "$MY_POD_NAME" ]]; then 8 | export EC2_REGION=`echo $MY_NODE_NAME | cut -d'.' -f2` 9 | export SERVER_NODE_HOST_NAME="${MY_POD_NAME}.${EC2_REGION}" 10 | fi -------------------------------------------------------------------------------- /deployment/docker/liveController/Dockerfile: -------------------------------------------------------------------------------- 1 | #docker build -t kaltura/livecontroller:1.26 . 2 | #docker run -d -v /myconfig/:/myconfig/ -v /var/log/liveController/:/var/log/liveController/ --network=host kaltura/livecontroller:1.26 ubuntudrm 3 | #docker exec -it 4 | 5 | ARG NODE_VERSION=11.9.0 6 | FROM node:$NODE_VERSION AS build 7 | 8 | #RUN apk add --update build-base curl nasm tar bzip2 zlib-dev yasm-dev 9 | 10 | RUN apt-get update && apt-get install -y nasm 11 | 12 | WORKDIR /opt/kaltura/liveController/ 13 | COPY package*.json ./ 14 | RUN npm install 15 | 16 | 17 | COPY ./build_scripts/ ./build_scripts/ 18 | COPY ./node_addons/ ./node_addons/ 19 | RUN ./build_scripts/build_ffmpeg.sh /tmp/ 3.0 20 | RUN ./build_scripts/build_addon.sh /opt/kaltura/liveController/ /tmp/ffmpeg-3.0/ 21 | 22 | COPY ./ ./ 23 | 24 | RUN rm -rf ./build 25 | 26 | FROM node:$NODE_VERSION-slim 27 | 28 | 29 | WORKDIR /opt/kaltura/liveController/ 30 | 31 | COPY --from=build /opt/kaltura/liveController/ ./ 32 | 33 | VOLUME /var/log/liveController/ 34 | #content folder 35 | VOLUME /web/content/kLive 36 | 37 | 38 | COPY ./deployment/docker/initScript.sh . 39 | COPY ./deployment/docker/liveController/entryPoint.sh . 40 | 41 | ENTRYPOINT ["./entryPoint.sh"] -------------------------------------------------------------------------------- /deployment/docker/liveController/entryPoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source initScript.sh 4 | 5 | echo applying new configMapping 6 | LOG_FILE_NAME=/var/log/liveController/kLiveController.log 7 | LIVE_CONTENT_PATH="${BASE_CONTENT_FOLDER}/live" 8 | LIVE_ARCHIVE_CONTENT_PATH="${BASE_CONTENT_FOLDER}/archive" 9 | RECORDING_FOLDER="${BASE_CONTENT_FOLDER}/liveRecorder" 10 | 11 | 12 | jsonFile=$1; 13 | 14 | node > ./common/config/configMapping.json << EOF 15 | const fs=require("fs"); 16 | //Read data 17 | let data = JSON.parse(fs.readFileSync('./common/config/configMapping.json.template', 'utf8')); 18 | let config=data[".*"]; 19 | config.rootFolderPath="$LIVE_CONTENT_PATH"; 20 | config.oldContentFolderPath="$LIVE_ARCHIVE_CONTENT_PATH"; 21 | config.logFileName="$LOG_FILE_NAME"; 22 | 23 | config.backendClient.serviceUrl="$SERVICE_URL"; 24 | config.backendClient.adminSecret="$PARTNER_ADMIN_SECRET"; 25 | config.backendClient.partnerId=$PARTNER_ID; 26 | 27 | config.mediaServer.hostname="$SERVER_NODE_HOST_NAME"; 28 | config.mediaServer.user="$WOWZA_ADMIN_USER"; 29 | config.mediaServer.password="$WOWZA_ADMIN_PASSWORD"; 30 | config.mediaServer.wowzaHost="$WOWZA_HOSTNAME"; 31 | config.mediaServer.wowzaMetadataHost="$WOWZA_METADATA_HOST"; 32 | config.mediaServer.port=80; 33 | 34 | config.recording.recordingFolderPath= "$RECORDING_FOLDER/recordings"; 35 | config.recording.completedRecordingFolderPath= "$RECORDING_FOLDER/incoming"; 36 | 37 | //Output data 38 | console.log(JSON.stringify(data,null,2)); 39 | 40 | EOF 41 | 42 | 43 | cat ./common/config/configMapping.json 44 | 45 | echo adding $SERVER_NODE_HOST_NAME to backend 46 | node ./deployment/addServerNode.js $SERVER_NODE_HOST_NAME 47 | exec node ./lib/App.js 48 | echo "it's the end of the world" -------------------------------------------------------------------------------- /deployment/docker/liveJobs/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | RUN apk add --update bash && rm -rf /var/cache/apk/* 4 | 5 | WORKDIR /bin/scripts/ 6 | 7 | COPY ./deployment/runtimeScripts/cleaner.sh ./cleaner.sh 8 | COPY ./deployment/runtimeScripts/recordingCleaner.sh ./recordingCleaner.sh 9 | COPY ./deployment/runtimeScripts/liveCleaner.sh ./liveCleaner.sh 10 | 11 | RUN echo "SHELL=/bin/bash" >> /var/spool/cron/crontabs/root 12 | RUN echo "* * * * * /bin/scripts/liveCleaner.sh >> /var/log/liveCleaner.log 2>&1" >> /var/spool/cron/crontabs/root 13 | RUN echo "* * * * * /bin/scripts/recordingCleaner.sh >> /var/log/recordingCleaner.log 2>&1" >> /var/spool/cron/crontabs/root 14 | 15 | ENV BASE_DIR /web/content/kLive 16 | ENV DAYS_TO_KEEP_LIVE 1 17 | ENV DAYS_TO_KEEP_RECORDINGS 2 18 | 19 | CMD crond -l 2 -f -------------------------------------------------------------------------------- /deployment/docker/livePackager/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:stretch-slim AS builder 2 | 3 | ARG CONF_FILE=/opt/nginx-vod-module-saas/conf/nginx.conf 4 | 5 | 6 | RUN apt-get update \ 7 | && apt-get install -y wget git procps zlib1g-dev build-essential libpcre3 libpcre3-dev libssl1.0-dev \ 8 | && rm -rf /var/lib/apt/lists/* 9 | 10 | 11 | WORKDIR /opt 12 | 13 | COPY ./deployment/docker/livePackager/deploy.sh . 14 | 15 | RUN ./deploy.sh 16 | 17 | WORKDIR /opt/nginx 18 | 19 | RUN ./configure --with-http_secure_link_module \ 20 | --add-module=/opt/nginx_mod_akamai_g2o/ \ 21 | --add-module=/opt/headers-more-nginx-module/ \ 22 | --add-module=/opt/nginx-vod-module/ \ 23 | --add-module=/opt/nginx-secure-token-module/ \ 24 | --add-module=/opt/nginx_requestid/ \ 25 | --with-http_stub_status_module \ 26 | --with-file-aio \ 27 | --with-threads \ 28 | --with-cc-opt="-O3 -DDISABLE_PTS_DELAY_COMPENSATION" \ 29 | --conf-path=$CONF_FILE && \ 30 | make && \ 31 | make install 32 | 33 | 34 | FROM debian:stretch-slim 35 | 36 | ARG CONF_FOLDER=/opt/nginx-vod-module-saas 37 | 38 | #copy build artifacts 39 | COPY --from=builder /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.2 /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.2 40 | COPY --from=builder /usr/local/nginx /usr/local/nginx 41 | 42 | COPY ./packager/config/ /opt/nginx-vod-module-saas/conf/ 43 | COPY ./packager/www /opt/nginx-vod-module-saas/static/ 44 | 45 | #log folder 46 | VOLUME /var/log/nginx 47 | 48 | #content folder 49 | VOLUME /web/content/kLive 50 | 51 | #conf folder 52 | VOLUME /usr/local/nginx/externalConf 53 | 54 | 55 | ENV PACKAGER_PORT 80 56 | ENV LIVE_ENCRYPT_HLS_KEY 1234 57 | 58 | EXPOSE $PACKAGER_PORT 59 | 60 | STOPSIGNAL SIGTERM 61 | 62 | WORKDIR /usr/local/nginx 63 | 64 | COPY ./deployment/docker/initScript.sh . 65 | COPY ./deployment/docker/livePackager/entryPoint.sh . 66 | 67 | ENTRYPOINT ["./entryPoint.sh"] -------------------------------------------------------------------------------- /deployment/docker/livePackager/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | nginxVersion=1.11.0 4 | 5 | wget http://nginx.org/download/nginx-${nginxVersion}.tar.gz 6 | tar -zxvf nginx-${nginxVersion}.tar.gz 7 | rm nginx-${nginxVersion}.tar.gz -f 8 | mv nginx-${nginxVersion} nginx 9 | 10 | echo "pulling nginx-secure-token-module" 11 | git clone https://github.com/kaltura/nginx-secure-token-module 12 | 13 | echo "pulling nginx-vod-module" 14 | git clone https://github.com/kaltura/nginx-vod-module 15 | 16 | echo "pulling nginx_requestid" 17 | git clone https://github.com/kaltura/nginx_requestid 18 | 19 | echo "pulling headers-more-nginx-module" 20 | git clone https://github.com/openresty/headers-more-nginx-module 21 | 22 | echo "pulling headers-more-nginx-module" 23 | git clone https://github.com/openresty/headers-more-nginx-module 24 | 25 | echo "pulling nginx_mod_akamai_g2o" 26 | git clone https://github.com/kaltura/nginx_mod_akamai_g2o 27 | -------------------------------------------------------------------------------- /deployment/docker/livePackager/entryPoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source initScript.sh 4 | 5 | echo "statrting nginx" 6 | cd /opt/nginx-vod-module-saas/conf/ 7 | 8 | 9 | CONF_FOLDER=/usr/local/nginx/externalConf 10 | WWW_DIR=/opt/nginx-vod-module-saas/static/ 11 | CONTENT_DIR=/web/content/kLive 12 | 13 | if [ "$(ls -A $CONF_FOLDER)" ]; then 14 | echo "copy configuration" 15 | cp -r $CONF_FOLDER/* /opt/nginx-vod-module-saas/ 16 | 17 | else 18 | echo "create configuration from templates" 19 | 20 | sed -e "s#@PORT@#$PACKAGER_PORT#g" -e "s#@WWW_DIR@#$WWW_DIR#g" ./nginx.conf.template > ./nginx.conf 21 | 22 | sed -e "s#@LIVE_ENCRYPT_HLS_KEY@#$LIVE_ENCRYPT_HLS_KEY#g" ./nginx.conf.live.protocols.template > ./nginx.conf.live.protocols 23 | 24 | sed -e "s#@CONTENT_DIR@#$CONTENT_DIR#g" ./nginx.conf.live.bootstrap.template > ./nginx.conf.live.bootstrap 25 | 26 | cp ./nginx.conf.live.conf.template ./nginx.conf.live.conf 27 | fi 28 | 29 | ln -sf /dev/stdout /usr/local/nginx/logs/access.log 30 | ln -sf /dev/stderr /usr/local/nginx/logs/error.log 31 | 32 | echo "start process" 33 | exec /usr/local/nginx/sbin/nginx -g "daemon off;" 34 | -------------------------------------------------------------------------------- /deployment/docker/liveRecorder/Dockerfile: -------------------------------------------------------------------------------- 1 | #docker build -t kaltura/live-recorder -f ./deployment/docker/liveRecorder/Dockerfile . 2 | #docker run -d kaltura/live-recorder 3 | #docker exec -it `docker ps | grep recorder | awk '{print $1}' ` bash 4 | 5 | 6 | 7 | FROM python:2.7.14-stretch AS builder 8 | 9 | RUN apt-get update \ 10 | && apt-get install -y wget curl nasm build-essential zlib1g-dev 11 | 12 | 13 | WORKDIR /opt/kaltura/workspace/ 14 | 15 | COPY ./build_scripts/ ./build_scripts/ 16 | RUN ./build_scripts/build_ffmpeg.sh /tmp/ 3.0 17 | 18 | 19 | COPY ./liveRecorder/ ./liveRecorder/ 20 | 21 | 22 | RUN ./build_scripts/build_ts2mp4_convertor.sh /opt/kaltura/workspace/liveRecorder/ /tmp/ffmpeg-3.0/ 23 | 24 | RUN pip install schedule m3u8 poster psutil Crypto 25 | 26 | 27 | FROM python:2.7.14-slim-stretch 28 | 29 | 30 | WORKDIR /opt/kaltura/liveRecorder/ 31 | 32 | VOLUME /web/content/kLive 33 | 34 | #copy PIP packages 35 | COPY --from=builder /usr/local/lib/python2.7/site-packages /usr/local/lib/python2.7/site-packages 36 | COPY --from=builder /opt/kaltura/workspace/liveRecorder . 37 | 38 | COPY ./deployment/docker/initScript.sh . 39 | COPY ./deployment/docker/liveRecorder/entryPoint.sh . 40 | 41 | 42 | ENTRYPOINT ["./entryPoint.sh"] -------------------------------------------------------------------------------- /deployment/docker/liveRecorder/entryPoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source initScript.sh 4 | 5 | LOGDIR=/var/log/liveRecorder 6 | HOSTNAME=`hostname` 7 | RECORDING_FOLDER=/web/content/kLive/liveRecorder 8 | 9 | sed -e "s#@HOSTNAME@#.*#g" \ 10 | -e "s#@RECORDING_FOLDER@#$RECORDING_FOLDER#g" \ 11 | -e "s#@KALTURA_PARTNER_ADMIN_SECRET@#$PARTNER_ADMIN_SECRET#g" \ 12 | -e "s#@KALTURA_PARTNER_ID@#$PARTNER_ID#g" \ 13 | -e "s#@VOD_UPLOAD_MODE@#remote#g" \ 14 | -e "s#@LIVE_PACKAGER_TOKEN@#$PACKAGER_SECURE_TOKEN#g" \ 15 | -e "s#@LOGS_BASE_PATH@#$LOGDIR#g" \ 16 | -e "s#@KALTURA_SERVICE_URL@#$SERVICE_URL#g" \ 17 | ./Config/configMapping.ini.template > ./Config/configMapping.ini 18 | 19 | pwd 20 | cat ./Config/configMapping.ini 21 | 22 | createFolders() { 23 | [ -d ${LOGDIR} ] || mkdir -p ${LOGDIR} 24 | [ -d ${RECORDING_FOLDER} ] || mkdir -p ${RECORDING_FOLDER} 25 | 26 | #create recording folders if they are not there 27 | mkdir -p ${RECORDING_FOLDER}/{incoming,done,error} 28 | mkdir -p ${RECORDING_FOLDER}/recordings/{append,newSession} 29 | 30 | SHARED_APP_DIR=$RECORDING_FOLDER 31 | echo "Creating folders in ${SHARED_APP_DIR} for ${HOSTNAME}" 32 | if [ -d ${SHARED_APP_DIR} ] ; then 33 | mkdir -p ${SHARED_APP_DIR}/${HOSTNAME}/{UploadTask/{incoming,failed,processing},ConcatenationTask/{failed,processing}} 34 | [ -L ${SHARED_APP_DIR}/${HOSTNAME}/ConcatenationTask/incoming ] || ln -s ${RECORDING_FOLDER}/incoming ${SHARED_APP_DIR}/${HOSTNAME}/ConcatenationTask/incoming 35 | else 36 | echo "can't find ${SHARED_APP_DIR}, cannot start, check mounts" 37 | exit 2 38 | fi 39 | } 40 | 41 | echo "starting liveRecorder..." 42 | createFolders || exit 4 43 | python main.py 44 | echo "ok!" 45 | 46 | 47 | -------------------------------------------------------------------------------- /deployment/kubernetes/config.template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: my-env 5 | data: 6 | SERVICE_URL: "http://www.kaltura.com" 7 | PARTNER_ID: "" 8 | PARTNER_ADMIN_SECRET: "" 9 | WSE_LIC: "" 10 | PACKAGER_SECURE_TOKEN: "" 11 | PACKAGER_PORT: "8080" -------------------------------------------------------------------------------- /deployment/kubernetes/live-front.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: live-front 5 | annotations: 6 | service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http 7 | spec: 8 | externalTrafficPolicy: Local 9 | selector: 10 | app: live-front 11 | type: LoadBalancer 12 | ports: 13 | - port: 80 14 | name: http 15 | targetPort: http 16 | --- 17 | apiVersion: v1 18 | kind: ConfigMap 19 | metadata: 20 | name: live-front-config 21 | data: 22 | nginx.conf: | 23 | server { 24 | listen 80; 25 | root /usr/share/nginx/html; 26 | resolver 100.64.0.10; # fill will the ip address of the KubeDNS 27 | ssl_verify_client off; 28 | 29 | location / { 30 | root /usr/share/nginx/html; 31 | index index.html index.htm; 32 | } 33 | location ~ /m/([^.]+).([^\/]+)/(.*) { 34 | proxy_set_header Host $host; #preserve host name 35 | proxy_set_header X-Real-IP $remote_addr; #preserve ip address 36 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 37 | 38 | proxy_http_version 1.1; 39 | proxy_set_header Upgrade $http_upgrade; 40 | proxy_set_header Connection "upgrade"; 41 | proxy_pass http://$1.live-publish.default.svc.cluster.local:8080/$3$is_args$args; 42 | } 43 | } 44 | --- 45 | apiVersion: apps/v1 46 | kind: Deployment 47 | metadata: 48 | name: live-front 49 | spec: 50 | selector: 51 | matchLabels: 52 | app: live-front 53 | replicas: 1 54 | template: 55 | metadata: 56 | labels: 57 | app: live-front 58 | spec: 59 | containers: 60 | - name: live-front 61 | image: nginx:1.13.10-alpine 62 | ports: 63 | - name: http 64 | containerPort: 80 65 | volumeMounts: 66 | - name: mysite-configs 67 | mountPath: /etc/nginx/conf.d/default.conf 68 | subPath: nginx.conf 69 | volumes: 70 | - name: mysite-configs 71 | configMap: 72 | name: live-front-config -------------------------------------------------------------------------------- /deployment/kubernetes/readme.md: -------------------------------------------------------------------------------- 1 | 2 | export NAME=live.a.rtc.kaltura.com 3 | export KOPS_STATE_STORE=s3://live.a.rtc.kaltura.com 4 | kops create cluster --zones eu-west-1a,eu-west-1b,eu-west-1c --node-count=1 --node-size c5.2xlarge $NAME 5 | kops edit cluster $NAME 6 | 7 | 8 | 9 | kubectl apply -f ./deployment/kubernetes/config.yaml 10 | kubectl apply -f ./deployment/kubernetes/live-front.yaml 11 | kubectl apply -f ./deployment/kubernetes/live-publish.yaml 12 | 13 | kubectl scale sts live-publish --replicas=0 14 | 15 | 16 | #find live -mtime +3 -ls -exec rm {} \; -------------------------------------------------------------------------------- /deployment/runtimeScripts/cleaner.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | function cleaner() { 4 | local BASE_DIR=$1 5 | local DAYS_TO_KEEP=$2 6 | 7 | 8 | for folder in "${FOLDER_TO_COPY[@]}"; 9 | do 10 | folderToClean="${1}/${folder}" 11 | echo "deleting files older than $DAYS_TO_KEEP days in $folderToClean" 12 | find $folderToClean -type f -mtime +$DAYS_TO_KEEP -delete 13 | echo "deleting empty folders older than $DAYS_TO_KEEP days in $folderToClean" 14 | find $folderToClean -type d -mtime +$DAYS_TO_KEEP -delete 15 | done 16 | } 17 | 18 | -------------------------------------------------------------------------------- /deployment/runtimeScripts/liveCleaner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | echo `date` 4 | 5 | sd=$(dirname "$0") 6 | #export BASE_DIR=/web/content/kLive 7 | #export DAYS_TO_KEEP_LIVE=7 8 | export FOLDER_TO_COPY=("archive" "live") 9 | 10 | source $sd/cleaner.sh 11 | 12 | cleaner $BASE_DIR $DAYS_TO_KEEP_LIVE -------------------------------------------------------------------------------- /deployment/runtimeScripts/recordingCleaner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | echo `date` 4 | 5 | sd=$(dirname "$0") 6 | BASE_REC_DIR="${BASE_DIR}/liveRecorder" 7 | #export DAYS_TO_KEEP_RECORDINGS=30 8 | export FOLDER_TO_COPY=("done" "error" "recordings/append" "recordings/newSession" ) 9 | 10 | source $sd/cleaner.sh 11 | 12 | cleaner $BASE_REC_DIR $folders $DAYS_TO_KEEP_RECORDINGS -------------------------------------------------------------------------------- /development.md: -------------------------------------------------------------------------------- 1 | ## OSX Dev ### 2 | ```run 3 | brew install wget 4 | brew install openssl 5 | 6 | $ cd /usr/local/include 7 | $ ln -s ../opt/openssl/include/openssl .``` -------------------------------------------------------------------------------- /grunt-config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/12/2015. 3 | */ 4 | module.exports = { 5 | 6 | app_files: { 7 | js: [ 'lib/**/*.js'], 8 | json: [ 'lib/**/*.json','package.json'], 9 | all:['lib/**/**'] 10 | }, 11 | 12 | test_files: { 13 | js: [ 14 | 'tests/**/*.js*' 15 | ] 16 | }, 17 | 18 | node_files:{ 19 | all:[ 20 | 'node_modules/**/**' 21 | ] 22 | }, 23 | 24 | reports_dir : 'reports', 25 | 26 | unit_tests : 'tests/unit/*.js', 27 | component_tests : 'tests/component/*.js', 28 | regression_tests : 'tests/regression/*.js', 29 | recording : 'tests/recording/*.js' 30 | 31 | }; 32 | -------------------------------------------------------------------------------- /gruntfile.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/12/2015. 3 | */ 4 | var match = require('matchdep'); 5 | 6 | module.exports = function (grunt) { 7 | 'use strict'; 8 | process.env['MOCHA_FILE'] = './reports/junit_test_report.xml'; 9 | process.env['PROPERTIES'] = `BUILD_ID:${process.env.BUILD_NUMBER}`; 10 | 11 | var userConfig = require('./grunt-config.js'); 12 | 13 | grunt.file.expand('./node_modules/grunt-*/tasks').forEach(grunt.loadTasks); 14 | grunt.initConfig(userConfig); 15 | 16 | if (process.env.UNIT_TEST_PATH) { 17 | console.log(`UNIT_TEST_PATH=${process.env.UNIT_TEST_PATH}`); 18 | userConfig.unit_tests = process.env.UNIT_TEST_PATH; 19 | } 20 | 21 | grunt.loadTasks('./lib/grunt/tasks'); 22 | grunt.loadTasks('./lib/grunt/config'); 23 | 24 | 25 | // Default task(s). 26 | grunt.registerTask('default', ['jshint:dev', 'unit-test:dev', 'component-test:dev', 'mocha_istanbul:dev']); 27 | }; 28 | -------------------------------------------------------------------------------- /lib/Adapters/APIQueryAdapter.js: -------------------------------------------------------------------------------- 1 | var Q = require('q'); 2 | var _ = require('underscore'); 3 | var config = require('../../common/Configuration'); 4 | var WowzaStreamInfo = require('./WowzaStreamInfo.js'); 5 | var BaseAdapter=require('./BaseAdapter.js').BaseAdapter; 6 | var util=require('util'); 7 | var backendClient=require('../BackendClientFactory.js').getBackendClient(); 8 | 9 | function APIQueryAdapter() { 10 | BaseAdapter.call(this); 11 | } 12 | util.inherits(APIQueryAdapter,BaseAdapter); 13 | 14 | var getWowzaStreamInfo = function() { 15 | return new WowzaStreamInfo(this.entryId, this.flavorParamsIds); 16 | }; 17 | 18 | APIQueryAdapter.prototype.getLiveEntries = function() { 19 | return backendClient.getLiveEntriesForMediaServer() 20 | .then(function(res) { 21 | _.each(res, function(r) { 22 | r.getStreamInfo = getWowzaStreamInfo; 23 | }); 24 | return Q.resolve(res); 25 | }); 26 | }; 27 | 28 | module.exports = APIQueryAdapter; -------------------------------------------------------------------------------- /lib/Adapters/AdapterFactory.js: -------------------------------------------------------------------------------- 1 | 2 | var config = require('../../common/Configuration'); 3 | var WowzaAdapter=require('./WowzaAdapter.js'); 4 | var CompositeWowzaAdapter=require('./CompositeWowzaAdapter.js'); 5 | var APIQueryAdapter=require('./APIQueryAdapter.js'); 6 | var config = require('../../common/Configuration'); 7 | const _ = require('underscore'); 8 | 9 | var simulateStreams = config.get('simulateStreams'); 10 | var regressionAdapterConfig = config.get('regressionAdapter'); 11 | 12 | var mediaServerConfig = config.get('mediaServer'); 13 | var adapter; 14 | 15 | exports.getAdapter = function() { 16 | 17 | if (!adapter) { 18 | if (regressionAdapterConfig && regressionAdapterConfig.enable) { 19 | let RegressionAdapter = require('./RegressionAdapter/RegressionAdapter'); 20 | adapter = new RegressionAdapter(); 21 | } else if (simulateStreams && simulateStreams.enable) { 22 | adapter = require('./TestAdapter.js').TestAdapter.instance; 23 | } else { 24 | if (_.isArray(mediaServerConfig.hostname)) { 25 | adapter = new CompositeWowzaAdapter(mediaServerConfig); 26 | } else { 27 | let wowzaHost = mediaServerConfig.wowzaHost ? mediaServerConfig.wowzaHost: mediaServerConfig.hostname; 28 | let metadataHostName = mediaServerConfig.wowzaMetadataHost ? mediaServerConfig.wowzaMetadataHost: wowzaHost; 29 | adapter = new WowzaAdapter(mediaServerConfig, wowzaHost, metadataHostName); 30 | } 31 | } 32 | } 33 | 34 | return adapter; 35 | }; 36 | -------------------------------------------------------------------------------- /lib/Adapters/BaseAdapter.js: -------------------------------------------------------------------------------- 1 | var Q = require('q'); 2 | var _ = require('underscore'); 3 | var util=require('util'); 4 | var events = require('events'); 5 | 6 | function BaseAdapter() { 7 | this.entries = {}; 8 | } 9 | 10 | BaseAdapter.prototype.getLiveEntries = function() {}; 11 | 12 | BaseAdapter.prototype.toJSON = function() { 13 | return {}; 14 | }; 15 | 16 | 17 | util.inherits(BaseAdapter, events.EventEmitter); 18 | 19 | function StreamInfo(entryId, flavorParamsIds) { 20 | this.entryId = entryId; 21 | this.flavorParamsIds = _.uniq(flavorParamsIds.split(',')); 22 | } 23 | 24 | StreamInfo.prototype.getAllFlavors = function() {} 25 | 26 | 27 | class BaseTestAdapter extends BaseAdapter { 28 | 29 | constructor() { 30 | super(); 31 | // controllerWrapper ref. 32 | // used to signal regression-tests (grunt unit test), that regression ended. 33 | this.controllerWrapper; 34 | this.once('exit', this.gracefullyExit); 35 | } 36 | } 37 | 38 | BaseTestAdapter.prototype.setControllerWrapper = function(controllerWrapper) { 39 | this.controllerWrapper = controllerWrapper; 40 | }; 41 | 42 | BaseTestAdapter.prototype.gracefullyExit = function(exit_code) { 43 | if (this.controllerWrapper) { 44 | this.controllerWrapper.emit('exit', exit_code); 45 | } else { 46 | process.exit(exit_code); 47 | } 48 | } 49 | 50 | module.exports = { 51 | BaseAdapter : BaseAdapter, 52 | BaseTestAdapter : BaseTestAdapter, 53 | StreamInfo : StreamInfo 54 | }; 55 | -------------------------------------------------------------------------------- /lib/Adapters/CompositeWowzaAdapter.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/26/2015. 3 | */ 4 | 5 | var Q = require('q'); 6 | var _ = require('underscore'); 7 | var baseAdapter = require('./BaseAdapter.js').BaseAdapter; 8 | var WowzaAdapter=require('./WowzaAdapter.js'); 9 | var util = require('util'); 10 | 11 | 12 | function CompositeWowzaAdapter(mediaServerConfig) { 13 | this.entries = {}; 14 | baseAdapter.call(this); 15 | this._mediaServers=_.map(mediaServerConfig.hostname,(hostname)=>{ 16 | return new WowzaAdapter(mediaServerConfig,hostname); 17 | }) 18 | } 19 | util.inherits(CompositeWowzaAdapter,baseAdapter); 20 | 21 | 22 | CompositeWowzaAdapter.prototype.getLiveEntries = function() { 23 | 24 | return Q.allSettled(_.map(this._mediaServers,(mediaServerAdapter)=>{ 25 | return mediaServerAdapter.getLiveEntries(); 26 | })).then(function(res) { 27 | 28 | let entries = _.reduce(res,(response,p) => { 29 | if (p.state === "fulfilled") { 30 | return response.concat(p.value); 31 | } 32 | return response; 33 | },[]); 34 | 35 | entries=_.uniq(entries,(ret)=> { 36 | return ret.entryId; 37 | }); 38 | 39 | return entries; 40 | }); 41 | }; 42 | 43 | module.exports = CompositeWowzaAdapter; -------------------------------------------------------------------------------- /lib/Adapters/EntryCache.js: -------------------------------------------------------------------------------- 1 | 2 | var Q = require('q'); 3 | var WowzaAdapter=require('./WowzaAdapter.js'); 4 | var _=require('underscore'); 5 | //var logger = require('../../common/logger/logger')(module); 6 | var backendClient=require('../BackendClientFactory.js').getBackendClient(); 7 | 8 | 9 | 10 | module.exports = function() { 11 | 12 | 13 | var entryCache={}; 14 | 15 | var outdatedTime=60*1000;//one minute 16 | 17 | var lastRevoke=0; 18 | var self=this; 19 | 20 | 21 | self.get=function(entryId) { 22 | return entryCache[entryId]; 23 | }; 24 | self.getEntries=function(entriesId) { 25 | 26 | var now=new Date(); 27 | 28 | var entriesToFetch=[]; 29 | 30 | //revoke from cache outdated entries 31 | if (now-lastRevoke>outdatedTime) { 32 | _.each(entryCache,function(entry) { 33 | if (now-entry.lastUpdated>outdatedTime) { 34 | console.warn("deleting ",entry.entryId); 35 | delete entryCache[entry.entryId]; 36 | }}); 37 | 38 | lastRevoke=now; 39 | } 40 | 41 | //determin what is missing from cache 42 | _.each(entriesId,function(entryId) { 43 | if (!entryCache[entryId]) { 44 | entriesToFetch.push(entryId); 45 | }}); 46 | 47 | var refreshCache= Q.resolve(true); 48 | if (entriesToFetch.length>0) { 49 | 50 | console.warn("Fetching ",entriesToFetch); 51 | refreshCache = backendClient.getEntries(entriesToFetch).then(function (entries) { 52 | _.each(entries, function (entry) { 53 | entry["lastUpdated"] = now; 54 | entryCache[entry.entryId] = entry; 55 | }); 56 | }); 57 | 58 | return refreshCache; 59 | 60 | 61 | } else { 62 | console.warn("No need to fetch!!"); 63 | 64 | }; 65 | 66 | 67 | return refreshCache; 68 | 69 | 70 | }; 71 | 72 | 73 | return self; 74 | }(); -------------------------------------------------------------------------------- /lib/Adapters/RegressionAdapter/RegressionTestStaticDiagram.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | 4 | class Controller { 5 | Adapter adapter 6 | } 7 | 8 | class BaseAdapter { 9 | 10 | } 11 | 12 | class RegressionAdapter { 13 | RegressionEngine regressionEngine 14 | getLiveEntries() 15 | } 16 | note top: for each entry (currently only one tested),\n return new WowzaStreamInfo 17 | 18 | class RegressionEngine { 19 | EntryInfo[] entries 20 | hlsRegressionValidatorBase[] validators 21 | } 22 | 23 | class RegressionEntryInfo { 24 | 25 | } 26 | 27 | note bottom: regression entries config 28 | 29 | class HlsChecksumRegressionValidator { 30 | this_run_results {} 31 | regression_results_db {} 32 | ValidateRegressionResults() 33 | } 34 | 35 | class RegressionConfig { 36 | 37 | } 38 | 39 | class RegressionValidatorFactory { 40 | getValidator 41 | } 42 | 43 | class RegressionValidatorBase { 44 | 45 | } 46 | 47 | class HlsAnalysisRegressionValidator { 48 | 49 | } 50 | 51 | Controller --> RegressionAdapter 52 | BaseAdapter <|-- RegressionAdapter 53 | RegressionAdapter --> RegressionEngine 54 | RegressionValidatorBase <|-- HlsChecksumRegressionValidator 55 | HlsChecksumRegressionValidator <|-- HlsAnalysisRegressionValidator 56 | RegressionEntryInfo <--* RegressionEngine 57 | RegressionValidatorBase <--* RegressionEngine 58 | RegressionValidatorFactory ..> HlsChecksumRegressionValidator 59 | RegressionValidatorFactory ..> HlsAnalysisRegressionValidator 60 | 61 | 62 | @enduml -------------------------------------------------------------------------------- /lib/Adapters/RegressionAdapter/RegressionValidatorBase.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by lilach.maliniak on 21/08/2016. 3 | */ 4 | const Q = require('q'); 5 | 6 | class RegressionValidatorBase { 7 | 8 | constructor(config, flavors, chunklistIndexes, verbose) { 9 | this.entryId = config.entryId; 10 | this.flavors = flavors; 11 | this.config = config.validator; 12 | this.last_chunklist_index = chunklistIndexes.end_index; 13 | this.start_chunklist_index = chunklistIndexes.start_index; 14 | this.verbose = verbose; 15 | } 16 | 17 | init() {}; 18 | validateAllFlavors(index) { return Q.reject('not implemented'); }; 19 | saveResultsToFile(reason) { return Q.reject('not implemented'); }; 20 | isEntryConfigValid() { 21 | if (!this.start_chunklist_index || !this.last_chunklist_index || 22 | this.start_chunklist_index === -1 || this.last_chunklist_index === -1 23 | || isNaN(this.start_chunklist_index) || isNaN(this.last_chunklist_index) || 24 | this.start_chunklist_index >= this.last_chunklist_index) { 25 | return false; 26 | } else { 27 | return true; 28 | } 29 | } 30 | } 31 | 32 | RegressionValidatorBase.prototype.createDestinationDir = function() {}; 33 | RegressionValidatorBase.prototype.addChunklist = function() {}; 34 | RegressionValidatorBase.prototype.validateResults = function() {}; 35 | RegressionValidatorBase.prototype.validateSingleFlavor = function(flavorId, index) {}; 36 | 37 | module.exports = RegressionValidatorBase; -------------------------------------------------------------------------------- /lib/Adapters/RegressionAdapter/RegressionValidatorFactory.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by lilach.maliniak on 21/08/2016. 3 | */ 4 | var _ = require('underscore'); 5 | var config = require('../../../common/Configuration'); 6 | var HlsChecksumRegressionValidator = require('./hlsChecksumRegressionValidator'); 7 | var HlsAnalysisRegressionValidator = require('./hlsAnalysisRegressionValidaotor'); 8 | var analysis = config.get('regressionAdapter').analysis; 9 | 10 | var getValidator = function(entryConfig, flavors, last_chunklist_index) { 11 | 12 | if (analysis) { 13 | return new HlsAnalysisRegressionValidator(entryConfig, flavors, last_chunklist_index); 14 | } else { 15 | return new HlsChecksumRegressionValidator(entryConfig, flavors, last_chunklist_index); 16 | } 17 | } 18 | 19 | module.exports = { 20 | getValidator: getValidator 21 | } 22 | 23 | -------------------------------------------------------------------------------- /lib/Adapters/RegressionAdapter/regression_utility.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by lilach.maliniak on 06/08/2016. 3 | */ 4 | var _ = require('underscore'); 5 | var logger = require('../../../common/logger').getLogger("regression_utility"); 6 | var util=require('util'); 7 | 8 | function logCommandLineArgs() { 9 | var argv = process.argv; 10 | var count = 0; 11 | logger.info('================================================================================'); 12 | logger.info('|LiveController\'s command line arguments: |'); 13 | logger.info('================================================================================'); 14 | _.each(process.argv, (arg) => { 15 | logger.info(util.format('(%s) %s', ++count, arg)); 16 | }); 17 | console.log('================================================================================'); 18 | count = 0; 19 | console.log('================================================================================'); 20 | console.log('|LiveController\'s command line arguments: |'); 21 | console.log('================================================================================'); 22 | _.each(process.argv, (arg) => { 23 | console.log(util.format('(%s) %s', ++count, arg)); 24 | }); 25 | console.log('================================================================================'); 26 | } 27 | 28 | module.exports = { 29 | logCommandLineArgs : logCommandLineArgs 30 | } 31 | -------------------------------------------------------------------------------- /lib/App.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 9/10/2015. 3 | */ 4 | 5 | var config = require('./../common/Configuration'); 6 | var http = require('http'); 7 | var _ = require('underscore'); 8 | var path = require('path'); 9 | var os = require('os'); 10 | 11 | /** 12 | * 13 | * Due to a misconfiguration, please do not write to the logger here. 14 | */ 15 | // Since node v0.10 sets this to 5 by default 16 | http.globalAgent.maxSockets = Infinity; 17 | 18 | if (os.platform() === 'darwin') { 19 | http.globalAgent.maxSockets = 50; 20 | } 21 | 22 | var prefixParam = _.find(process.argv, function(arg){ 23 | return arg.match(/prefix:.*?/) 24 | }); 25 | 26 | var prefix = ""; 27 | if (prefixParam) 28 | { 29 | prefix = prefixParam.match(/prefix:(.*?)$/)[1]; 30 | includePrefixInLogFileName(prefix, 'logFileName'); 31 | } 32 | 33 | function includePrefixInLogFileName(prefix, configPropertyName){ 34 | var originalFilePath = config.get(configPropertyName) 35 | var fileDir = path.dirname(originalFilePath); 36 | var fileName = path.basename(originalFilePath); 37 | var newFileName = prefix + "_" + fileName; 38 | var newFilePath = path.join(fileDir, newFileName); 39 | config.set(configPropertyName, newFilePath); 40 | } 41 | 42 | var ControllerCtor = require('./Controller'); 43 | var controller = new ControllerCtor(prefix); 44 | controller.start(); 45 | -------------------------------------------------------------------------------- /lib/BackendClientFactory.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/25/2015. 3 | */ 4 | 5 | var config = require('./../common/Configuration'); 6 | 7 | 8 | module.exports = (function(){ 9 | 10 | var getBackendClient = function getNetworkClient(){ 11 | var networkClient; 12 | if (config.get('mockBackend')) 13 | { 14 | if (config.get('readMockBackendResponseFromFile')) 15 | { 16 | networkClient = require('./mocks/BackendClientFileBasedMock'); 17 | } 18 | else 19 | { 20 | networkClient = require('./mocks/BackendClientMock'); 21 | } 22 | 23 | } 24 | else 25 | { 26 | networkClient = require('./BackendClient'); 27 | } 28 | 29 | return networkClient; 30 | }; 31 | 32 | return { 33 | getBackendClient : getBackendClient 34 | }; 35 | })(); 36 | -------------------------------------------------------------------------------- /lib/Diagnostics/InvalidClipError.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by igors on 13/10/2016. 3 | */ 4 | const diagnosticsAlerts = require('./DiagnosticsAlerts') 5 | 6 | class InvalidClipError extends Error { 7 | constructor(type, fileInfo, other) { 8 | let alert 9 | if (type instanceof diagnosticsAlerts.Alert) { 10 | alert = type; 11 | } else { 12 | other = other || []; 13 | alert = Reflect.construct(type, [fileInfo.flavor, fileInfo.chunkName].concat(other)); 14 | } 15 | super(alert.msg) 16 | this._alert = alert; 17 | } 18 | } 19 | 20 | module.exports = InvalidClipError; 21 | -------------------------------------------------------------------------------- /lib/MonitorServer.js: -------------------------------------------------------------------------------- 1 | var http = require("http"); 2 | var logger = require('../common/logger').getLogger("Controller"); 3 | var process= require('process'); 4 | 5 | 6 | var re = /\/v?([^\/]*)\/lib/.exec(__dirname); 7 | if (re != null && re.length === 2) { 8 | app_version=re[1]; 9 | } else { 10 | app_version=__dirname; 11 | 12 | } 13 | 14 | if (process.env.VERSION) { 15 | app_version=process.env.VERSION; 16 | } 17 | 18 | class MonitorServer { 19 | constructor(prefix) { 20 | var _this=this; 21 | this._regexes = []; 22 | this.prefix = prefix; 23 | 24 | this.register('/info',this.info); 25 | 26 | this.httpServer = http.createServer(function(request, response) { 27 | try { 28 | 29 | 30 | var url=request.url; 31 | 32 | for (const reg of _this._regexes) { 33 | var re=url.match(reg[0]); 34 | if (re) { 35 | request.args = re.splice(1); 36 | try { 37 | reg[1].bind(_this)(request, response); 38 | } catch(e) { 39 | response.writeHead(500, {"Content-Type": "text/plain"}); 40 | response.end(e.toString()); 41 | } 42 | return; 43 | } 44 | } 45 | 46 | response.writeHead(404); 47 | response.end(); 48 | }catch(e) { 49 | 50 | logger.info("Exception returning response to monitor server", e, e.stack); 51 | } 52 | }) 53 | } 54 | 55 | listen(port) { 56 | logger.info("Listening on port %s", port); 57 | this.httpServer.listen(port); 58 | } 59 | 60 | register(regex,func) { 61 | this._regexes.push([regex,func]); 62 | } 63 | 64 | info(request,response) { 65 | response.writeHead(200, {"Content-Type": "application/json"}); 66 | var content= { 67 | uptime: Math.floor(process.uptime()), 68 | nodeVersion: process.version 69 | }; 70 | content.version = app_version; 71 | content.prefix = this.prefix; 72 | response.end(JSON.stringify(content) + '\n'); 73 | 74 | } 75 | } 76 | 77 | 78 | module.exports= MonitorServer; 79 | -------------------------------------------------------------------------------- /lib/NetworkClient.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/24/2015. 3 | */ 4 | 5 | var Q = require('q'); 6 | var request = require('request'); 7 | var util = require('util'); 8 | var logger = require('../common/logger').getLogger("NetworkClient"); 9 | 10 | function requestPromisify(requestData) { 11 | return Q.Promise((resolve, reject) => { 12 | request(requestData, (error, response, body)=> { 13 | if (error) { 14 | error.retryable = true; 15 | reject(error); 16 | return; 17 | } 18 | resolve({body: body, response: response}); 19 | }); 20 | }); 21 | 22 | } 23 | function readRequest(requestData, retries = 0) 24 | { 25 | if (!requestData.encoding) { 26 | requestData.encoding = null; // Treat as binary 27 | } 28 | return requestPromisify(requestData).then((res)=> { 29 | if (res.response.statusCode === 200) { 30 | return Q.resolve({body: res.body, headers: res.response.headers}); 31 | } 32 | return Q.reject(new Error(util.format("Response for %s returned with status code %d", requestData.url, res.response.statusCode))); 33 | 34 | }).catch((err) => { 35 | if (err.retryable && retries > 0) { 36 | logger.error("Failed with: " + err + " retries of: " + retries); 37 | return Q.delay(1000).then(()=> { 38 | return readRequest(requestData, --retries); 39 | }); 40 | } 41 | return Q.reject(err); 42 | }); 43 | } 44 | 45 | module.exports = { 46 | read : readRequest 47 | }; -------------------------------------------------------------------------------- /lib/NetworkClientFactory.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/24/2015. 3 | */ 4 | 5 | var config = require('./../common/Configuration'); 6 | var realNetworkClient = require('./NetworkClient'); 7 | module.exports = (function(){ 8 | 9 | var getNetworkClient = function getNetworkClient(){ 10 | var networkClient; 11 | 12 | if (config.get('mockNetwork')) 13 | { 14 | networkClient = require('./mocks/NetworkClientMock'); 15 | } 16 | else 17 | { 18 | // In production, no need to require the same module over and over - it has its overhead 19 | networkClient = realNetworkClient; 20 | } 21 | 22 | return networkClient; 23 | }; 24 | 25 | return { 26 | getNetworkClient : getNetworkClient 27 | }; 28 | })(); -------------------------------------------------------------------------------- /lib/entry/StateManagerUMLState.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | scale 350 width 3 | 4 | [*] --> init 5 | init --> broadcasting : broadcast 6 | broadcasting --> playing : play 7 | playing --> playing: play 8 | suspending --> playing: play 9 | broadcasting --> broadcasting : update 10 | suspending --> broadcasting : update 11 | playing --> playing : update 12 | broadcasting --> suspending : suspend 13 | playing --> suspending : suspend 14 | suspending --> suspending : suspend 15 | suspending --> stopped : stop 16 | stopped --> [*] 17 | 18 | init: start state 19 | stopped: rename chunklist 20 | @enduml -------------------------------------------------------------------------------- /lib/grunt/config/component-tests-config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 9/2/2015. 3 | */ 4 | 5 | module.exports = function (grunt) { 6 | 'use strict'; 7 | 8 | grunt.config.set('component-test', { 9 | dev: { 10 | options: { 11 | reporter: 'spec', 12 | require: 'chai', 13 | timeout: 10000 14 | }, 15 | src: ['<%=component_tests%>'] 16 | }, 17 | ci: { 18 | options: { 19 | reporter: 'xunit', 20 | require: 'chai', 21 | timeout: 10000, 22 | quiet : true, 23 | captureFile: '<%=reports_dir%>/component-tests-report.xml' 24 | 25 | }, 26 | src: ['<%=component_tests%>'] 27 | } 28 | }); 29 | }; 30 | -------------------------------------------------------------------------------- /lib/grunt/config/coverage-config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/20/2015. 3 | */ 4 | 5 | module.exports = function (grunt) { 6 | 'use strict'; 7 | 8 | grunt.config.set('mocha_istanbul', { 9 | dev: { 10 | src: 'tests', 11 | options: { 12 | mask: '**/unit/*-spec.js', 13 | root: './lib', 14 | check: { 15 | lines: 85 16 | }, 17 | excludes : ['**/*config.js','kaltura-client-lib/**'], 18 | reportFormats: ['lcov'], 19 | coverageFolder: '<%= reports_dir %>/coverage' 20 | } 21 | }, 22 | ci: { 23 | src: 'tests', 24 | options: { 25 | mask: '**/unit/*-spec.js', 26 | root: './lib', 27 | check: { 28 | lines: 85 29 | }, 30 | excludes : ['**/*config.js', 'kaltura-client-lib/**'], 31 | reportFormats: ['lcov', 'cobertura'], 32 | coverageFolder: '<%= reports_dir %>/coverage' 33 | } 34 | } 35 | }); 36 | }; 37 | -------------------------------------------------------------------------------- /lib/grunt/config/jshint-config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/12/2015. 3 | */ 4 | 5 | module.exports = function (grunt) { 6 | 'use strict'; 7 | 8 | grunt.config.set('jshint', { 9 | dev: { 10 | src: ['gruntfile.js', 'lib/**/*.js*', 'tests/**/*.js*'], 11 | options: { 12 | jshintrc: '.jshintrc', 13 | reporter: require('jshint-junit-reporter'), 14 | ignores: ['lib/kaltura-client-lib/*'], 15 | reporterOutput: '<%= reports_dir %>/jshint-results.xml' 16 | } 17 | }, 18 | ci : { 19 | src: ['gruntfile.js', 'lib/**/*.js*', 'tests/**/*.js*'], 20 | options: { 21 | jshintrc: '.jshintrc', 22 | reporter: require('jshint-junit-reporter'), 23 | reporterOutput: '<%= reports_dir %>/jshint-results.xml', 24 | ignores: ['lib/kaltura-client-lib/*'] 25 | } 26 | } 27 | } 28 | ); 29 | }; -------------------------------------------------------------------------------- /lib/grunt/config/recording-config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by lilach.maliniak on 18/08/2016. 3 | */ 4 | 5 | /* 6 | Note: running recording from terminal: 7 | >npm test 8 | 9 | update package.json: 10 | 11 | "scripts": { 12 | "test": "MOCHA_FILE=./reports/last_recording_run/recording-report.xml mocha ./tests/recording/recording-spec.js --reporter mocha-junit-reporter --timeout 86400000" 13 | } 14 | 15 | option #2: 16 | from command line run: 17 | grunt recording-test:dev -record -url=http://kalseglive-a.akamaihd.net:80/dc-0/m/pa-live-publish4/kLive/smil:0_6gox09ym_all.smil/playlist.m3u8 18 | 19 | comment: 20 | >>>> option #1 requires all configuration set in configMapping.json or config.json.template 21 | >>>> option #2 can use command line args and configuration 22 | 23 | */ 24 | 25 | module.exports = function (grunt) { 26 | 'use strict'; 27 | 28 | grunt.config.set('recording-test', { 29 | 30 | dev: { 31 | options: { 32 | reporter: 'mocha-junit-reporter', 33 | /*reporterOptions: { 34 | mochaFile: './<%=reports_dir%>/recording/recording-report.xml', 35 | properties: { 36 | BUILD_ID: process.env.BUILD_NUMBER, 37 | useFullSuiteTitle: false 38 | }, 39 | },*/ 40 | //reporter: 'spec', 41 | require: 'chai', 42 | timeout: 86400000, 43 | captureFile: '<%=reports_dir%>/recording/recording.log' 44 | }, 45 | src: ['<%=recording%>'] 46 | }, 47 | ci: { 48 | options: { 49 | //reporter: 'xunit', 50 | reporter: 'mocha-junit-reporter', 51 | /*reporterOptions: { 52 | mochaFile: './<%=reports_dir%>/recording/recording-report.xml', 53 | properties: { 54 | BUILD_ID: process.env.BUILD_NUMBER, 55 | useFullSuiteTitle: false 56 | }, 57 | },*/ 58 | require: 'chai', 59 | timeout: 86400000, 60 | quiet : true, 61 | captureFile: '<%=reports_dir%>/recording/recording.log' 62 | 63 | }, 64 | src: ['<%=recording%>'] 65 | } 66 | }); 67 | }; 68 | -------------------------------------------------------------------------------- /lib/grunt/config/regression-tests-config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by lilach.maliniak on 02/08/2016. 3 | */ 4 | /* Todo: add support in multiple reporters both in regression and recording 5 | /* 6 | Note: running recording from terminal: 7 | >npm test 8 | 9 | update package.json: 10 | 11 | /* 12 | Note: 13 | running recording from terminal: 14 | 15 | option #1: 16 | 17 | update package.json: 18 | 19 | "scripts": { 20 | "test": "MOCHA_FILE=./reports/last_regression_run/regression-test-report.xml mocha ./tests/regression/regression-spec.js --reporter mocha-junit-reporter --timeout 86400000" 21 | } 22 | 23 | from terminal run command: npm test 24 | 25 | option #2: 26 | grunt regression-tests:dev -run_regression -analysis=true -entry_id=1_oorxcge2 -analyzer_ignore_alerts="'INFTagDurationDoesntMatch','BitrateOutsideOfTargetAlert','InvalidKeyFrameIntervalAlert','SegmentCountMismatchAlert'" -entry_path=1_oorxcge2-3 -result_path=reports -analyzer_root=/Users/lilach.maliniak/Desktop/dev/repositories/live-monitor -override=false 27 | 28 | comments: 29 | >>>>> option #1 uses configMapping.json ad config.json.template configuration 30 | >>>>> option #2 can use command line args and configuration 31 | 32 | */ 33 | 34 | 35 | module.exports = function (grunt) { 36 | 'use strict'; 37 | 38 | grunt.config.set('regression-tests', { 39 | 40 | dev: { 41 | options: { 42 | reporter: 'mocha-junit-reporter', 43 | /*reporterOptions: { 44 | mochaFile: './<%=reports_dir%>/last_regression_run/regression-tests-report.xml', 45 | properties: { 46 | BUILD_ID: process.env.BUILD_NUMBER, 47 | useFullSuiteTitle: true, 48 | suiteTitleSeparedBy: '.' 49 | }, 50 | },*/ 51 | require: 'chai', 52 | //require: 'mocha-junit-reporter', 53 | timeout: 86400000, 54 | captureFile: '<%=reports_dir%>/regression/regression-tests.log' 55 | }, 56 | src: ['<%=regression_tests%>'] 57 | }, 58 | ci: { 59 | options: { 60 | reporter: 'mocha-junit-reporter', 61 | /*reporterOptions: { 62 | mochaFile: './<%=reports_dir%>/last_regression_run/regression-tests-report.xml', 63 | properties: { 64 | BUILD_ID: process.env.BUILD_NUMBER, 65 | useFullSuiteTitle: true, 66 | suiteTitleSeparedBy: '.' 67 | }, 68 | },*/ 69 | //reporter: 'Xunit', 70 | require: 'chai', 71 | timeout: 86400000, 72 | quiet: true, 73 | captureFile: '<%=reports_dir%>/regression/regression-tests.log' 74 | 75 | }, 76 | src: ['<%=regression_tests%>'] 77 | }, 78 | }); 79 | }; 80 | -------------------------------------------------------------------------------- /lib/grunt/config/unit-tests-config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/12/2015. 3 | */ 4 | 5 | module.exports = function (grunt) { 6 | 'use strict'; 7 | 8 | grunt.config.set('unit-test', { 9 | dev: { 10 | options: { 11 | reporter: 'spec', 12 | require: 'chai', 13 | timeout: 10000, 14 | captureFile: '<%=reports_dir%>/unit-tests.log' 15 | }, 16 | src: ['<%=unit_tests%>'] 17 | }, 18 | ci: { 19 | options: { 20 | reporter: 'mocha-junit-reporter', 21 | require: 'chai', 22 | timeout: 10000, 23 | captureFile: '<%=reports_dir%>/unit-tests.log' 24 | }, 25 | src: ['<%=unit_tests%>'] 26 | } 27 | }); 28 | }; 29 | -------------------------------------------------------------------------------- /lib/grunt/tasks/component-test-task.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 9/2/2015. 3 | */ 4 | 5 | var fs = require('fs'); 6 | 7 | module.exports = function (grunt) { 8 | 'use strict'; 9 | 10 | grunt.registerMultiTask('component-test', 'Run unit tests', function() { 11 | 12 | 13 | var c = {}; 14 | c[this.target] = grunt.config.get('component-test')[this.target]; 15 | grunt.config.set('mochaTest',c); 16 | 17 | var reportsDir = grunt.config.get('reports_dir'); 18 | if (!fs.existsSync(reportsDir)) { 19 | fs.mkdirSync(reportsDir); 20 | } 21 | 22 | grunt.task.run('mochaTest:' + this.target); 23 | }); 24 | }; 25 | -------------------------------------------------------------------------------- /lib/grunt/tasks/recording-task.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by lilach.maliniak on 18/08/2016. 3 | */ 4 | 5 | var fs = require('fs'); 6 | 7 | module.exports = function (grunt) { 8 | 'use strict'; 9 | 10 | grunt.registerMultiTask('recording-test', 'Run HLS stream recording', function() { 11 | 12 | 13 | var c = {}; 14 | c[this.target] = grunt.config.get('recording-test')[this.target]; 15 | grunt.config.set('mochaTest',c); 16 | 17 | var reportsDir = grunt.config.get('reports_dir'); 18 | if (!fs.existsSync(reportsDir)) { 19 | fs.mkdirSync(reportsDir); 20 | } 21 | 22 | grunt.task.run('mochaTest:' + this.target); 23 | }); 24 | 25 | }; 26 | -------------------------------------------------------------------------------- /lib/grunt/tasks/regression-tests-task.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by lilach.maliniak on 02/08/2016. 3 | */ 4 | 5 | var fs = require('fs'); 6 | 7 | module.exports = function (grunt) { 8 | 'use strict'; 9 | 10 | grunt.registerMultiTask('regression-tests', 'Run regression test', function() { 11 | 12 | 13 | var c = {}; 14 | c[this.target] = grunt.config.get('regression-tests')[this.target]; 15 | grunt.config.set('mochaTest',c); 16 | 17 | var reportsDir = grunt.config.get('reports_dir'); 18 | if (!fs.existsSync(reportsDir)) { 19 | fs.mkdirSync(reportsDir); 20 | } 21 | 22 | grunt.task.run('mochaTest:' + this.target); 23 | }); 24 | 25 | }; 26 | 27 | -------------------------------------------------------------------------------- /lib/grunt/tasks/unit-tests-task.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/12/2015. 3 | */ 4 | 5 | var fs = require('fs'); 6 | 7 | module.exports = function (grunt) { 8 | 'use strict'; 9 | 10 | grunt.registerMultiTask('unit-test', 'Run unit tests', function() { 11 | 12 | 13 | var c = {}; 14 | c[this.target] = grunt.config.get('unit-test')[this.target]; 15 | grunt.config.set('mochaTest',c); 16 | 17 | var reportsDir = grunt.config.get('reports_dir'); 18 | if (!fs.existsSync(reportsDir)) { 19 | fs.mkdirSync(reportsDir); 20 | } 21 | 22 | grunt.task.run('mochaTest:' + this.target); 23 | }); 24 | }; 25 | -------------------------------------------------------------------------------- /lib/m3u8/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 TED Conferences 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /lib/m3u8/Makefile: -------------------------------------------------------------------------------- 1 | REPORTER = list 2 | 3 | test: 4 | ./node_modules/.bin/mocha \ 5 | --reporter $(REPORTER) \ 6 | test/*.js test/acceptance/*.js 7 | 8 | test-acceptance: 9 | ./node_modules/.bin/mocha \ 10 | --reporter $(REPORTER) \ 11 | test/acceptance/*.js 12 | 13 | .PHONY: test test-acceptance -------------------------------------------------------------------------------- /lib/m3u8/README.md: -------------------------------------------------------------------------------- 1 | m3u8 2 | ==== 3 | 4 | node-m3u8 is a streaming m3u8 parser tailored for dealing with [Apple's HTTP 5 | Live Streaming protocol](http://tools.ietf.org/html/draft-pantos-http-live-streaming). 6 | It may work for other m3u files, but I have not tested it for those uses. 7 | 8 | example 9 | ------- 10 | 11 | ``` js 12 | var m3u8 = require('m3u8'); 13 | var fs = require('fs'); 14 | 15 | var parser = m3u8.createStream(); 16 | var file = fs.createReadStream('/path/to/file.m3u8'); 17 | file.pipe(parser); 18 | 19 | parser.on('item', function(item) { 20 | // emits PlaylistItem, MediaItem, StreamItem, and IframeStreamItem 21 | }); 22 | parser.on('m3u', function(m3u) { 23 | // fully parsed m3u file 24 | }); 25 | ``` 26 | 27 | All items and the m3u object have `toString()` methods for conversion to m3u8. 28 | Attributes and properties have getter/setters on m3u and item objects: 29 | 30 | ``` 31 | parser.on('item', function(item) { 32 | var duration = item.get('bandwidth'); 33 | item.set('uri', 'http://example.com/' + item.get('uri')); 34 | }); 35 | ``` 36 | 37 | The M3U and Item objects are available on m3u8: 38 | ``` 39 | var m3u8 = require('m3u8'); 40 | 41 | var m3u = m3u8.M3U.create(); 42 | m3u.addPlaylistItem({ 43 | duration : 10, 44 | uri : 'file' 45 | }); 46 | ``` 47 | 48 | See tests for more usage patterns. -------------------------------------------------------------------------------- /lib/m3u8/m3u/IframeStreamItem.js: -------------------------------------------------------------------------------- 1 | var util = require('util'), 2 | StreamItem = require('./StreamItem'); 3 | 4 | var IframeStreamItem = module.exports = function IframeStreamItem(attributes) { 5 | StreamItem.apply(this, arguments); 6 | 7 | delete this.properties['uri']; 8 | }; 9 | 10 | util.inherits(IframeStreamItem, StreamItem); 11 | 12 | IframeStreamItem.create = function createIframeStreamItem(data) { 13 | var item = new IframeStreamItem(); 14 | item.setData(data); 15 | return item; 16 | }; 17 | 18 | IframeStreamItem.prototype.toString = function toString() { 19 | return '#EXT-X-I-FRAME-STREAM-INF:' + this.attributes.toString(); 20 | }; -------------------------------------------------------------------------------- /lib/m3u8/m3u/Item.js: -------------------------------------------------------------------------------- 1 | var AttributeList = require('./AttributeList'); 2 | 3 | var Item = module.exports = function Item(attributes) { 4 | this.attributes = new AttributeList(attributes); 5 | this.properties = { 6 | byteRange : null, 7 | date : null, 8 | discontinuity : null, 9 | duration : null, 10 | title : null, 11 | uri : null 12 | }; 13 | }; 14 | 15 | Item.prototype.get = function get(key) { 16 | if (this.propertiesHasKey(key)) { 17 | return this.properties[key]; 18 | } else { 19 | return this.attributes.get(key); 20 | } 21 | }; 22 | 23 | Item.prototype.set = function set(key, value) { 24 | if (this.propertiesHasKey(key)) { 25 | this.properties[key] = value; 26 | } else { 27 | this.attributes.set(key, value); 28 | } 29 | 30 | return this; 31 | }; 32 | 33 | Item.prototype.serialize = function serialize() { 34 | return { 35 | attributes : this.attributes.serialize(), 36 | properties : this.properties 37 | } 38 | }; 39 | 40 | Item.unserialize = function unserialize(constructor, object) { 41 | var item = new constructor; 42 | item.attributes = AttributeList.unserialize(object.attributes); 43 | item.properties = object.properties; 44 | return item; 45 | }; 46 | 47 | Item.prototype.setData = function setData(data) { 48 | var self = this; 49 | Object.keys(data).forEach(function(key) { 50 | self.set(key, data[key]); 51 | }); 52 | }; 53 | 54 | Item.prototype.propertiesHasKey = function hasKey(key) { 55 | return Object.keys(this.properties).indexOf(key) > -1; 56 | }; 57 | -------------------------------------------------------------------------------- /lib/m3u8/m3u/MediaItem.js: -------------------------------------------------------------------------------- 1 | var util = require('util'), 2 | Item = require('./Item'); 3 | 4 | var MediaItem = module.exports = function MediaItem(attributes) { 5 | Item.apply(this, arguments); 6 | 7 | delete this.properties['uri']; 8 | }; 9 | 10 | util.inherits(MediaItem, Item); 11 | 12 | MediaItem.create = function createMediaItem(data) { 13 | var item = new MediaItem(); 14 | item.setData(data); 15 | return item; 16 | }; 17 | 18 | MediaItem.prototype.toString = function toString() { 19 | return '#EXT-X-MEDIA:' + this.attributes.toString(); 20 | }; -------------------------------------------------------------------------------- /lib/m3u8/m3u/PlaylistItem.js: -------------------------------------------------------------------------------- 1 | var util = require('util'), 2 | Item = require('./Item'); 3 | 4 | var PlaylistItem = module.exports = function PlaylistItem() { 5 | Item.call(this); 6 | }; 7 | 8 | util.inherits(PlaylistItem, Item); 9 | 10 | PlaylistItem.create = function createPlaylistItem(data) { 11 | var item = new PlaylistItem(); 12 | item.setData(data); 13 | return item; 14 | }; 15 | 16 | PlaylistItem.prototype.toString = function toString() { 17 | var output = []; 18 | if (this.get('discontinuity')) { 19 | output.push('#EXT-X-DISCONTINUITY'); 20 | } 21 | if (this.get('date')) { 22 | var date = this.get('date'); 23 | if (date.getMonth) { 24 | date = date.toISOString(); 25 | } 26 | output.push('#EXT-X-PROGRAM-DATE-TIME:' + date); 27 | } 28 | if (this.get('duration') != null || this.get('title') != null) { 29 | output.push( 30 | '#EXTINF:' + [this.get('duration').toFixed(4), this.get('title')].join(',') 31 | ); 32 | } 33 | if (this.get('byteRange') != null) { 34 | output.push('#EXT-X-BYTERANGE:' + this.get('byteRange')); 35 | } 36 | output.push(this.get('uri')); 37 | 38 | return output.join('\n'); 39 | }; 40 | -------------------------------------------------------------------------------- /lib/m3u8/m3u/StreamItem.js: -------------------------------------------------------------------------------- 1 | var util = require('util'), 2 | Item = require('./Item'), 3 | AttributeList = require('./AttributeList'); 4 | 5 | var StreamItem = module.exports = function StreamItem(attributes) { 6 | Item.apply(this, arguments); 7 | }; 8 | 9 | util.inherits(StreamItem, Item); 10 | 11 | StreamItem.create = function createStreamItem(data) { 12 | var item = new StreamItem(); 13 | item.setData(data); 14 | return item; 15 | }; 16 | 17 | StreamItem.prototype.toString = function toString() { 18 | var output = []; 19 | output.push('#EXT-X-STREAM-INF:' + this.attributes.toString()); 20 | output.push(this.get('uri')); 21 | 22 | return output.join('\n'); 23 | }; -------------------------------------------------------------------------------- /lib/m3u8/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "m3u8", 3 | "version" : "0.0.6", 4 | "description" : "streaming m3u8 parser for Apple's HTTP Live Streaming protocol", 5 | "main" : "./parser.js", 6 | "keywords" : [ 7 | "m3u", 8 | "m3u8", 9 | "hls", 10 | "http live streaming", 11 | "stream" 12 | ], 13 | "repository" : { 14 | "type" : "git", 15 | "url" : "http://github.com/tedconf/node-m3u8.git" 16 | }, 17 | "dependencies" : { 18 | "chunked-stream": "~0.0.1" 19 | }, 20 | "devDependencies" : { 21 | "mocha" : "~1.6.0", 22 | "sinon" : "~1.5.0", 23 | "should" : "7.1.1" 24 | }, 25 | "engine" : { 26 | "node" : ">=0.6.0" 27 | }, 28 | "scripts": { 29 | "test" : "make test" 30 | }, 31 | "author" : { 32 | "name" : "Mark Bogdanoff", 33 | "email" : "bog@ted.com", 34 | "url" : "http://www.ted.com" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /lib/m3u8/test/acceptance/parse-iframe.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | 3 | describe('parsing iframe m3u8', function() { 4 | it('should emit 36 items', function(done) { 5 | var parser = getParser(); 6 | 7 | var items = 0; 8 | parser.on('item', function() { 9 | items++; 10 | }); 11 | parser.on('m3u', function() { 12 | items.should.equal(36); 13 | done(); 14 | }); 15 | }); 16 | 17 | it('should have proper headers', function(done) { 18 | var parser = getParser(); 19 | parser.on('m3u', function(m3u) { 20 | m3u.get('version').should.equal(4); 21 | m3u.get('targetDuration').should.equal(10); 22 | m3u.get('playlistType').should.equal('VOD'); 23 | m3u.get('mediaSequence').should.equal(0); 24 | m3u.get('iframesOnly').should.be.true; 25 | done(); 26 | }); 27 | }); 28 | 29 | describe('first IframeStreamItem', function() { 30 | it('should match first item in fixture', function(done) { 31 | var parser = getParser(); 32 | 33 | parser.on('m3u', function(m3u) { 34 | var item = m3u.items.PlaylistItem[0]; 35 | item.get('title').should.equal(''); 36 | item.get('duration').should.equal(5); 37 | item.get('byteRange').should.equal('376@940'); 38 | item.get('uri').should.equal('hls_1500k_video.ts'); 39 | 40 | done(); 41 | }); 42 | }); 43 | }); 44 | }); 45 | 46 | function getParser() { 47 | var parser = require('../../parser').createStream(); 48 | var variantFile = fs.createReadStream( 49 | __dirname + '/../fixtures/iframe.m3u8' 50 | ); 51 | variantFile.pipe(parser); 52 | return parser; 53 | } 54 | -------------------------------------------------------------------------------- /lib/m3u8/test/acceptance/parse-playlist.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'), 2 | should = require('should'); 3 | 4 | describe('parsing playlist m3u8', function() { 5 | it('should emit 17 items', function(done) { 6 | var parser = getParser(); 7 | 8 | var items = 0; 9 | parser.on('item', function() { 10 | items++; 11 | }); 12 | parser.on('m3u', function() { 13 | items.should.equal(17); 14 | done(); 15 | }); 16 | }); 17 | 18 | it('should have proper headers', function(done) { 19 | var parser = getParser(); 20 | parser.on('m3u', function(m3u) { 21 | m3u.get('version').should.equal(4); 22 | m3u.get('targetDuration').should.equal(10); 23 | m3u.get('playlistType').should.equal('VOD'); 24 | m3u.get('mediaSequence').should.equal(0); 25 | should.not.exist(m3u.get('iframesOnly')); 26 | done(); 27 | }); 28 | }); 29 | 30 | describe('first PlaylistItem', function() { 31 | it('should match first item in fixture', function(done) { 32 | var parser = getParser(); 33 | 34 | parser.on('m3u', function(m3u) { 35 | var item = m3u.items.PlaylistItem[0]; 36 | item.get('title').should.equal(''); 37 | item.get('duration').should.equal(10); 38 | item.get('byteRange').should.equal('522828@0'); 39 | item.get('uri').should.equal('hls_450k_video.ts'); 40 | 41 | done(); 42 | }); 43 | }); 44 | }); 45 | }); 46 | 47 | function getParser() { 48 | var parser = require('../../parser').createStream(); 49 | var variantFile = fs.createReadStream( 50 | __dirname + '/../fixtures/playlist.m3u8' 51 | ); 52 | variantFile.pipe(parser); 53 | return parser; 54 | } 55 | -------------------------------------------------------------------------------- /lib/m3u8/test/acceptance/parse-variant.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | 3 | describe('parsing variant m3u8', function() { 4 | it('should emit 16 items', function(done) { 5 | var parser = getParser(); 6 | 7 | var items = 0; 8 | parser.on('item', function() { 9 | items++; 10 | }); 11 | parser.on('m3u', function() { 12 | items.should.equal(16); 13 | done(); 14 | }); 15 | }); 16 | 17 | it('should have version 4', function(done) { 18 | var parser = getParser(); 19 | parser.on('m3u', function(m3u) { 20 | m3u.get('version').should.equal(4); 21 | done(); 22 | }); 23 | }); 24 | 25 | describe('first StreamItem', function() { 26 | it('should match first stream item in fixture', function(done) { 27 | var parser = getParser(); 28 | 29 | parser.on('m3u', function(m3u) { 30 | var item = m3u.items.StreamItem[0]; 31 | item.get('bandwidth').should.equal(69334); 32 | item.get('program-id').should.equal(1); 33 | item.get('codecs').should.equal('avc1.42c00c'); 34 | item.get('resolution')[0].should.equal(320); 35 | item.get('resolution')[1].should.equal(180); 36 | item.get('audio').should.equal('600k'); 37 | 38 | done(); 39 | }); 40 | }); 41 | }); 42 | 43 | describe('first IframeStreamItem', function() { 44 | it('should match first iframe stream item in fixture', function(done) { 45 | var parser = getParser(); 46 | 47 | parser.on('m3u', function(m3u) { 48 | var item = m3u.items.IframeStreamItem[0]; 49 | item.get('bandwidth').should.equal(28361); 50 | item.get('uri').should.equal('hls_64k_iframe.m3u8'); 51 | 52 | done(); 53 | }); 54 | }); 55 | }); 56 | 57 | describe('first MediaItem', function() { 58 | it('should match first media item in fixture', function(done) { 59 | var parser = getParser(); 60 | 61 | parser.on('m3u', function(m3u) { 62 | var item = m3u.items.MediaItem[0]; 63 | item.get('group-id').should.equal('600k'); 64 | item.get('language').should.equal('eng'); 65 | item.get('name').should.equal('Audio'); 66 | item.get('autoselect').should.be.true; 67 | item.get('default').should.be.true; 68 | item.get('uri').should.equal('hls_600k_audio.m3u8'); 69 | item.get('type').should.equal('AUDIO'); 70 | 71 | done(); 72 | }); 73 | }); 74 | }); 75 | }); 76 | 77 | function getParser() { 78 | var parser = require('../../parser').createStream(); 79 | var variantFile = fs.createReadStream( 80 | __dirname + '/../fixtures/variant.m3u8' 81 | ); 82 | variantFile.pipe(parser); 83 | return parser; 84 | } -------------------------------------------------------------------------------- /lib/m3u8/test/attributelist.test.js: -------------------------------------------------------------------------------- 1 | var AttributeList = require('../m3u/AttributeList'), 2 | should = require('should'); 3 | 4 | describe('AttributeList', function() { 5 | describe('#mergeAttributes', function() { 6 | it('should merge in attributes', function() { 7 | var list = createAttributeList(); 8 | list.mergeAttributes([{ key: 'forced', value: true }]); 9 | 10 | list.get('bandwidth').should.eql(1); 11 | list.get('forced').should.be.true; 12 | }); 13 | }); 14 | 15 | describe('#set', function() { 16 | it('should set coerce and set attributes', function() { 17 | var list = createAttributeList(); 18 | 19 | list.attributes.bandwidth.should.equal(1); 20 | }); 21 | }); 22 | 23 | describe('#get', function() { 24 | it('should get attribute', function() { 25 | var list = createAttributeList(); 26 | 27 | list.get('bandwidth').should.eql(1); 28 | }); 29 | }); 30 | 31 | describe('#getCoerced', function() { 32 | it('should get attribute value ready to be written out', function() { 33 | var list = createAttributeList(); 34 | 35 | list.getCoerced('audio').should.eql('"hello"'); 36 | }); 37 | }); 38 | 39 | describe('#serialize', function() { 40 | it('should return the attributs object', function() { 41 | var list = createAttributeList(); 42 | list.serialize().should.eql(list.attributes); 43 | }); 44 | }); 45 | 46 | describe('unserialize', function() { 47 | it('should populate the attributes object', function() { 48 | var data = { 49 | bandwidth: 1, 50 | audio: 'hello' 51 | }; 52 | var list = AttributeList.unserialize(data); 53 | data.should.eql(list.attributes); 54 | list.should.be.instanceof(AttributeList); 55 | }); 56 | }); 57 | }); 58 | 59 | function createAttributeList() { 60 | var list = new AttributeList; 61 | list.set('bandwidth', 1); 62 | list.attributes.audio = 'hello'; 63 | return list; 64 | } -------------------------------------------------------------------------------- /lib/m3u8/test/fixtures/iframe.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-TARGETDURATION:10 3 | #EXT-X-VERSION:4 4 | #EXT-X-MEDIA-SEQUENCE:0 5 | #EXT-X-PLAYLIST-TYPE:VOD 6 | #EXT-X-I-FRAMES-ONLY 7 | #EXTINF:5, 8 | #EXT-X-BYTERANGE:376@940 9 | hls_1500k_video.ts 10 | #EXTINF:4.9583, 11 | #EXT-X-BYTERANGE:134420@1644436 12 | hls_1500k_video.ts 13 | #EXTINF:4.9583, 14 | #EXT-X-BYTERANGE:13348@2391548 15 | hls_1500k_video.ts 16 | #EXTINF:4.9583, 17 | #EXT-X-BYTERANGE:44744@2708704 18 | hls_1500k_video.ts 19 | #EXTINF:4.9583, 20 | #EXT-X-BYTERANGE:76328@4934624 21 | hls_1500k_video.ts 22 | #EXTINF:4.9583, 23 | #EXT-X-BYTERANGE:37224@5553896 24 | hls_1500k_video.ts 25 | #EXTINF:4.9583, 26 | #EXT-X-BYTERANGE:40044@6203436 27 | hls_1500k_video.ts 28 | #EXTINF:4.9583, 29 | #EXT-X-BYTERANGE:21620@6780784 30 | hls_1500k_video.ts 31 | #EXTINF:4.9583, 32 | #EXT-X-BYTERANGE:22372@7135352 33 | hls_1500k_video.ts 34 | #EXTINF:0.66667, 35 | #EXT-X-BYTERANGE:24064@7516240 36 | hls_1500k_video.ts 37 | #EXTINF:4.9583, 38 | #EXT-X-BYTERANGE:83096@7590876 39 | hls_1500k_video.ts 40 | #EXTINF:1.2083, 41 | #EXT-X-BYTERANGE:103964@9978664 42 | hls_1500k_video.ts 43 | #EXTINF:5, 44 | #EXT-X-BYTERANGE:36848@10439452 45 | hls_1500k_video.ts 46 | #EXTINF:4.9583, 47 | #EXT-X-BYTERANGE:33276@10531760 48 | hls_1500k_video.ts 49 | #EXTINF:4.9583, 50 | #EXT-X-BYTERANGE:22560@10668436 51 | hls_1500k_video.ts 52 | #EXTINF:4.9583, 53 | #EXT-X-BYTERANGE:26132@11119824 54 | hls_1500k_video.ts 55 | #EXTINF:1.5, 56 | #EXT-X-BYTERANGE:30456@11651112 57 | hls_1500k_video.ts 58 | #EXTINF:4.9583, 59 | #EXT-X-BYTERANGE:101144@11847948 60 | hls_1500k_video.ts 61 | #EXTINF:4.9583, 62 | #EXT-X-BYTERANGE:95128@15011236 63 | hls_1500k_video.ts 64 | #EXTINF:4.9583, 65 | #EXT-X-BYTERANGE:71440@15891640 66 | hls_1500k_video.ts 67 | #EXTINF:4.9583, 68 | #EXT-X-BYTERANGE:16732@16631608 69 | hls_1500k_video.ts 70 | #EXTINF:4.9583, 71 | #EXT-X-BYTERANGE:18612@17076980 72 | hls_1500k_video.ts 73 | #EXTINF:4.9583, 74 | #EXT-X-BYTERANGE:24816@17525360 75 | hls_1500k_video.ts 76 | #EXTINF:4.9583, 77 | #EXT-X-BYTERANGE:84036@17995172 78 | hls_1500k_video.ts 79 | #EXTINF:4.9583, 80 | #EXT-X-BYTERANGE:94188@20472448 81 | hls_1500k_video.ts 82 | #EXTINF:4.9583, 83 | #EXT-X-BYTERANGE:68056@21629212 84 | hls_1500k_video.ts 85 | #EXTINF:0.70833, 86 | #EXT-X-BYTERANGE:76516@22618468 87 | hls_1500k_video.ts 88 | #EXTINF:4.9583, 89 | #EXT-X-BYTERANGE:21244@22847828 90 | hls_1500k_video.ts 91 | #EXTINF:4.9583, 92 | #EXT-X-BYTERANGE:17108@23288500 93 | hls_1500k_video.ts 94 | #EXTINF:4.9583, 95 | #EXT-X-BYTERANGE:22372@23721464 96 | hls_1500k_video.ts 97 | #EXTINF:4.9583, 98 | #EXT-X-BYTERANGE:22936@24190712 99 | hls_1500k_video.ts 100 | #EXTINF:4.9583, 101 | #EXT-X-BYTERANGE:27448@24687596 102 | hls_1500k_video.ts 103 | #EXTINF:5, 104 | #EXT-X-BYTERANGE:32712@25232420 105 | hls_1500k_video.ts 106 | #EXTINF:5, 107 | #EXT-X-BYTERANGE:41924@25853196 108 | hls_1500k_video.ts 109 | #EXTINF:4.4583, 110 | #EXT-X-BYTERANGE:60348@26581696 111 | hls_1500k_video.ts 112 | #EXTINF:3.875, 113 | #EXT-X-BYTERANGE:59972@27279364 114 | hls_1500k_video.ts 115 | #EXT-X-ENDLIST 116 | -------------------------------------------------------------------------------- /lib/m3u8/test/fixtures/playlist.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-TARGETDURATION:10 3 | #EXT-X-VERSION:4 4 | #EXT-X-MEDIA-SEQUENCE:0 5 | #EXT-X-PLAYLIST-TYPE:VOD 6 | #EXTINF:10, 7 | #EXT-X-BYTERANGE:522828@0 8 | hls_450k_video.ts 9 | #EXTINF:10, 10 | #EXT-X-BYTERANGE:587500@522828 11 | hls_450k_video.ts 12 | #EXTINF:10, 13 | #EXT-X-BYTERANGE:713084@1110328 14 | hls_450k_video.ts 15 | #EXTINF:10, 16 | #EXT-X-BYTERANGE:476580@1823412 17 | hls_450k_video.ts 18 | #EXTINF:10, 19 | #EXT-X-BYTERANGE:535612@2299992 20 | hls_450k_video.ts 21 | #EXTINF:10, 22 | #EXT-X-BYTERANGE:207176@2835604 23 | hls_450k_video.ts 24 | #EXTINF:10, 25 | #EXT-X-BYTERANGE:455900@3042780 26 | hls_450k_video.ts 27 | #EXTINF:10, 28 | #EXT-X-BYTERANGE:657248@3498680 29 | hls_450k_video.ts 30 | #EXTINF:10, 31 | #EXT-X-BYTERANGE:571708@4155928 32 | hls_450k_video.ts 33 | #EXTINF:10, 34 | #EXT-X-BYTERANGE:485040@4727636 35 | hls_450k_video.ts 36 | #EXTINF:10, 37 | #EXT-X-BYTERANGE:709136@5212676 38 | hls_450k_video.ts 39 | #EXTINF:10, 40 | #EXT-X-BYTERANGE:730004@5921812 41 | hls_450k_video.ts 42 | #EXTINF:10, 43 | #EXT-X-BYTERANGE:456276@6651816 44 | hls_450k_video.ts 45 | #EXTINF:10, 46 | #EXT-X-BYTERANGE:468684@7108092 47 | hls_450k_video.ts 48 | #EXTINF:10, 49 | #EXT-X-BYTERANGE:444996@7576776 50 | hls_450k_video.ts 51 | #EXTINF:10, 52 | #EXT-X-BYTERANGE:331444@8021772 53 | hls_450k_video.ts 54 | #EXTINF:1.4167, 55 | #EXT-X-BYTERANGE:44556@8353216 56 | hls_450k_video.ts 57 | #EXT-X-ENDLIST 58 | -------------------------------------------------------------------------------- /lib/m3u8/test/fixtures/variant.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-VERSION:4 3 | #EXT-X-STREAM-INF:BANDWIDTH=69334, PROGRAM-ID=1, CODECS="avc1.42c00c", RESOLUTION=320x180, AUDIO="600k" 4 | hls_64k_video.m3u8 5 | #EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=28361, PROGRAM-ID=1, CODECS="avc1.42c00c", RESOLUTION=320x180, URI="hls_64k_iframe.m3u8" 6 | #EXT-X-STREAM-INF:BANDWIDTH=223946, PROGRAM-ID=1, CODECS="avc1.42c015", RESOLUTION=512x288, AUDIO="600k" 7 | hls_180k_video.m3u8 8 | #EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=60590, PROGRAM-ID=1, CODECS="avc1.42c015", RESOLUTION=512x288, URI="hls_180k_iframe.m3u8" 9 | #EXT-X-STREAM-INF:BANDWIDTH=416458, PROGRAM-ID=1, CODECS="avc1.42c015", RESOLUTION=512x288, AUDIO="600k" 10 | hls_320k_video.m3u8 11 | #EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=99264, PROGRAM-ID=1, CODECS="avc1.42c015", RESOLUTION=512x288, URI="hls_320k_iframe.m3u8" 12 | #EXT-X-STREAM-INF:BANDWIDTH=584003, PROGRAM-ID=1, CODECS="avc1.42c015", RESOLUTION=512x288, AUDIO="600k" 13 | hls_450k_video.m3u8 14 | #EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=135360, PROGRAM-ID=1, CODECS="avc1.42c015", RESOLUTION=512x288, URI="hls_450k_iframe.m3u8" 15 | #EXT-X-STREAM-INF:BANDWIDTH=775914, PROGRAM-ID=1, CODECS="avc1.42c01e", RESOLUTION=640x360, AUDIO="600k" 16 | hls_600k_video.m3u8 17 | #EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=193371, PROGRAM-ID=1, CODECS="avc1.42c01e", RESOLUTION=640x360, URI="hls_600k_iframe.m3u8" 18 | #EXT-X-STREAM-INF:BANDWIDTH=1458339, PROGRAM-ID=1, CODECS="avc1.4d401f", RESOLUTION=853x480, AUDIO="600k" 19 | hls_950k_video.m3u8 20 | #EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=328731, PROGRAM-ID=1, CODECS="avc1.4d401f", RESOLUTION=853x480, URI="hls_950k_iframe.m3u8" 21 | #EXT-X-STREAM-INF:BANDWIDTH=3244393, PROGRAM-ID=1, CODECS="avc1.640028", RESOLUTION=1280x720, AUDIO="600k" 22 | hls_1500k_video.m3u8 23 | #EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=688313, PROGRAM-ID=1, CODECS="avc1.640028", RESOLUTION=1280x720, URI="hls_1500k_iframe.m3u8" 24 | #EXT-X-STREAM-INF:BANDWIDTH=67155, PROGRAM-ID=1, CODECS="mp4a.40.2" 25 | hls_600k_audio.m3u8 26 | #EXT-X-MEDIA:GROUP-ID="600k", LANGUAGE="eng", NAME="Audio", AUTOSELECT=YES, DEFAULT=YES, URI="hls_600k_audio.m3u8", TYPE=AUDIO 27 | -------------------------------------------------------------------------------- /lib/m3u8/test/item.test.js: -------------------------------------------------------------------------------- 1 | var AttributeList = require('../m3u/AttributeList'), 2 | Item = require('../m3u/Item'), 3 | sinon = require('sinon'), 4 | should = require('should'); 5 | 6 | describe('Item', function() { 7 | describe('#set', function() { 8 | it('should set property on item if a property', function() { 9 | var item = new Item; 10 | 11 | item.set('title', 'hello there'); 12 | item.properties.title.should.eql('hello there'); 13 | }); 14 | it('should set an attribute if not a property', function() { 15 | var item = new Item; 16 | 17 | item.set('autoselect', true); 18 | item.attributes.get('autoselect').should.be.true; 19 | }); 20 | }); 21 | 22 | describe('#get', function() { 23 | it('should get property from item if a property', function() { 24 | var item = new Item; 25 | 26 | item.properties.uri = '/path'; 27 | item.get('uri').should.eql('/path'); 28 | }); 29 | it('should get property from AttributeList if not a property', function() { 30 | var item = new Item; 31 | 32 | item.attributes.set('autoselect', true); 33 | item.get('autoselect').should.be.true; 34 | }); 35 | }); 36 | 37 | describe('#setData', function() { 38 | it('should set multiple properties/attributes', function() { 39 | var item = new Item; 40 | item.setData({ autoselect: true, uri: '/path' }); 41 | 42 | item.get('autoselect').should.be.true; 43 | item.get('uri').should.eql('/path'); 44 | }); 45 | }); 46 | 47 | describe('#serialize', function() { 48 | it('should return an object containing properties and attributes', function() { 49 | var item = new Item; 50 | item.setData({ 51 | autoselect: true, 52 | uri: '/path' 53 | }); 54 | var data = item.serialize(); 55 | data.attributes.should.eql(item.attributes.serialize()); 56 | data.properties.should.eql(item.properties); 57 | }); 58 | }); 59 | 60 | describe('unserialize', function() { 61 | it('should return an Item object with attributes and properties', function() { 62 | var list = new AttributeList; 63 | list.set('autoselect', true); 64 | var data = { 65 | attributes: list.serialize(), 66 | properties: { url: '/path' } 67 | }; 68 | var item = Item.unserialize(Item, data); 69 | item.attributes.should.eql(list); 70 | item.properties.should.eql(data.properties); 71 | item.should.be.instanceof(Item); 72 | }); 73 | }); 74 | }); 75 | -------------------------------------------------------------------------------- /lib/manifest/promise-m3u8.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/23/2015. 3 | */ 4 | 5 | var m3u8Parser = require('../m3u8'); 6 | var fs = require('fs'); 7 | var Q = require('q'); 8 | 9 | module.exports.M3U = m3u8Parser.M3U; 10 | 11 | /** 12 | * 13 | * @param {string or stream} manifestStringOrStream - either a stream containing the M3U8 content or a string. 14 | * In case a string is supplied without options it is considered to be the path to a local M3U8 file. 15 | * @param {object} options - in case the 'verbatim' property is set to true, the string passed in manifestStringOrStream 16 | * will be taken verbatim (i.e. as the content for the M3U8) 17 | * @returns {*|promise} 18 | */ 19 | module.exports.parseM3U8 = function(manifestStringOrStream, options){ 20 | 21 | var getStreamFromInput = function(manifestStringOrStream){ 22 | var d = Q.defer(); 23 | 24 | if (manifestStringOrStream.pipe) 25 | { 26 | d.resolve(manifestStringOrStream); 27 | } 28 | else 29 | { 30 | if (options && options.verbatim) 31 | { 32 | // Use value verbatim 33 | var Readable = require('stream').Readable; 34 | var s = new Readable(); 35 | s.push(manifestStringOrStream); 36 | s.push(null); // Stream end 37 | d.resolve(s); 38 | } 39 | else 40 | { 41 | // Value is the M3U8 path 42 | var stream = fs.createReadStream(manifestStringOrStream); 43 | 44 | // This will wait until we know the readable stream is actually valid before piping 45 | stream.on('readable', function () { 46 | d.resolve(stream); 47 | }); 48 | 49 | // This catches any errors that happen while creating the readable stream (usually invalid names) 50 | stream.on('error', function(err) { 51 | var newErr = new Error("Error reading file " + manifestStringOrStream + " when trying to parse m3u8" + "\n" + err); 52 | d.reject(newErr); 53 | }); 54 | } 55 | } 56 | return d.promise; 57 | }; 58 | 59 | var d = Q.defer(); 60 | 61 | var streamPromise = getStreamFromInput(manifestStringOrStream); 62 | streamPromise.then(function(stream){ 63 | var parser = m3u8Parser.createStream(); 64 | parser.on('m3u', function(m3u) 65 | { 66 | d.resolve(m3u); 67 | }); 68 | 69 | parser.on('error', function(error) 70 | { 71 | d.reject(error); 72 | }); 73 | stream.pipe(parser); 74 | 75 | }, function(err){ 76 | d.reject(err); 77 | }); 78 | 79 | return d.promise; 80 | }; 81 | -------------------------------------------------------------------------------- /lib/mocks/BackendClientFileBasedMock.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 9/7/2015. 3 | */ 4 | 5 | var qio = require('q-io/fs'); 6 | var path = require('path'); 7 | 8 | module.exports = { 9 | getLiveEntriesForMediaServer : function(){ 10 | return qio.read(path.join(__dirname, 'mockBackendResults.json')).then(function(jsonString){ 11 | return JSON.parse(jsonString); 12 | }); 13 | } 14 | }; -------------------------------------------------------------------------------- /lib/mocks/NetworkClientMock.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/24/2015. 3 | */ 4 | 5 | var Q = require('q'); 6 | var sinon = require('sinon'); 7 | 8 | module.exports = { 9 | read : sinon.stub().returns(Q()) 10 | }; -------------------------------------------------------------------------------- /lib/mocks/mockBackendResults.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "entryId": "0_64w3gwx6", 4 | "dvrWindow": 7200 5 | }, 6 | { 7 | "entryId": "0_18zfyfs8", 8 | "dvrWindow": 7200 9 | } 10 | ] -------------------------------------------------------------------------------- /lib/playlistGenerator/BroadcastEventEmitter.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var playlistUtils = require('./playlistGen-utils'); 3 | 4 | 5 | function BroadcastEventEmitter(){ 6 | } 7 | 8 | BroadcastEventEmitter.prototype.addListener = function(eventName,listener){ 9 | if(typeof eventName === 'string'){ 10 | var arr = this[eventName]; 11 | if(arr){ 12 | var index = _.indexOf(arr,listener); 13 | if(index < 0) { 14 | arr.push(listener) 15 | } 16 | } else { 17 | this[eventName] = [listener]; 18 | } 19 | } else { 20 | if(eventName !== this) { 21 | this.listenerCb = eventName; 22 | } 23 | } 24 | }; 25 | 26 | 27 | BroadcastEventEmitter.prototype.removeListener = function(eventName,listener){ 28 | if(typeof eventName === 'string' && this[eventName] ){ 29 | var arr = this[eventName]; 30 | var index = _.indexOf(arr,listener); 31 | if(index >= 0) { 32 | arr.splice(index,1); 33 | } 34 | if(arr.length === 0){ 35 | delete this[eventName]; 36 | } 37 | } else if(this.listenerCb === listener) { 38 | delete this.listenerCb; 39 | } 40 | }; 41 | 42 | // NB: arguments passed as *this* from outer scope 43 | var applyEvent = function(listenerCb){ 44 | listenerCb.handleEvent.apply(listenerCb,this); 45 | }; 46 | 47 | BroadcastEventEmitter.prototype.emit = function(){ 48 | if(this[arguments[0]]){ 49 | _.each(this[arguments[0]], applyEvent,arguments); 50 | } else if(this.listenerCb) { 51 | this.listenerCb.handleEvent.apply(this.listenerCb,arguments); 52 | } 53 | }; 54 | 55 | BroadcastEventEmitter.prototype.handleEvent = function () { 56 | this.emit.apply(this,arguments); 57 | }; 58 | 59 | module.exports = BroadcastEventEmitter; -------------------------------------------------------------------------------- /lib/playlistGenerator/PlaylistItem.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var util = require('util'); 3 | var BroadcastEventEmitter = require('./BroadcastEventEmitter'); 4 | var playlistUtils = require('./playlistGen-utils'); 5 | 6 | /* 7 | base class for playlist items 8 | provides interface used to serialize to and from JSON string 9 | and validation method 10 | * */ 11 | 12 | function PlaylistItem(logger, playlistObj, inner) { 13 | 14 | BroadcastEventEmitter.prototype.constructor.call(this); 15 | 16 | this.inner = inner || {}; 17 | this.logger = logger; 18 | this.playlist = playlistObj; 19 | 20 | if(inner){ 21 | this.onUnserialize(); 22 | if(!this.validate()){ 23 | throw new Error('serialization error'); 24 | } 25 | } 26 | } 27 | 28 | util.inherits(PlaylistItem,BroadcastEventEmitter); 29 | 30 | PlaylistItem.prototype.onUnserialize = function(){ 31 | }; 32 | 33 | PlaylistItem.prototype.isInitialized = function () { 34 | return _.isObject(this.inner) && _.keys(this.inner).length > 0; 35 | }; 36 | 37 | PlaylistItem.prototype.doValidate = function(){ 38 | if(!this.logger){ 39 | return false; 40 | } 41 | if(!this.playlist){ 42 | this.logger.warn("PlaylistItem.doValidate !this.playlist"); 43 | return false; 44 | } 45 | if(!this.inner){ 46 | this.logger.warn("PlaylistItem.doValidate !this.inner"); 47 | return false; 48 | } 49 | return true; 50 | }; 51 | 52 | PlaylistItem.prototype.validate = function(){ 53 | return playlistUtils.playlistConfig.debug ? this.doValidate(playlistUtils.playlistConfig) : true; 54 | }; 55 | 56 | PlaylistItem.prototype.toJSON = function(){ 57 | return this.inner; 58 | }; 59 | 60 | module.exports = PlaylistItem; 61 | -------------------------------------------------------------------------------- /lib/playlistGenerator/SearchIndex.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by igors on 24/11/2016. 3 | */ 4 | 5 | const _ = require('underscore') 6 | const path = require ('path') 7 | const loggerModule = require('../../common/logger') 8 | 9 | class SearchIndex { 10 | constructor(loggerInfo){ 11 | this.map = new Map(); 12 | this.logger = loggerModule.getLogger("SearchIndex", loggerInfo); 13 | } 14 | 15 | getItemHash(item){ 16 | return path.basename(item) 17 | } 18 | 19 | add(item){ 20 | item = this.getItemHash(item) 21 | let index = this.map.get(item) || 0 22 | this.map.set(item,++index); 23 | this.logger.trace("add.item %s => %d",item,index); 24 | } 25 | 26 | remove(item){ 27 | item = this.getItemHash(item) 28 | let index = this.map.get(item) 29 | if(_.isNumber(index) && --index > 0) { 30 | this.logger.trace("remove.item %s => %d",item,index); 31 | this.map.set(item, index); 32 | } else { 33 | this.logger.trace("remove.item %s => 0",item); 34 | this.map.delete(item); 35 | } 36 | } 37 | has (item){ 38 | item = this.getItemHash(item) 39 | this.logger.trace("has.item %s",item); 40 | return this.map.has(item); 41 | } 42 | } 43 | 44 | module.exports = SearchIndex; 45 | -------------------------------------------------------------------------------- /lib/playlistGenerator/ValueHolder.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by igors on 5/30/16. 3 | */ 4 | var _ = require('underscore'); 5 | var util = require('util'); 6 | var BroadcastEventEmitter = require('./BroadcastEventEmitter'); 7 | var playlistUtils = require('./playlistGen-utils'); 8 | 9 | // this class turns a value type - string, number into object 10 | // it can be passed by reference 11 | // it can be printed or json'd transparently 12 | // triggers event when modified 13 | module.exports = ValueHolder = function ValueHolder (val){ 14 | this.value = val; 15 | }; 16 | 17 | util.inherits(ValueHolder,BroadcastEventEmitter); 18 | 19 | Object.defineProperty(ValueHolder.prototype , "value", { 20 | get: function get_Value() { 21 | return this._value; 22 | }, 23 | set: function set_Value(val) { 24 | if(this._value != val) { 25 | this._value = val; 26 | this.emit(playlistUtils.ClipEvents.value_changed, this._value); 27 | } 28 | } 29 | }); 30 | 31 | ValueHolder.prototype.valueOf = function () { 32 | return this.value; 33 | }; 34 | 35 | ValueHolder.prototype.toJSON = function () { 36 | return this._value; 37 | }; 38 | 39 | -------------------------------------------------------------------------------- /lib/utils/error-utils.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by AsherS on 9/2/15. 3 | */ 4 | 5 | var _ = require('underscore'); 6 | 7 | var ErrorUtils = (function(){ 8 | 9 | var obj = {}; 10 | 11 | obj.error2string=function (err) { 12 | return _.isObject(err) ? "Message: " + err.message + "; Stack: " + err.stack : err; 13 | }; 14 | obj.aggregateErrors = function(promises) { 15 | 16 | var msg = ""; 17 | var numErrors = 0; 18 | 19 | _.chain(promises) 20 | .filter(function(p) { 21 | return p.state === "rejected"; 22 | }).forEach(function (rejectedPromise) { 23 | numErrors += 1; 24 | var reason = rejectedPromise.reason; 25 | if (reason instanceof Error) { 26 | msg += "Error " + numErrors + ": " + reason.message + " Stacktrace:\n" + reason.stack + "\n"; 27 | } 28 | else { 29 | msg += "Error " + numErrors + ": " + reason + '\n'; 30 | } 31 | }); 32 | 33 | var retVal = { 34 | numErrors: numErrors 35 | }; 36 | if (numErrors > 0) { 37 | retVal.err = new Error(msg); 38 | } 39 | return retVal; 40 | }; 41 | return obj; 42 | })(); 43 | 44 | module.exports = ErrorUtils; 45 | 46 | -------------------------------------------------------------------------------- /lib/utils/fs-utils.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 1/18/2016. 3 | */ 4 | 5 | var logger = require('../../common/logger').getLogger('fs-utils'); 6 | var config = require('./../../common/Configuration'); 7 | var persistenceFormat = require('./../../common/PersistenceFormat'); 8 | var _ = require('underscore'); 9 | var qio = require('q-io/fs'); 10 | var Q = require('q'); 11 | var path = require('path'); 12 | var mkdirp = require('mkdirp'); 13 | var fs = require('fs'); 14 | var errorUtils = require('./../utils/error-utils'); 15 | var util=require('util'); 16 | 17 | 18 | var sessionId = Math.ceil(Math.random()*10000000); 19 | var tempNumber = 0; 20 | var pid = process.pid; 21 | let oldContentFolderPath = config.get("oldContentFolderPath"); 22 | var mkdirFunc = Q.denodeify(mkdirp); 23 | 24 | //create archived content folder 25 | mkdirFunc(oldContentFolderPath); 26 | 27 | function writeFileAtomically(targetPath, content){ 28 | let t0=new Date(); 29 | let tempPath=util.format("%s.%s.%d.%d.tmp",targetPath,pid,sessionId,tempNumber); 30 | tempNumber=(tempNumber+1) % 10000000; 31 | return qio.write(tempPath, content).then(function () { 32 | let t1=new Date(); 33 | return qio.move(tempPath, targetPath).then( ()=>{ 34 | let t2=new Date(); 35 | logger.debug("Saving %s took %d ms (%d + %d)",targetPath,t2-t0, t1-t0,t2-t1); 36 | }) 37 | }); 38 | } 39 | 40 | function cleanFolder(targetPath, newName) { 41 | // 1. Remove folder if exists 42 | return qio.isDirectory(targetPath).then(function (res) { 43 | if (res) { 44 | logger.debug("Removing directory " + targetPath); 45 | return persistenceFormat.createHierarchyPath(oldContentFolderPath, "entry", newName) 46 | .then(({fullPath}) => { 47 | let renamedFolderName = path.join(fullPath, newName + '_' + (new Date()).getTime().toString()); 48 | return qio.rename(targetPath, renamedFolderName); 49 | }); 50 | } 51 | }).then(function () { 52 | // 2. Create (clean) folder 53 | logger.debug("[%s] Creating directory: %s", newName, targetPath); 54 | return mkdirFunc(targetPath); 55 | }); 56 | } 57 | 58 | function existsAndNonZero(path) { 59 | return qio.stat(path) 60 | .then(function(file) { 61 | return !(file.size === 0); 62 | }, function() { 63 | return false; 64 | }); 65 | } 66 | 67 | module.exports = { 68 | writeFileAtomically : writeFileAtomically, 69 | cleanFolder : cleanFolder, 70 | existsAndNonZero : existsAndNonZero 71 | }; -------------------------------------------------------------------------------- /lib/utils/math-utils.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by lilach.maliniak on 21/08/2017. 3 | */ 4 | 5 | /* convert time in msec to other time unit, return a string with fixed digits after decimal point 6 | * @value: duration of time in milliseconds 7 | * @numFixed: number of digits after the decimal point 8 | * @divisor: number to divide the value to get desired units 9 | */ 10 | const SEC_IN_MSEC = 1000; 11 | const MINUTE_IN_MSEC = 60000; 12 | const HOUR_IN_MSEC = 3600000; 13 | 14 | function durationToStringWithUnitsConversion(value, numFixed, divisor=1) { 15 | 16 | let convertedNumber = (value / divisor).toFixed(numFixed); 17 | return `${convertedNumber}`; 18 | } 19 | 20 | module.exports = { 21 | durationToString: durationToStringWithUnitsConversion, 22 | HOUR_IN_MSEC: HOUR_IN_MSEC, 23 | MINUTE_IN_MSEC: MINUTE_IN_MSEC, 24 | HOUR_IN_MSEC: HOUR_IN_MSEC 25 | 26 | } -------------------------------------------------------------------------------- /lib/utils/mp4-utils.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by igors on 12/09/2016. 3 | */ 4 | var fs = require('fs'); 5 | var Q = require('q'); 6 | var loggerModule = require('../../common/logger'); 7 | var errorUtils = require('./../utils/error-utils'); 8 | const _ = require('underscore') 9 | 10 | const logger = loggerModule.getLogger("mp4-utils"); 11 | 12 | const comentStr = 'cmt'; //'�cmt'; 13 | 14 | 15 | class DummyFileInfo { 16 | saveAsTS(){ 17 | return Q.resolve(); 18 | } 19 | } 20 | 21 | function handleError (def,err){ 22 | if(err instanceof DummyFileInfo) { 23 | def.resolve(err); 24 | } else { 25 | logger.warn("handleError %j", errorUtils.error2string(err)); 26 | def.reject(err); 27 | } 28 | }; 29 | 30 | function finalize(def,err,fd){ 31 | if(fd) { 32 | fs.close(fd, () => { 33 | handleError(def,err); 34 | }); 35 | } else { 36 | handleError(def,err); 37 | } 38 | 39 | }; 40 | 41 | var extractMetadata = function (filePath) { 42 | let def = Q.defer(); 43 | fs.open(filePath, 'r', function (err, fd) { 44 | if (err) { 45 | finalize(def, err); 46 | } else { 47 | fs.fstat(fd, function (err, stats) { 48 | if (err) { 49 | finalize(def, err,fd); 50 | } else { 51 | let buf = new Buffer(1024 * 4); 52 | 53 | fs.read(fd, buf, 0, buf.length, stats.size - buf.length, (err, bytesRead, buffer) => { 54 | try { 55 | if (err) 56 | throw err; 57 | // read metadata table 58 | var pos = buf.indexOf(comentStr); 59 | if (pos < 0) 60 | throw new Error('fileinfo object not found'); 61 | pos += comentStr.length; 62 | let dataLength = buf.readInt32BE(pos) - 'data'.length - 12; //size 63 | pos += 'data'.length; 64 | pos += 12; 65 | let str = buf.toString('utf-8', pos, pos + dataLength); 66 | var fileInfo = _.create(DummyFileInfo.prototype,JSON.parse(str)); 67 | fileInfo.path = filePath; 68 | logger.trace("readMetadataFromMP4File %j", fileInfo); 69 | finalize(def,fileInfo,fd) 70 | } catch (err) { 71 | finalize(def, err,fd); 72 | } 73 | }); 74 | } 75 | }); 76 | } 77 | }); 78 | return def.promise; 79 | }; 80 | 81 | module.exports = { 82 | extractMetadata: extractMetadata 83 | }; -------------------------------------------------------------------------------- /liveRecorder/Config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaltura/liveDVR/0f12e06e7ab139cbce4313ddc289311c0056f468/liveRecorder/Config/__init__.py -------------------------------------------------------------------------------- /liveRecorder/Config/config.ini: -------------------------------------------------------------------------------- 1 | [default_config] 2 | recording_base_dir = /web/content/kLive/liveRecorder 3 | max_task_count= 10 4 | polling_interval_sec = 60 5 | concat_processors_count = 2 6 | uploading_processors_count = 1 7 | upload_token_buffer_size_mb = 6 8 | num_of_upload_thread = 7 9 | session_duration = 86400 10 | failed_tasks_max_retries = 3 11 | failed_tasks_handling_interval = 1 12 | ks_privileges = all:*,disableentitlement 13 | 14 | 15 | # in minutes 16 | mode = local 17 | # Mode is choose between remote (ecdn), local (saas) or none (not doing anything with recording) 18 | 19 | nginx_port = 80 20 | nginx_host = localhost 21 | 22 | cron_job_polling_interval_hours = 2 23 | cron_job_log_file_name = /var/log/liveRecorder/recording_cron.log 24 | cron_jon_stamp = cronjob 25 | 26 | log_level = DEBUG 27 | log_to_console = False 28 | log_file_name = /var/log/liveRecorder/liveRecorder.log 29 | log_rotate_windows_files = 30 30 | 31 | recover_log_file_name = /var/log/liveRecorder/recover.log 32 | 33 | admin_secret = admin secret 34 | partner_id = 12345 35 | api_service_url = http://serviceURL 36 | -------------------------------------------------------------------------------- /liveRecorder/Config/config.py: -------------------------------------------------------------------------------- 1 | import ConfigParser 2 | import socket 3 | import logging 4 | import os 5 | import re 6 | Config = ConfigParser.ConfigParser() 7 | config_file_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'config.ini') 8 | Config.read(config_file_path) 9 | hostname = socket.gethostname() 10 | config_json = {} 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | def fill(section): 15 | options = Config.options(section) 16 | for option in options: 17 | try: 18 | config_json[option] = Config.get(section, option) 19 | except Exception : 20 | print("exception on %s!" % option) 21 | config_json[option] = None 22 | 23 | 24 | def config_section_map(): 25 | current_path = os.path.dirname(os.path.abspath(__file__)) 26 | Config.read(os.path.join(current_path, "configMapping.ini")) 27 | config_sections = Config.sections() 28 | for config_section in config_sections: 29 | pattern = re.compile(config_section) 30 | match = pattern.match(hostname) 31 | if match: 32 | fill(config_section) 33 | 34 | return config_json 35 | 36 | 37 | def config_section_default(): 38 | Config.read("config.ini") 39 | section = "default_config" 40 | fill(section) 41 | 42 | def set_config(key, value): 43 | config_json[key] = value 44 | 45 | 46 | def get_config(key, type = 'str'): 47 | if key in config_json: 48 | if type is not 'str': 49 | return eval(config_json[key]) 50 | 51 | return config_json[key] 52 | else: 53 | logger.warn("key %s is not configuration list", key) 54 | return None 55 | 56 | config_section_default() 57 | config_section_map() 58 | -------------------------------------------------------------------------------- /liveRecorder/Config/configMapping.ini.template: -------------------------------------------------------------------------------- 1 | [@HOSTNAME@] # change to .* if you want this configuration to be use for all hosts 2 | recording_base_dir = @RECORDING_FOLDER@ 3 | admin_secret = @KALTURA_PARTNER_ADMIN_SECRET@ 4 | partner_id = @KALTURA_PARTNER_ID@ 5 | mode = @VOD_UPLOAD_MODE@ 6 | token_key = @LIVE_PACKAGER_TOKEN@ 7 | cron_job_log_file_name = @LOGS_BASE_PATH@/recording_cron.log 8 | log_file_name = @LOGS_BASE_PATH@/liveRecorder.log 9 | recover_log_file_name = @LOGS_BASE_PATH@/recorder.log 10 | api_service_url = @KALTURA_SERVICE_URL@ -------------------------------------------------------------------------------- /liveRecorder/KalturaClient/Plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaltura/liveDVR/0f12e06e7ab139cbce4313ddc289311c0056f468/liveRecorder/KalturaClient/Plugins/__init__.py -------------------------------------------------------------------------------- /liveRecorder/KalturaClient/__init__.py: -------------------------------------------------------------------------------- 1 | from Client import KalturaClient 2 | from Base import KalturaConfiguration 3 | -------------------------------------------------------------------------------- /liveRecorder/KalturaClient/poster/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Chris AtLee 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | """poster module 21 | 22 | Support for streaming HTTP uploads, and multipart/form-data encoding 23 | 24 | ```poster.version``` is a 3-tuple of integers representing the version number. 25 | New releases of poster will always have a version number that compares greater 26 | than an older version of poster. 27 | New in version 0.6.""" 28 | 29 | import poster.streaminghttp 30 | import poster.encode 31 | 32 | version = (0, 8, 0) # Thanks JP! 33 | -------------------------------------------------------------------------------- /liveRecorder/Logger/LoggerDecorator.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | def logger_decorator(class_name, decorate): 5 | logger_info = class_name + '][' + decorate 6 | return logging.getLogger(logger_info) 7 | 8 | 9 | def log_subprocess_output(process, title, logger): 10 | header = "[{}] [pid={}]".format(title, process.pid) 11 | while True: 12 | nextline = process.stdout.readline() 13 | if nextline == '' and process.poll() is not None: 14 | break 15 | logger.info(header + ' %r', nextline) 16 | -------------------------------------------------------------------------------- /liveRecorder/Logger/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaltura/liveDVR/0f12e06e7ab139cbce4313ddc289311c0056f468/liveRecorder/Logger/__init__.py -------------------------------------------------------------------------------- /liveRecorder/README.md: -------------------------------------------------------------------------------- 1 | # liveRecorder 2 | 3 | ## Preqrequisites 4 | * python 2.7 5 | * pip 6 | * shared nfs storage 7 | 8 | 9 | ## Clone repository 10 | ``` 11 | git clone https://github.com/kaltura/liveDVR.git ; cd liveRecorder/ 12 | ``` 13 | 14 | ## Compile code: 15 | ``` 16 | ./build_scripts/build_ffmpeg.sh [Release/Debug] 17 | ./build_scripts/ts2mp4_build.sh 18 | FFMPEGPATH is the path to the ffmpeg root folder (for example root/ffmpeg/ffmpeg-3.0) 19 | LIVERECORDER_PATH is the path to liveRecorder directory tree 20 | ``` 21 | 22 | ## fill configMapping.ini in Config 23 | ``` 24 | admin_secret = admin secret 25 | partner_id = 12345 26 | api_service_url = http://serviceURL 27 | session_duration = *** (in seconds) 28 | ``` 29 | 30 | ### Run the following script 31 | ``` 32 | ./install.sh 33 | ``` 34 | 35 | ## Setup service script 36 | ``` 37 | cp liveRecorder.sh /etc/init.d/liveRecorder 38 | ``` 39 | 40 | 41 | ``` 42 | service liveRecorder start 43 | ``` 44 | 45 | To stop the server 46 | 47 | ``` 48 | service liveRecorder stop 49 | ``` 50 | 51 | To restart the server 52 | 53 | ``` 54 | service liveRecorder restart 55 | ``` 56 | 57 | 58 | To check the server status 59 | 60 | ``` 61 | service liveRecorder status 62 | ``` 63 | -------------------------------------------------------------------------------- /liveRecorder/RecordingException.py: -------------------------------------------------------------------------------- 1 | class UnequallStampException(Exception): 2 | pass -------------------------------------------------------------------------------- /liveRecorder/RecoverRecording.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from Tasks.TaskRunner import TaskRunner 3 | from Tasks.ConcatinationTask import ConcatenationTask 4 | from Tasks.UploadTask import UploadTask 5 | from Logger.Logger import init_logger 6 | from Config.config import set_config, get_config 7 | import os 8 | from BackendClient import * 9 | 10 | import logging.handlers 11 | 12 | class UploadTaskCustom(UploadTask): 13 | 14 | def __init__(self, base_directory, param): 15 | super(UploadTaskCustom, self).__init__(param, "UploadTaskCustom") 16 | self.output_file_path = os.path.join(base_directory, self.output_filename) 17 | 18 | def check_stamp(self): 19 | self.logger.warn("check_stamp mock") 20 | 21 | 22 | class ConcatenationTaskCustom(ConcatenationTask): 23 | def __init__(self, base_directory, param): 24 | super(ConcatenationTaskCustom, self).__init__(param, "ConcatenationTaskCustom") 25 | self.recording_path = base_directory 26 | 27 | 28 | def parser_argument_configure(): 29 | 30 | parser.add_argument('-e', '--entyId', help='Specified a custom live entryId (assume exist)') 31 | parser.add_argument('-r', '--recordedId', help='Specified a custom recording entryId (assume exist)') 32 | parser.add_argument('-d', '--recordingDuration', help='Specified a custom recording duration (assume exist)') 33 | parser.add_argument('-p', '--path', help='Path to recorded entry (mandatory)', required=False) 34 | 35 | 36 | def get_arg_params(): 37 | if args.entyId is not None: 38 | param['entry_id'] = args.entyId.lstrip() 39 | 40 | if args.recordedId is not None: 41 | param['recorded_id'] = args.recordedId.lstrip() 42 | 43 | if args.recordingDuration is not None: 44 | param['duration'] = args.recordingDuration.lstrip() 45 | logger.info("Parameters: %s", str(param)) 46 | 47 | set_config("log_to_console", "True") 48 | parser = argparse.ArgumentParser() 49 | parser_argument_configure() 50 | args = parser.parse_args() 51 | recover_log_file_name = get_config('recover_log_file_name') 52 | init_logger(recover_log_file_name) 53 | logger = logging.getLogger(__name__) 54 | path = args.path.lstrip() 55 | path_split = path.rsplit('/', 1) 56 | base_directory = path 57 | directory_name = path_split[1] 58 | has_all_custom_param = args.entyId is not None and args.recordedId is not None and args.recordingDuration is not None 59 | param ={} 60 | if TaskRunner.match(directory_name) is None: 61 | if has_all_custom_param is False: 62 | logger.error("Can't find all parameters, entyId [%s], recordedId [%s] recordingDuration [%s]", args.entyId, args.recordedId, args.recordingDuration) 63 | exit(1) 64 | else: 65 | get_arg_params() 66 | else: 67 | param = TaskRunner.get_param(directory_name) 68 | get_arg_params() 69 | 70 | ConcatenationTaskCustom(base_directory, param).run() 71 | UploadTaskCustom(base_directory, param).run() -------------------------------------------------------------------------------- /liveRecorder/Tasks/Iso639Wrapper.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from Logger.LoggerDecorator import logger_decorator 4 | 5 | class Iso639Wrapper: 6 | 7 | def __init__(self, logger_info): 8 | self.logger = logger_decorator(self.__class__.__name__, logger_info) 9 | dir_path = os.path.dirname(os.path.realpath(__file__)) 10 | full_path = os.path.join(dir_path, "database", "iso639-3.json") 11 | # load iso639 database 12 | with open(full_path) as database: 13 | self.language_database = json.load(database) 14 | 15 | def convert_language_to_iso639_3(self, in_language): 16 | size_lang = len(in_language) 17 | out_language = "und" 18 | if size_lang == 3: 19 | self.logger.debug("no conversion done. Language = %s", in_language) 20 | return in_language 21 | if size_lang == 2: 22 | for entry in self.language_database[u'639-3']: 23 | if u'alpha_2' in entry and entry[u'alpha_2'] == in_language: 24 | out_language = entry[u'alpha_3'].encode("utf8") 25 | self.logger.debug("found conversion for language: iso639-1: %s --> iso639-3: %s", in_language, 26 | out_language) 27 | break 28 | if out_language == "und": 29 | self.logger.error("no conversion found for language \'%s\'", in_language) 30 | return out_language 31 | 32 | self.logger.error("unrecognized or invalid input language \'%s\'", in_language) 33 | return out_language 34 | 35 | 36 | -------------------------------------------------------------------------------- /liveRecorder/Tasks/MockFileObject.py: -------------------------------------------------------------------------------- 1 | class MockFileObject: 2 | def __init__(self, file_name, _buffer): 3 | self.length = len(_buffer) 4 | self.buffer = _buffer 5 | self.name = file_name 6 | self.offset = 0 7 | 8 | def read(self, index=-1): 9 | if index < 0: 10 | result = self.buffer[self.offset:] 11 | self.offset = self.length 12 | return result 13 | result = self.buffer[self.offset:self.offset+index] 14 | self.offset += index 15 | if self.offset > self.length: 16 | self.offset = self.length 17 | return result 18 | 19 | def seek(self, offset, whence = 0): 20 | if whence == 0: 21 | self.offset = offset 22 | if whence == 1: 23 | self.offset = self.offset - offset 24 | if whence == 2: 25 | self.offset = self.length - offset 26 | if self.offset > self.length: 27 | self.offset = self.length 28 | 29 | def tell(self): 30 | return self.offset 31 | -------------------------------------------------------------------------------- /liveRecorder/Tasks/ThreadWorkers.py: -------------------------------------------------------------------------------- 1 | from BackendClient import * 2 | from Config.config import get_config 3 | import Queue 4 | from threading import Thread, Lock 5 | import logging 6 | import traceback 7 | 8 | ''' 9 | This class is a singleton class for each upload process. 10 | However, the job (upload) is done one by one, 11 | ''' 12 | 13 | 14 | class Singleton(type): 15 | _instances = {} 16 | _lock = Lock() 17 | 18 | def __call__(cls, *args, **kwargs): 19 | with Singleton._lock: 20 | if cls not in cls._instances: 21 | cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) 22 | return cls._instances[cls] 23 | 24 | 25 | class ThreadWorkers: # singleton object, 26 | __metaclass__ = Singleton 27 | num_of_thread = get_config('num_of_upload_thread', 'int') 28 | logger = logging.getLogger(__name__) 29 | 30 | def __init__(self): 31 | self.q = Queue.Queue() 32 | self.generate_upload_thread() 33 | self.job_failed = [] 34 | 35 | def generate_upload_thread(self): 36 | for i in range(1, self.num_of_thread+1): 37 | t = Thread(target=self.worker, args=(i,)) 38 | t.setName("UploadTasks-"+str(i)) # note this is not work for multiple uploader process 39 | t.daemon = True 40 | t.start() 41 | 42 | def worker(self, index): 43 | self.logger.info("Thread %d started working", index) 44 | while True: 45 | upload_chunk_job = self.q.get() 46 | if not upload_chunk_job: 47 | self.logger.warning("Got \'None\' as upload job. Check if it's a bug") 48 | continue 49 | try: 50 | upload_chunk_job.upload() 51 | except Exception as e: 52 | self.logger.error("Failed to upload chunk %s from file %s : %s \n %s", upload_chunk_job.chunk_index, 53 | upload_chunk_job.upload_session.file_name, str(e), traceback.format_exc()) 54 | self.job_failed.append(upload_chunk_job) 55 | finally: 56 | self.q.task_done() 57 | 58 | def add_job(self, job): 59 | self.q.put(job) 60 | 61 | def wait_jobs_done(self): 62 | self.q.join() # wait for all task finish 63 | job_failed_to_return = self.job_failed 64 | self.job_failed = [] # initial array for the next job 65 | return job_failed_to_return 66 | -------------------------------------------------------------------------------- /liveRecorder/Tasks/UploadChunkJob.py: -------------------------------------------------------------------------------- 1 | from MockFileObject import MockFileObject 2 | from threading import Thread, Lock 3 | 4 | 5 | class UploadChunkJob: # todo move it to other file 6 | #global backend_client 7 | mutex = Lock() # mutex for prevent race when reading file 8 | 9 | def __init__(self, upload_session, final_chunk, resume_at, resume, chunk_index): 10 | self.upload_session = upload_session 11 | self.final_chunk = final_chunk 12 | self.resume_at = resume_at 13 | self.resume = resume 14 | self.chunk_index = chunk_index 15 | 16 | def upload(self): 17 | pointer_to_read = self.resume_at 18 | self.mutex.acquire() 19 | try: 20 | self.upload_session.infile.seek(pointer_to_read) 21 | data = self.upload_session.infile.read(self.upload_session.upload_token_buffer_size) 22 | except IOError as e: 23 | raise e # if exception then do not continue 24 | finally: # called anyway 25 | self.mutex.release() 26 | self.file_obj = MockFileObject(self.upload_session.infile.name, data) 27 | result = self.upload_session.backend_client.upload_token_upload(self) #todo what to do with result -------------------------------------------------------------------------------- /liveRecorder/Tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaltura/liveDVR/0f12e06e7ab139cbce4313ddc289311c0056f468/liveRecorder/Tasks/__init__.py -------------------------------------------------------------------------------- /liveRecorder/ZombieEntryCatchers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import time 3 | import logging.handlers 4 | import os 5 | import re 6 | import traceback 7 | from Config.config import get_config 8 | from Logger.Logger import init_logger 9 | import glob 10 | 11 | logger = logging.getLogger('ZombieEntryCatchers') 12 | recording_base_dir = get_config('recording_base_dir') 13 | recordings_dir = os.path.join(recording_base_dir, 'recordings') 14 | recording_incoming_dir = os.path.join(recording_base_dir, 'incoming') 15 | entry_regex = '^[01]_\w{8}' 16 | pattern = re.compile(entry_regex) 17 | threshold_time_sec = 3600 # 1 hour 18 | log_full_path = get_config('cron_job_log_file_name') 19 | init_logger(log_full_path) 20 | polling_interval_sec = get_config('cron_job_polling_interval_hours', 'int') * 60 * 60 21 | cron_jon_stamp = get_config('cron_jon_stamp') 22 | 23 | def job(): 24 | logger.info("Start scanning directory in %s", recordings_dir) 25 | now = int(time.time()) 26 | recording_list = glob.glob(recordings_dir + '/*/*/*/*') 27 | 28 | for recorded_id_path in recording_list: 29 | try: 30 | recorded_id = os.path.basename(recorded_id_path) 31 | entry_id_path = os.path.dirname(recorded_id_path) 32 | entry_id = os.path.basename(entry_id_path) 33 | if not pattern.match(recorded_id) or not os.path.isdir(recorded_id_path) or not pattern.match(entry_id): 34 | continue 35 | session_id = ''.join([entry_id, '-', recorded_id]) 36 | done_path = os.path.join(recorded_id_path, 'done') 37 | stamp_path = os.path.join(recorded_id_path, 'stamp') 38 | last_modify_time = os.path.getmtime(recorded_id_path) 39 | is_expired = (now - last_modify_time) > threshold_time_sec 40 | done_exist = os.path.isfile(done_path) 41 | logger.debug("[%s] now %s last_modify_time %s, diff: %s, is_expired : %s, done_exist: %s", session_id, 42 | now, last_modify_time, now-last_modify_time, is_expired, done_exist) 43 | if is_expired and not done_exist: 44 | logger.warn("[%s] Found zombie entry", session_id) 45 | directory_new_name = ''.join([entry_id, '_', recorded_id, '_', str(time.time())]) 46 | destination_target = os.path.join(recording_incoming_dir, directory_new_name) 47 | logger.debug("[%s] About to create sym link from %s into %s", session_id, directory_new_name, 48 | destination_target) 49 | with open(stamp_path,'w') as stamp_file: 50 | stamp_file.truncate() 51 | stamp_file.write(cron_jon_stamp) 52 | os.symlink(recorded_id_path, destination_target) 53 | logger.debug("[%s] Successfully created symlink, about to create done file in %s ", session_id, 54 | done_path) 55 | with open(done_path, 'a'): 56 | os.utime(done_path, None) 57 | except Exception as e: 58 | logger.error("[%s] Failed to catch Zombies entries %s : %s", str(e), entry_id, traceback.format_exc()) 59 | 60 | 61 | job() 62 | -------------------------------------------------------------------------------- /liveRecorder/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # pythonInstall.sh 4 | # 5 | # 6 | # Created by Ron Yadgar on 25/09/2016. 7 | # 8 | SOURCE_DIRECTORY="/opt/kaltura/liveController/latest" 9 | HOME_DIRECTORY=`grep recording_base_dir $SOURCE_DIRECTORY/liveRecorder/Config/config.ini | awk '{ print $3 }'` 10 | HOSTNAME=$(hostname) 11 | HOSTNAME_DIRECTORY="$HOME_DIRECTORY/$HOSTNAME" 12 | echo "home directory $HOME_DIRECTORY" 13 | if [ ! -d $HOME_DIRECTORY ]; then 14 | echo "ERROR: can't find recording path" 15 | exit 1 16 | fi 17 | if ! [[ $(python --version 2>&1) == *2\.7\.* ]]; then 18 | echo "Python version >= 2.7.0 is required"; 19 | exit 2 20 | fi 21 | pip install poster 22 | pip install psutil 23 | pip install m3u8 24 | pip install schedule 25 | pip install pycrypto 26 | mkdir -p $HOME_DIRECTORY 27 | mkdir -p "$HOME_DIRECTORY/recordings" 28 | mkdir -p "$HOME_DIRECTORY/recordings/newSession" 29 | mkdir -p "$HOME_DIRECTORY/recordings/append" 30 | mkdir -p "$HOME_DIRECTORY/error" 31 | mkdir -p "$HOME_DIRECTORY/incoming" 32 | mkdir -p $HOSTNAME_DIRECTORY 33 | UPLOAD_TASK_DIRECTORY="$HOSTNAME_DIRECTORY/UploadTask" 34 | CONCATINATION_TASK_DIRECTORY="$HOSTNAME_DIRECTORY/ConcatenationTask" 35 | mkdir -p $CONCATINATION_TASK_DIRECTORY 36 | ln -s "$HOME_DIRECTORY/incoming" "$CONCATINATION_TASK_DIRECTORY/incoming" 37 | mkdir -p "$CONCATINATION_TASK_DIRECTORY/failed" 38 | mkdir -p "$CONCATINATION_TASK_DIRECTORY/processing" 39 | mkdir -p $UPLOAD_TASK_DIRECTORY 40 | mkdir -p "$UPLOAD_TASK_DIRECTORY/failed" 41 | mkdir -p "$UPLOAD_TASK_DIRECTORY/incoming" 42 | mkdir -p "$UPLOAD_TASK_DIRECTORY/processing" 43 | cp $SOURCE_DIRECTORY/recordingUploader/liveRecorder.sh /etc/init.d/liveRecorder 44 | /etc/init.d/liveRecorder.sh restart 45 | -------------------------------------------------------------------------------- /liveRecorder/installPython.sh: -------------------------------------------------------------------------------- 1 | # installation of Python 2.7 on RHEL/CentOS 6: 2 | rpm -ihv http://mirror.centos.org/centos/6/extras/x86_64/Packages/centos-release-scl-rh-2-3.el6.centos.noarch.rpm 3 | yum install python27 -y 4 | echo "run . /opt/rh/python27/enable to work with python version suitable for Kaltura live platform" -------------------------------------------------------------------------------- /liveRecorder/main.py: -------------------------------------------------------------------------------- 1 | from Tasks.TaskRunner import TaskRunner 2 | from Tasks.ConcatinationTask import ConcatenationTask 3 | from Tasks.UploadTask import UploadTask 4 | from Config.config import get_config 5 | from os import path 6 | from Logger.Logger import init_logger 7 | import sys 8 | import signal 9 | import psutil 10 | import socket 11 | #todo list 12 | 13 | # 5. should recording also the backup ? 14 | # install psutilg 15 | # How to recover from case that live-controller crash, when need to cread hard link/Wrote to json- maybe flavor download should send event to all his chunks on disk 16 | # the recording entry should created after session ended, and the 17 | # initial logger 18 | 19 | 20 | def signal_term_handler(signal, frame): 21 | print 'got signal '+str(signal) 22 | for my_process in processes: 23 | print ("kill process "+str(my_process.pid)) 24 | try: 25 | parent = psutil.Process(my_process.pid) 26 | except psutil.NoSuchProcess: 27 | print ("Not child process for " + str(my_process.pid)) 28 | children = parent.children(recursive=True) 29 | for child_process in children: 30 | print ("Found child process " + str(child_process.pid)+ ", send SIGTERM") 31 | child_process.kill() 32 | 33 | my_process.terminate() 34 | sys.exit(0) 35 | 36 | 37 | log_full_path = get_config('log_file_name') 38 | init_logger(log_full_path) 39 | processes = [] 40 | max_task_count = get_config("max_task_count", 'int') 41 | concat_processors_count = get_config('concat_processors_count', 'int') 42 | uploading_processors_count = get_config('uploading_processors_count', 'int') 43 | base_directory = get_config('recording_base_dir') 44 | tasks_done_directory = path.join(base_directory, 'done') 45 | incoming_upload_directory = path.join(base_directory, socket.gethostname(), UploadTask.__name__, 'incoming') 46 | 47 | 48 | signal.signal(signal.SIGTERM, signal_term_handler) 49 | signal.signal(signal.SIGINT, signal_term_handler) 50 | 51 | 52 | ConcatenationTaskRunner = TaskRunner(ConcatenationTask, concat_processors_count, incoming_upload_directory, 53 | max_task_count, tasks_done_directory).start() 54 | 55 | UploadTaskRunner = TaskRunner(UploadTask, uploading_processors_count, tasks_done_directory, max_task_count, tasks_done_directory).start() 56 | 57 | for p in ConcatenationTaskRunner: 58 | processes.append(p) 59 | 60 | for p in UploadTaskRunner: 61 | processes.append(p) 62 | 63 | for process in processes: 64 | process.join() 65 | 66 | # todo should add p.join? -------------------------------------------------------------------------------- /liveRecorder/scripts/token_generator: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys 4 | import getopt 5 | import os 6 | import re 7 | import hashlib 8 | import base64 9 | 10 | def tokenizeUrl(url, secret): 11 | dir_name = os.path.dirname(url) 12 | dir_name = re.sub(r'https?://', '', dir_name) 13 | token = "{0} {1}/".format(secret, dir_name) 14 | hash = hashlib.md5(token).digest() 15 | encoded_hash = base64.urlsafe_b64encode(hash).rstrip('=') 16 | return encoded_hash 17 | 18 | 19 | def main(argv): 20 | secret = '' 21 | url = '' 22 | 23 | try: 24 | opts, args = getopt.getopt(argv, "hi:o:", ["ifile=", "ofile="]) 25 | except getopt.GetoptError: 26 | print 'token_generator.py -u -h ' 27 | sys.exit(2) 28 | for opt, arg in opts: 29 | if opt == '-h': 30 | secret = arg 31 | elif opt == '-u': 32 | url = arg 33 | else: 34 | print 'invalid or missing params.\n Following is correct format: token_generator.py -u -h ' 35 | sys.exit(-1) 36 | tokenizeUrl(url, secret) 37 | print tokenizeUrl(url, secret) 38 | 39 | 40 | if __name__ == "__main__": 41 | main(sys.argv[1:]) -------------------------------------------------------------------------------- /liveRecorder/serviceWrappers/defaults/liveRecorder: -------------------------------------------------------------------------------- 1 | # Kaltura liveRecorder defaults file 2 | LOG_DIR=/var/log 3 | PID_DIR=/var/run 4 | -------------------------------------------------------------------------------- /liveRecorder/serviceWrappers/defaults/liveRecorder.template: -------------------------------------------------------------------------------- 1 | PID_DIR="@RUN_DIR@" 2 | LOG_DIR="@LOG_DIR@" 3 | KLIVE_RECORDER_PREFIX="@KLIVE_RECORDER_PREFIX@" -------------------------------------------------------------------------------- /liveRecorder/serviceWrappers/linux/getRecordingBaseDir.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) 3 | from Config.config import get_config 4 | print get_config("recording_base_dir") 5 | -------------------------------------------------------------------------------- /liveRecorder/ts_to_mp4_convertor/Makefile: -------------------------------------------------------------------------------- 1 | CC = gcc -v -std=c99 2 | SRC = $(wildcard *.c) 3 | DEPS = $(wildcard *.h) 4 | ODIR=obj 5 | _OBJ = $(SRC:.c=.o) 6 | OBJ = $(patsubst %,$(ODIR)/%,$(_OBJ)) 7 | _EXE = ts_to_mp4_convertor 8 | EXE = $(patsubst %,$(ODIR)/%,$(_EXE)) 9 | FFMPEG_LIB_DIR := $(shell echo $(FFMPEG_BUILD_PATH)) 10 | IDIRS = -I. -I$(FFMPEG_LIB_DIR) 11 | LDIR = -L$(FFMPEG_LIB_DIR)/libswscale -L$(FFMPEG_LIB_DIR)/libavdevice -L$(FFMPEG_LIB_DIR)/libavutil -L$(FFMPEG_LIB_DIR)/libavformat -L$(FFMPEG_LIB_DIR)/libavcodec -L/usr/local/lib 12 | FFMPEG_LIBS = -lswscale -lavdevice -lavformat -lavcodec -lavutil 13 | LIBS = -lpthread -lm -lz 14 | CFLAGS = -Wall -g $(IDIRS) 15 | LDFLAGS = $(LDIR) $(FFMPEG_LIBS) $(LIBS) 16 | OS := $(shell uname) 17 | 18 | ifeq ($(OS), Linux) 19 | LIBS += -lrt 20 | else 21 | LIBS += -liconv 22 | endif 23 | 24 | $(ODIR)/%.o: %.c $(DEPS) 25 | $(CC) -c -o $@ $< $(CFLAGS) 26 | 27 | $(EXE): $(OBJ) 28 | $(CC) -o $@ $^ $(LDFLAGS) 29 | 30 | $(phony install): install 31 | 32 | install: $(EXE) 33 | mkdir -p ../bin 34 | install $(EXE) ../bin/ 35 | 36 | .PHONY: clean 37 | 38 | clean: 39 | rm -rf $(ODIR)/*.o $(EXE) 40 | -------------------------------------------------------------------------------- /liveRecorder/ts_to_mp4_convertor/audio_filler.h: -------------------------------------------------------------------------------- 1 | // 2 | // audio.h 3 | // ts_to_mp4_convertor 4 | // 5 | // Created by Guy.Jacubovski on 25/10/2017. 6 | // Copyright © 2017 Kaltura. All rights reserved. 7 | // 8 | 9 | #ifndef audio_h 10 | #define audio_h 11 | 12 | 13 | void createSilentAudio(AVCodecContext *pContext,AVPacket* dst); 14 | 15 | 16 | #endif /* audio_h */ 17 | -------------------------------------------------------------------------------- /liveRecorder/ts_to_mp4_convertor/downloader.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | import re 3 | import os 4 | import sys 5 | 6 | regex = r"href=\"([^\"]+.ts)\"" 7 | 8 | 9 | baseUrl = 'http://192.168.10.127/kLive/liveRecorder/done/1_1yxe2oi8_1_mngvnyea_11422102' 10 | 11 | folder = os.path.basename(baseUrl) 12 | 13 | print "Reading: ", baseUrl 14 | req = urllib2.Request(baseUrl) 15 | #req.headers['Range'] = 'bytes=%s-%s' % (start, end) 16 | f = urllib2.urlopen(req) 17 | content = f.read() 18 | 19 | def ensure_dir(file_path): 20 | directory = os.path.dirname(file_path) 21 | if not os.path.exists(directory): 22 | os.makedirs(directory) 23 | 24 | def chunk_copy(response, output,total,chunk_size=8192): 25 | bytes_so_far = 0 26 | 27 | while 1: 28 | chunk = response.read(chunk_size) 29 | bytes_so_far += len(chunk) 30 | print "\rtotal ", bytes_so_far, '/' , fileSize, ' (',100*bytes_so_far/fileSize,'% )' 31 | sys.stdout.flush() 32 | 33 | 34 | if not chunk: 35 | print "total ", bytes_so_far 36 | break 37 | 38 | output.write(chunk) 39 | 40 | 41 | matches = re.finditer(regex, content) 42 | 43 | chunk_size = 188*1024 44 | checks_to_copy = 1000 45 | for match in enumerate(matches): 46 | 47 | fileName = match[1].group(1) 48 | localFile = "./"+folder+"/"+fileName 49 | 50 | print "Downloading: ", baseUrl+"/"+fileName, " into ",localFile 51 | req2 = urllib2.Request(baseUrl+"/"+fileName) 52 | fileSize = checks_to_copy*chunk_size 53 | req2.add_header('Range','bytes=%s-%s' % (0, fileSize)) 54 | f2 = urllib2.urlopen(req2) 55 | ensure_dir(localFile) 56 | 57 | with open(localFile,'wb') as output: 58 | chunk_copy(f2,output,fileSize,chunk_size) 59 | # 60 | -------------------------------------------------------------------------------- /liveRecorder/ts_to_mp4_convertor/tests.h: -------------------------------------------------------------------------------- 1 | // 2 | // tests.h 3 | // ts_to_mp4_convertor 4 | // 5 | // Created by Guy.Jacubovski on 13/03/2018. 6 | // Copyright © 2018 Kaltura. All rights reserved. 7 | // 8 | 9 | #ifndef tests_h 10 | #define tests_h 11 | 12 | 13 | char *test1[] = {"", 14 | "1_iac50ypm_1_2lkq3suw_1231253_f35_out.ts", 15 | "1_iac50ypm_1_2lkq3suw_1231253_f35_out.mp4", 16 | "und", 17 | "1_iac50ypm_1_2lkq3suw_1231253_f43_out.ts", 18 | "1_iac50ypm_1_2lkq3suw_1231253_f43_out.mp4", 19 | "und", 20 | "1_iac50ypm_1_2lkq3suw_1231253_f42_out.ts", 21 | "1_iac50ypm_1_2lkq3suw_1231253_f42_out.mp4", 22 | "und", 23 | "1_iac50ypm_1_2lkq3suw_1231253_f1230131_out.ts", 24 | "1_iac50ypm_1_2lkq3suw_1231253_f1230131_out.mp4", 25 | "und", 26 | "1_iac50ypm_1_2lkq3suw_1231253_f33_out.ts", 27 | "1_iac50ypm_1_2lkq3suw_1231253_f33_out.mp4", 28 | "und", 29 | "1_iac50ypm_1_2lkq3suw_1231253_f34_out.ts", 30 | "1_iac50ypm_1_2lkq3suw_1231253_f34_out.mp4", 31 | "und", 32 | "1_iac50ypm_1_2lkq3suw_1231253_f111_out.ts", 33 | "1_iac50ypm_1_2lkq3suw_1231253_f111_outt.mp4", 34 | "swe", 35 | "1_iac50ypm_1_2lkq3suw_1231253_f110_out.ts", 36 | "1_iac50ypm_1_2lkq3suw_1231253_f110_out.mp4", 37 | "fin"}; 38 | 39 | 40 | char *test2[] = {"", 41 | "0_wntu9dd2_0_o03fregx_3052229_f1851641_out.ts", 42 | "0_wntu9dd2_0_o03fregx_3052229_f1851641_out.mp4", 43 | "und", 44 | "0_wntu9dd2_0_o03fregx_3052229_f33_out.ts", 45 | "0_wntu9dd2_0_o03fregx_3052229_f33_out.mp4", 46 | "und", 47 | "0_wntu9dd2_0_o03fregx_3052229_f34_out.ts", 48 | "0_wntu9dd2_0_o03fregx_3052229_f34_out.mp4", 49 | "und", 50 | "0_wntu9dd2_0_o03fregx_3052229_f35_out.ts", 51 | "0_wntu9dd2_0_o03fregx_3052229_f35_out.mp4", 52 | "und"}; 53 | 54 | 55 | char *test3[] = {"", 56 | "0_tcwszl6n_0_2sk6or15_444434_f36_out.ts", 57 | "0_tcwszl6n_0_2sk6or15_444434_f36_out.mp4", 58 | "und", 59 | "0_tcwszl6n_0_2sk6or15_444434_f32_out.ts", 60 | "0_tcwszl6n_0_2sk6or15_444434_f32_out.mp4", 61 | "und", 62 | "0_tcwszl6n_0_2sk6or15_444434_f37_out.ts", 63 | "0_tcwszl6n_0_2sk6or15_444434_f37_out.mp4", 64 | "und"}; 65 | 66 | char *test4[] = {"", 67 | "1_8x9l5leh_1_nqgux1am_3382038_f42_out.ts", 68 | "1_8x9l5leh_1_nqgux1am_3382038_f42_out.mp4", 69 | "und", 70 | "1_8x9l5leh_1_nqgux1am_3382038_f43_out.ts", 71 | "1_8x9l5leh_1_nqgux1am_3382038_f43_out.mp4", 72 | "und"}; 73 | 74 | 75 | #endif /* tests_h */ 76 | -------------------------------------------------------------------------------- /liveRecorder/ts_to_mp4_convertor/ts_to_mp4_convertor.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /node_addons/FormatConverter/binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "ffmpegDir": "<(module_root_dir)/build/FFmpeg" 4 | }, 5 | "targets": [ 6 | 7 | { 8 | "target_name": "FormatConverter", 9 | "type": "<(library)", 10 | "sources": [ 11 | "src/NodeFormatConverter.cpp" , 12 | "src/Converter.cpp" , 13 | "src/AVFormat.cpp" , 14 | "src/NodeStream.cpp", 15 | "src/Stream.cpp", 16 | "include/Utils.h", 17 | "include/Stream.h", 18 | "include/Converter.h", 19 | "include/AVFormat.h", 20 | "include/MemoryStream.h", 21 | "include/NodeStream.h", 22 | "include/NodeFormatConverter.h" 23 | ], 24 | "include_dirs": [ " 16 | #include 17 | #include 18 | #include 19 | } 20 | #include 21 | #include "Stream.h" 22 | #include "Utils.h" 23 | 24 | 25 | namespace converter{ 26 | 27 | class CFtCtx { 28 | 29 | CFtCtx(const CFtCtx&); 30 | void operator=(const CFtCtx&); 31 | 32 | protected: 33 | AVFormatContext *m_pCtx; 34 | std::shared_ptr m_stream; 35 | 36 | static int stream_write(void *opaque, uint8_t *buf, int size) 37 | { 38 | return reinterpret_cast(opaque)->Write(buf,size); 39 | } 40 | 41 | static int64_t stream_seek(void *opaque, int64_t offset, int whence) 42 | { 43 | return reinterpret_cast(opaque)->Seek(offset,whence); 44 | } 45 | 46 | static int stream_read(void *opaque, uint8_t *buf, int size){ 47 | 48 | return reinterpret_cast(opaque)->Read(buf,size); 49 | } 50 | 51 | public: 52 | 53 | CFtCtx(AVFormatContext *pCtx = NULL) 54 | :m_pCtx(pCtx) 55 | {} 56 | 57 | AVFormatContext *operator->(){ 58 | return m_pCtx; 59 | } 60 | 61 | AVFormatContext *operator *(){ 62 | return m_pCtx; 63 | } 64 | 65 | AVFormatContext **operator &(){ 66 | return &m_pCtx; 67 | } 68 | 69 | int Dispose(); 70 | 71 | virtual ~CFtCtx(){ 72 | Dispose(); 73 | } 74 | 75 | void EmitInfo(const MediaFileInfo &mfi) 76 | { 77 | if(m_stream){ 78 | m_stream->EmitInfo(mfi); 79 | } 80 | } 81 | 82 | std::shared_ptr GetStream() const { 83 | return m_stream; 84 | } 85 | }; 86 | 87 | class CInputCtx : public CFtCtx{ 88 | AVInputFormat *m_format; 89 | public: 90 | 91 | int checkStreams(); 92 | 93 | int init(const std::string &type,std::shared_ptr stream); 94 | 95 | int Close(); 96 | 97 | ~CInputCtx(); 98 | }; 99 | 100 | class COutputCtx : public CFtCtx 101 | { 102 | public: 103 | int init(const std::string &type,size_t bufSize,std::shared_ptr stream); 104 | 105 | int Close(); 106 | 107 | ~COutputCtx(); 108 | }; 109 | 110 | 111 | 112 | }; 113 | 114 | 115 | #endif 116 | -------------------------------------------------------------------------------- /node_addons/FormatConverter/include/FileStream.h: -------------------------------------------------------------------------------- 1 | // 2 | // FileStream.h 3 | // FormatConverter 4 | // 5 | // Created by Igor Shevach on 2/29/16. 6 | // Copyright (c) 2016 Igor Shevach. All rights reserved. 7 | // 8 | 9 | #ifndef FormatConverter_FileStream_h 10 | #define FormatConverter_FileStream_h 11 | 12 | #include "Stream.h" 13 | 14 | namespace converter{ 15 | 16 | 17 | 18 | class FileStream : public StreamBase { 19 | protected: 20 | FILE *m_fp; 21 | 22 | void close(){ 23 | 24 | if(m_fp){ 25 | fclose(m_fp); 26 | m_fp = NULL; 27 | } 28 | } 29 | 30 | public: 31 | 32 | FileStream() 33 | :m_fp(NULL) 34 | {} 35 | 36 | ~FileStream() 37 | { 38 | close(); 39 | } 40 | 41 | int open(const std::string &path, const std::string &mode){ 42 | close(); 43 | m_fp = fopen(path.c_str(),mode.c_str()); 44 | return m_fp ? 0 : -1; 45 | } 46 | 47 | virtual int Write(uint8_t *buf, int size) { 48 | return m_fp ? fwrite(buf,1,size,m_fp) : 0; 49 | } 50 | virtual int Read(uint8_t *buf, int size){ 51 | return m_fp ? fread(buf,1,size,m_fp) : 0; 52 | } 53 | virtual int64_t Seek(int64_t offset, int whence) { 54 | return m_fp ? fseek(m_fp,offset,whence) : 0; 55 | } 56 | }; 57 | 58 | }; 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /node_addons/FormatConverter/include/MemoryStream.h: -------------------------------------------------------------------------------- 1 | // 2 | // MemoryStream.h 3 | // binding 4 | // 5 | // Created by Igor Shevach on 3/2/16. 6 | // 7 | // 8 | 9 | #ifndef binding_MemoryStream_h 10 | #define binding_MemoryStream_h 11 | 12 | #include "Stream.h" 13 | extern "C"{ 14 | #include 15 | } 16 | 17 | namespace converter{ 18 | 19 | class MemoryInputStream : public StreamBase{ 20 | const uint8_t *m_buf,*m_last; 21 | size_t m_length; 22 | protected: 23 | void initData(const uint8_t *pb,size_t len){ 24 | m_buf = pb; 25 | m_length = len; 26 | } 27 | public: 28 | MemoryInputStream(const uint8_t *pb = nullptr,size_t len = 0) 29 | :m_buf(pb), 30 | m_length(len) 31 | { 32 | m_last = m_buf; 33 | } 34 | 35 | size_t get_length() const { 36 | return m_length; 37 | } 38 | 39 | virtual int Write(uint8_t *buf, int size) { 40 | return -1; 41 | } 42 | virtual int Read(uint8_t *buf, int size) { 43 | //av_log(nullptr,AV_LOG_TRACE,"MemoryInputStream::Read(%p,%d)",buf,size); 44 | size_t avail = std::min(get_length() - (m_last - m_buf),(size_t)size); 45 | if(avail > 0){ 46 | std::copy(m_last,m_last+avail,buf); 47 | m_last += avail; 48 | } 49 | av_log(nullptr,AV_LOG_TRACE,"MemoryInputStream::Read(%p,%d) available %zu",buf,size,avail); 50 | return avail; 51 | } 52 | virtual int64_t Seek(int64_t offset, int whence) { 53 | 54 | switch(whence) 55 | { 56 | case SEEK_SET: 57 | m_last = m_buf + offset; 58 | break; 59 | case SEEK_END: 60 | m_last = m_buf + get_length() - offset; 61 | break; 62 | case SEEK_CUR: 63 | m_last += offset; 64 | break; 65 | case AVSEEK_SIZE: 66 | return get_length(); 67 | break; 68 | }; 69 | if(m_last < m_buf) 70 | m_last = m_buf; 71 | if(m_last > m_buf + get_length()) 72 | m_last = m_buf + get_length(); 73 | 74 | av_log(nullptr,AV_LOG_TRACE,"MemoryInputStream::Seek(%lld,%d). total len: %zu pos: %ld",offset,whence,get_length(),m_last - m_buf); 75 | 76 | return m_last - m_buf; 77 | } 78 | 79 | }; 80 | 81 | }; 82 | #endif 83 | -------------------------------------------------------------------------------- /node_addons/FormatConverter/include/NodeStream.h: -------------------------------------------------------------------------------- 1 | // 2 | // NodeStream.h 3 | // binding 4 | // 5 | // Created by Igor Shevach on 3/8/16. 6 | // 7 | // 8 | 9 | #ifndef binding_NodeStream_h 10 | #define binding_NodeStream_h 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | using namespace v8; 17 | using namespace node; 18 | 19 | namespace converter { 20 | 21 | class InputStreamOnBuffer : public MemoryInputStream 22 | { 23 | Nan::Persistent m_buffer; 24 | public: 25 | InputStreamOnBuffer(Local b, const uint8_t *pb,size_t len) 26 | :MemoryInputStream(pb,len) { 27 | m_buffer.Reset(b); 28 | } 29 | 30 | ~InputStreamOnBuffer(){ 31 | m_buffer.Reset(); 32 | } 33 | }; 34 | 35 | class NodeOutputStream : public StreamBase{ 36 | 37 | public: 38 | 39 | std::vector m_output; 40 | std::vector::iterator m_pos; 41 | size_t m_written; 42 | std::ofstream m_ofs; 43 | 44 | NodeOutputStream(); 45 | 46 | ~NodeOutputStream(); 47 | 48 | virtual int Write(uint8_t *buf, int size); 49 | 50 | virtual int Read(uint8_t *buf, int size); 51 | 52 | virtual int64_t Seek(int64_t offset, int whence); 53 | 54 | MediaFileInfo m_fileInfo; 55 | 56 | virtual void EmitInfo(const MediaFileInfo &mfi); 57 | 58 | static void FreeCallback(char* data, void* hint); 59 | 60 | Local createFastBuffer(); 61 | 62 | Local GetData(Isolate *isolate); 63 | Local GetFileInfo(Isolate *isolate); 64 | 65 | }; 66 | 67 | } 68 | 69 | #endif 70 | -------------------------------------------------------------------------------- /node_addons/FormatConverter/readme.md: -------------------------------------------------------------------------------- 1 | ## File Format Converter addon for node 2 | # prerequisite 3 | on OSX make sure you have yasm 4 | which yasm 5 | brew install yasm 6 | 7 | #Mac and linux: 8 | 9 | - cd FormatConverter 10 | - chmod +x setenv.sh 11 | - ./setenv.sh 12 | 13 | * to build debug version use: 14 | - ./setenv.sh DEBUG 15 | -------------------------------------------------------------------------------- /node_addons/FormatConverter/src/main.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // main.c 3 | // mpegts-mp4-converter 4 | // 5 | // Created by Igor Shevach on 2/25/16. 6 | // Copyright (c) 2016 Igor Shevach. All rights reserved. 7 | // 8 | 9 | #include "Utils.h" 10 | #include "AVFormat.h" 11 | #include "FileStream.h" 12 | #include "Converter.h" 13 | using namespace converter; 14 | 15 | 16 | class CPushStream : public StreamBase{ 17 | 18 | std::shared_ptr m_stream; 19 | Converter &m_conv; 20 | std::vector m_pushbuf; 21 | std::vector::iterator m_pos,m_end; 22 | 23 | virtual int Write(uint8_t *buf, int size){ return -1;} 24 | virtual int Read(uint8_t *buf, int size) { 25 | // return m_stream->Read(buf,size ); 26 | std::vector::difference_type n = m_end - m_pos; 27 | std::vector::difference_type available = std::min(m_end - m_pos,(std::vector::difference_type )size); 28 | std::copy(m_pos, m_pos+available, buf); 29 | m_pos += available; 30 | return available; 31 | } 32 | virtual int64_t Seek(int64_t offset, int whence) { 33 | 34 | int64_t retVal = m_stream->Seek(offset,whence); 35 | internalRead(); 36 | return retVal; 37 | } 38 | 39 | int internalRead(){ 40 | m_end = m_pos = m_pushbuf.end(); 41 | std::vector::difference_type n = m_stream->Read(&m_pushbuf.at(0),m_pushbuf.size() ); 42 | if( n > 0 ){ 43 | m_pos = m_pushbuf.begin(); 44 | m_end = m_pos + n; 45 | } 46 | return n; 47 | } 48 | 49 | public: 50 | CPushStream(std::shared_ptr stream,Converter &conv) 51 | :m_stream(stream), 52 | m_conv(conv) 53 | { 54 | m_pushbuf.resize(1024 * 1024); 55 | m_pos = m_end = m_pushbuf.end(); 56 | } 57 | 58 | 59 | void run(){ 60 | while(true){ 61 | if( m_pos == m_end ){ 62 | internalRead(); 63 | } 64 | m_conv.onData(); 65 | } 66 | } 67 | 68 | }; 69 | 70 | bool g_done = false; 71 | void handle_siginint(int){ 72 | g_done = true; 73 | } 74 | 75 | int main(int argc, const char * argv[]) { 76 | 77 | _S(ConverterAppInst::instance().init()); 78 | 79 | signal(SIGINT, handle_siginint); 80 | 81 | // while(!g_done){ 82 | 83 | std::shared_ptr inputStream( new FileStream() ),outputStream( new FileStream() ); 84 | 85 | std::string inputFileName (argv[1]); 86 | _S(inputStream->open(inputFileName,"r")); 87 | _S(outputStream->open(inputFileName + ".mp4","w")); 88 | 89 | Converter conv; 90 | 91 | std::shared_ptr ps( new CPushStream(std::static_pointer_cast(inputStream),conv) ); 92 | 93 | _S(conv.init(std::static_pointer_cast(inputStream), 94 | std::static_pointer_cast(outputStream) )); 95 | 96 | // ps->run(); 97 | conv.onData(); 98 | // } 99 | // insert code here... 100 | printf("exiting...\n"); 101 | return 0; 102 | } 103 | -------------------------------------------------------------------------------- /node_addons/FormatConverter/test/mp4writerTest.js: -------------------------------------------------------------------------------- 1 | var config = require('../../../common/Configuration'); 2 | const preserveOriginalHLS = config.get('preserveOriginalHLS'); 3 | preserveOriginalHLS.enable = false; 4 | config.set('preserveOriginalHLS',preserveOriginalHLS); 5 | var MP4WRITER = require('../../../lib/MP4WriteStream'); 6 | var fs = require('fs'); 7 | var http = require('http'); 8 | var Url = require('url'); 9 | var _ = require('underscore'); 10 | var util = require('util'); 11 | 12 | var tsFilePath = process.argv[2];//'/Users/igors/media-u4cc7m30h_b1496000_3383.ts';//"/Users/igors/dvr/dvrContentRootPath/1_abc123/1/media-ul0o1lom6_w1600782441_670.ts.mp4_saved.ts";//__dirname+'/../resources/media-uixh2a1qh_w1892051821_472.ts'; 13 | var httpPath = 'http://localhost/wn/media-uhe4wm3o6_b475136_144354218.ts'; 14 | console.log("tsFilePath=",tsFilePath); 15 | var ts2mp4 = new MP4WRITER.MP4WriteStream(tsFilePath, ""); 16 | 17 | var origUrl = { 18 | 'http:': function (url, cb) { 19 | var req = http.get(url); 20 | req.on('error', function (err) { 21 | console.log(err); 22 | }); 23 | req.on('response', function (responce, error) { 24 | if (error) { 25 | req.emit('error', error); 26 | return; 27 | } 28 | cb(response); 29 | req.end(); 30 | }); 31 | }, 32 | 'file:': function (url, cb) { 33 | var rs = fs.createReadStream(url.path); 34 | cb(rs); 35 | } 36 | }; 37 | 38 | 39 | var createPipe = function(url,cb){ 40 | var resolved = Url.parse(url); 41 | return origUrl[resolved.protocol ? resolved.protocol : 'file:'](resolved,cb); 42 | 43 | }; 44 | 45 | 46 | createPipe( tsFilePath, function(readable) { 47 | readable.pipe(ts2mp4) 48 | .on('data', function (chunk) { 49 | console.log("data " + chunk.length); 50 | }) 51 | .on('end', function (fileInfo) { 52 | console.log("end: %j",util.inspect(fileInfo)); 53 | }) 54 | .on('error', function (err) { 55 | console.log( "%s chunk %s", err, err.chunkName); 56 | }); 57 | }); 58 | -------------------------------------------------------------------------------- /node_addons/FormatConverter/test/playlistTest.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by igors on 3/26/16. 3 | */ 4 | var MP4WriteStream = require('../../../lib/MP4WriteStream'); 5 | var fs = require('fs'); 6 | var util = require('util'); 7 | 8 | var consoleLogger = { 9 | debug: console.log, 10 | info: console.log, 11 | warn: console.log, 12 | error: console.log 13 | }; 14 | 15 | var fileList = [ 16 | __dirname+'/../resources/media-uixh2a1qh_w1892051821_476.ts', 17 | __dirname+'/../resources/media-uixh2a1qh_w1892051821_477.ts', 18 | __dirname+'/../resources/media-uixh2a1qh_w1892051821_478.ts', 19 | __dirname+'/../resources/media-uixh2a1qh_w1892051821_479.ts', 20 | __dirname+'/../resources/media-uixh2a1qh_w1892051821_480.ts' 21 | ]; 22 | 23 | var processchunk = function(tsPath,cbDone) { 24 | var readable = fs.createReadStream(tsPath); 25 | var ts2mp4 = new MP4Writer(tsPath, consoleLogger); 26 | readable.pipe(ts2mp4) 27 | .on('data', function (chunk) { 28 | consoleLogger.debug("data " + chunk.length); 29 | }) 30 | .on('end', function (fileInfo) { 31 | consoleLogger.info("end"); 32 | cbDone(fileInfo); 33 | }) 34 | .on('error', function (err) { 35 | consoleLogger.warn(err); 36 | }); 37 | }; 38 | 39 | var palylist = []; 40 | 41 | var processNextChunk = function processNextChunk() { 42 | if(fileList.length) { 43 | var path = fileList.splice(0,1); 44 | processchunk(path[0], function (fi) { 45 | palylist.push(fi); 46 | processNextChunk(); 47 | }); 48 | } else { 49 | palylist.reduce( function(val,fi,index){ 50 | consoleLogger.info("fi[%d]=",index,util.inspect(fi)); 51 | var minDTS = Math.min(fi.videoFirstDTS, fi.audioFirstDTS), 52 | maxDTS = Math.max(fi.videoFirstDTS + fi.videoDuration, fi.audioFirstDTS + fi.audioDuration); 53 | if(val && val !== minDTS ) { 54 | consoleLogger.info("lastDTS=%d diff=%d ms", val, minDTS - val); 55 | } 56 | return maxDTS; 57 | },undefined); 58 | } 59 | }(); 60 | 61 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kLiveController", 3 | "version": "1.0.0", 4 | "repository": { 5 | "type": "git", 6 | "url": "git+https://github.com/kaltura/liveDVR.git" 7 | }, 8 | "devDependencies": { 9 | "chai": "^3.2.0", 10 | "chai-as-promised": "4.1.1", 11 | "chai-http": "^1.0.0", 12 | "grunt": "~0.4.5", 13 | "grunt-contrib-jshint": "~0.10.0", 14 | "grunt-contrib-nodeunit": "~0.4.1", 15 | "grunt-mocha-istanbul": "3.0.1", 16 | "grunt-mocha-test": "0.9.3", 17 | "istanbul": "0.3.18", 18 | "jshint-junit-reporter": "0.0.6", 19 | "matchdep": "~0.3.0", 20 | "mocha": "2.2.5", 21 | "mocha-html-reporter": "0.0.1", 22 | "mocha-junit-reporter": "^1.12.0", 23 | "proxyquire": "1.7.0", 24 | "rimraf": "2.4.3", 25 | "sinon": "1.9.0", 26 | "sinon-chai": "2.5.0", 27 | "tmp": "0.0.33" 28 | }, 29 | "dependencies": { 30 | "chunked-stream": "0.0.2", 31 | "commander": "2.19.0", 32 | "forever": "0.15.3", 33 | "glob": "7.1.3", 34 | "log4js": "4.0.2", 35 | "mkdirp": "0.5.1", 36 | "nconf": "^0.8.4", 37 | "q": "1.5.1", 38 | "q-io": "1.13.4", 39 | "request": "2.88.0", 40 | "socket.io-client": "2.2.0", 41 | "touch": "3.1.0", 42 | "underscore": "1.9.1", 43 | "nan": "2.12.1" 44 | }, 45 | "description": "DVR implementation built on top of live streams", 46 | "bugs": { 47 | "url": "https://github.com/kaltura/liveDVR/issues" 48 | }, 49 | "homepage": "https://github.com/kaltura/liveDVR#readme", 50 | "main": "./lib/App.js", 51 | "directories": { 52 | "test": "tests" 53 | }, 54 | "scripts": { 55 | "test": "mocha" 56 | }, 57 | "keywords": [], 58 | "author": "" 59 | } 60 | -------------------------------------------------------------------------------- /packager/bin/build_nginx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | currentDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 5 | 6 | echo ${currentDir} 7 | devRootDir=${devRootDir:-"${currentDir}/../build"} 8 | mkdir ${devRootDir} 9 | echo "${devRootDir}" 10 | 11 | ffmpegLibsDir=${ffmpegLibsDir:-${devRootDir}/liveDVR/node_addons/FormatConverter/build/FFmpeg} 12 | packagerDir="${devRootDir}/nginx-vod-module" 13 | nginxVersion=${nginxVersion:-1.11.0} 14 | nginxDir="${devRootDir}/nginx-${nginxVersion}" 15 | os_name=`uname` 16 | 17 | 18 | export LIB_AV_CODEC="${ffmpegLibsDir}/libavcodec/libavcodec__.a" 19 | export LIB_AV_FILTER="${ffmpegLibsDir}/libavfilter/libavfilter.a" 20 | export LIB_AV_UTIL=${ffmpegLibsDir}/libavutil/libavutil.a 21 | case ${os_name} in 22 | "Darwin") 23 | ;; 24 | "Linux") 25 | LIB_AV_FILTER="${LIB_AV_FILTER} -lpthread -lm -lrt" 26 | ;; 27 | esac 28 | 29 | if [ "$1" = "clean" ] 30 | then 31 | echo "Cleaning ${packagerDir} and ${nginxDir} directories" 32 | rm -rf ${packagerDir} 33 | rm -rf ${nginxDir} 34 | fi 35 | 36 | 37 | cd ${devRootDir} 38 | 39 | if which git &> /dev/null 40 | then 41 | if [ ! -d "${packagerDir}" ] 42 | then 43 | echo "${packagerDir} does not exist." 44 | git clone https://github.com/kaltura/nginx-vod-module || echo "error $?" 45 | fi 46 | cd ${packagerDir} 47 | git checkout master 48 | git pull 49 | fi 50 | 51 | cd ${devRootDir} 52 | 53 | if [ ! -d "${nginxDir}" ] 54 | then 55 | wget http://nginx.org/download/nginx-${nginxVersion}.tar.gz 56 | tar -zxvf nginx-${nginxVersion}.tar.gz 57 | rm nginx-${nginxVersion}.tar.gz -f 58 | fi 59 | 60 | 61 | echo "${nginxDir}" 62 | cd ${nginxDir} 63 | 64 | ./configure --with-http_secure_link_module --add-module=${packagerDir} --with-debug --with-cc-opt="-O0 -DDISABLE_PTS_DELAY_COMPENSATION" 65 | make 66 | #make install 67 | 68 | mkdir -p ${currentDir}/../../bin 69 | echo "Copying ${nginxDir}/objs/nginx to ${currentDir}/../../bin" 70 | cp ${nginxDir}/objs/nginx ${currentDir}/../../bin/ 71 | 72 | 73 | -------------------------------------------------------------------------------- /packager/bin/monitorKeyFrameAlignment.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # usage: bash /opt/kaltura/liveDVR/packager/bin/monitorKeyFrameAlignment.sh | awk -f /opt/kaltura/liveDVR/packager/bin/monitorKeyFrames.awk 3 | 4 | server=${1:-localhost} 5 | entryId=${entryId:-0_bvu1eq3g} 6 | urlPrefix=${1:-"http://$server:8080/hls/$entryId/playlist.json"} 7 | 8 | #echo "$urlPrefix/master.m3u8" 9 | 10 | indices=`curl "$urlPrefix/master.m3u8" | grep index` 11 | waitInterval=${waitInterval:-10} 12 | 13 | g_done=0 14 | 15 | trap "g_done=1" SIGINT SIGTERM 16 | 17 | #echo "indices=${indices[@]}" 18 | 19 | while [ "$g_done" -ne "1" ] 20 | do 21 | for index in ${indices[@]} 22 | do 23 | chunklist=`curl "$urlPrefix/$index" | grep "seg-"` 24 | for chunk in ${chunklist[@]} 25 | do 26 | echo "$chunk" 27 | done 28 | done 29 | sleep $waitInterval 30 | done -------------------------------------------------------------------------------- /packager/bin/monitorKeyFrames.awk: -------------------------------------------------------------------------------- 1 | BEGIN{ 2 | diffThresholdMsec=200 3 | flavorCnt=0 4 | } 5 | 6 | { 7 | callCnt++ 8 | n = split($0,b,"-"); 9 | segIdx=b[4] 10 | segTime=b[2] 11 | flavor=b[5] 12 | if(!(flavor in flavors)){ 13 | flavors[flavor]=1 14 | flavorCnt++ 15 | } 16 | #print "INFO: segIdx="segIdx" adding "segTime" for "flavor 17 | if(mins[segIdx]){ 18 | if( min[segIdx]>segTime){ 19 | min[segIdx]=segTime 20 | } else if( max[segIdx]diffThresholdMsec){ 24 | print "WARN: segIdx="segIdx" exceeded threshold "max[segIdx]-min[segIdx] 25 | } 26 | } else { 27 | min[segIdx]=segTime 28 | max[segIdx]=segTime 29 | } 30 | if(flavorCnt > 0 && callCnt == 10*flavorCnt ){ 31 | # for(seg in max){ 32 | # print "INFO: segIdx="seg" diff "max[seg]-min[seg] 33 | #} 34 | delete max 35 | delete min 36 | callCnt = 0 37 | } 38 | } -------------------------------------------------------------------------------- /packager/bin/replayLiveSession.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | entryDir=${entryDir:-/Users/igors/kLive_content/live/0_0tqdn8vw/} 4 | 5 | echo "entryDir=$entryDir" 6 | 7 | function errorCheck { 8 | local err=$? 9 | if [[ "$err" -ne "0" ]] 10 | then 11 | echo "error $err" 12 | exit $err 13 | fi 14 | } 15 | 16 | 17 | function Sleep { 18 | sleep $1 19 | #echo "" 20 | } 21 | 22 | last=0 23 | #files=("/Users/igors/kLive_content/live/0_0tqdn8vw/playlist.json_1470056933749" "/Users/igors/kLive_content/live/0_0tqdn8vw/playlist.json_1470056942873" ) 24 | files=${files:-`ls $entryDir/playlist.json_* | sort`} 25 | entryId=`basename $entryDir` 26 | 27 | for f in ${files[@]} 28 | do 29 | if [[ "$f" =~ (.*)_(.*)$ ]] 30 | then 31 | disp_time=${BASH_REMATCH[2]} 32 | playlistFileName=`basename $f` 33 | lines=(`curl http://localhost:8080/hls/$entryId/$playlistFileName/index-f1-v1-a1.m3u8`) 34 | if [[ "last" -ne "0" ]] 35 | then 36 | diff=$(((disp_time-last)/1000)) 37 | echo "$playlistFileName last=$last disp_time=$disp_time media-seq=$(((disp_time-946684800000)/10000)) sleeping $diff sec ${lines[3]} ${lines[4]}" 38 | Sleep $diff 39 | else 40 | echo "$playlistFileName last=$last disp_time=$disp_time media-seq=$(((disp_time-946684800000)/10000)) ${lines[3]} ${lines[4]}" 41 | Sleep 10 42 | fi 43 | echo "rename playlist.json" 44 | ln -sf "$f" $entryDir/playlist.json || errorCheck 45 | last=$disp_time 46 | fi 47 | done 48 | 49 | echo "done simulating" -------------------------------------------------------------------------------- /packager/bin/run_nginx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | dirname=`dirname $0` 5 | 6 | [[ ${dirname} =~ ^/ ]] || dirname="`pwd`/${dirname}" 7 | 8 | cd $dirname 9 | 10 | scriptName=`basename $0` 11 | 12 | confDir=${dirname}/../config/ 13 | 14 | echo "dirname=${dirname}" 15 | 16 | echo "confDir=${confDir}" 17 | 18 | nginxPath="${dirname}/../../bin/nginx" 19 | 20 | contentDir=`cat "${dirname}/../../common/config/config.json" | awk '$0 ~ /rootFolderPath/ { printf substr($2,2,length($2)-8) }'` 21 | 22 | if [ -z "${contentDir}" ] 23 | then 24 | echo "could not infer contentDir!" 25 | exit 1 26 | fi 27 | 28 | echo "contentDir = ${contentDir}" 29 | 30 | wwwDir="${dirname}/../www" 31 | echo wwwDir = ${wwwDir} 32 | port=${2:-8080} 33 | 34 | rm -rf "/var/tmp/*nginx.conf*" 35 | 36 | echo "copying ${confDir}*" 37 | for file in ${confDir}* ; do 38 | filename=${file##*/} 39 | newFile="/var/tmp/${filename/.template/}" 40 | echo "${file}" to "${newFile}" 41 | sed -e "s#@CONTENT_DIR@#${contentDir}/#" -e "s#@PORT@#${port}#" -e "s#@WWW_DIR@#${wwwDir}#" ${file} > ${newFile} 42 | done 43 | 44 | function getNginxPids(){ 45 | ps -fA | grep nginx | grep -vE "grep|${scriptName}" | awk '{print $2}' 46 | } 47 | 48 | processes=(`getNginxPids`) 49 | 50 | echo "processes=$processes" 51 | 52 | for p in ${processes[@]} 53 | do 54 | echo "killing ${p}" 55 | kill -9 ${p} 56 | done 57 | 58 | echo "running ${nginxPath} -c /var/tmp/nginx.conf" 59 | 60 | nginxDir="/usr/local/nginx" 61 | 62 | [ -d "${nginxDir}" ] || mkdir -p ${nginxDir} 63 | 64 | [ -d "${nginxDir}/logs" ] || mkdir -p "${nginxDir}/logs" 65 | 66 | ${nginxPath} -c /var/tmp/nginx.conf & 67 | 68 | echo "nginx log dir is: ${nginxDir}/logs" 69 | 70 | echo "nginx pid(s) is:" 71 | getNginxPids 72 | -------------------------------------------------------------------------------- /packager/bin/test.sh: -------------------------------------------------------------------------------- 1 | clear 2 | (( $# )) && entryId=$1 || entryId=0_simstream1 3 | partnerId=123 4 | address=http://localhost:8080 5 | set -x 6 | echo HLS: 7 | curl $address/live/hls/p/$partnerId/e/$entryId/master.m3u8 -i 8 | curl $address/live/hls/p/$partnerId/e/$entryId/index-f1.m3u8 -i 9 | echo HDS 10 | curl $address/live/hds/p/$partnerId/e/$entryId/manifest.f4m -i 11 | curl $address/live/hds/p/$partnerId/e/$entryId/bootstrap-v1-a1.abst -i 12 | echo MSS 13 | curl $address/live/mss/p/$partnerId/e/$entryId/manifest -i 14 | echo DASH 15 | curl $address/live/dash/p/$partnerId/e/$entryId/manifest.mpd -i 16 | 17 | 18 | 19 | echo HLS legacy : 20 | curl $address/kLive/smil:${entryId}_all.smil/master.m3u8 -i 21 | curl $address/kLive/smil:${entryId}_all.smil/1/index.m3u8 -i 22 | curl $address/kLive/smil:${entryId}_all.smil/manifest.f4m -i 23 | curl $address/kLive/smil:${entryId}_all.smil/bootstrap-v1-a1.abst -i -------------------------------------------------------------------------------- /packager/config/cors.common.conf: -------------------------------------------------------------------------------- 1 | if ($request_method = 'OPTIONS') { 2 | add_header 'Access-Control-Allow-Origin' '*'; 3 | add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; 4 | # 5 | # Custom headers and headers various browsers *should* be OK with but aren't 6 | # 7 | add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; 8 | # 9 | # Tell client that this pre-flight info is valid for 20 days 10 | # 11 | add_header 'Access-Control-Max-Age' 1728000; 12 | add_header 'Content-Type' 'text/plain charset=UTF-8'; 13 | add_header 'Content-Length' 0; 14 | return 204; 15 | } 16 | if ($request_method = 'POST') { 17 | add_header 'Access-Control-Allow-Origin' '*'; 18 | add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; 19 | add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; 20 | add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; 21 | } 22 | if ($request_method = 'GET') { 23 | add_header 'Access-Control-Allow-Origin' '*'; 24 | add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; 25 | add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; 26 | add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; 27 | } 28 | -------------------------------------------------------------------------------- /packager/config/nginx.conf.live.bootstrap.template: -------------------------------------------------------------------------------- 1 | if ($playlist = '') { 2 | set $playlist "playlist.json"; 3 | } 4 | 5 | alias @CONTENT_DIR@/$live_type/$entryHash/$entryId/$playlist; 6 | include cors.common.conf; -------------------------------------------------------------------------------- /packager/config/nginx.conf.live.conf.template: -------------------------------------------------------------------------------- 1 | # shared vod settings 2 | vod_upstream_extra_args "clientTag=vod:$hostname-$request_id"; 3 | vod_max_mapping_response_size 20m; 4 | vod_mode mapped; 5 | vod_live_window_duration 0; 6 | vod_align_segments_to_key_frames on; 7 | vod_hls_absolute_master_urls off; 8 | vod_hls_absolute_index_urls off; 9 | vod_hls_mpegts_output_id3_timestamps on; 10 | vod_dash_profiles urn:mpeg:dash:profile:isoff-live:2011; 11 | expires off; 12 | vod_upstream_location ""; 13 | 14 | location ~ \/(?live|recording|live-stg|clip)\/[^\/]+\/p\/(?\d+)\/e\/(?[^\/]+(?\w))\/(sd\/(?\d+)\/)?(type\/(?\w+)\/)?t\/(?[^\/]+)\/ { 15 | #this is for backward compatability to fetch from wowza 16 | location ~ ^/m/(?[^/]+)/live/legacy/p/\d+/e/[^/]+(/sd/[^/]+)?/t/[^/]+/(?.*) { 17 | proxy_pass http://${mediaServer}:1935/kLive/ngrp:${entryId}_1_all/$path$is_args$args; 18 | } 19 | 20 | location ~ /live|live-stg/ { 21 | vod_media_set_override_json $overrideClipTo; 22 | location ~ /sd/2000/ { 23 | vod_expires_live_time_dependent 1; 24 | vod_live_mapping_cache live_mapping_cache_low_latency 256m 1; 25 | vod_live_response_cache live_response_cache_low_latency 256m 1; 26 | vod_segment_duration 2000; 27 | include nginx.conf.live.protocols; 28 | } 29 | location ~ /sd/4000/ { 30 | vod_expires_live_time_dependent 1; 31 | vod_segment_duration 4000; 32 | include nginx.conf.live.protocols; 33 | } 34 | location ~ /sd/6000/ { 35 | vod_expires_live_time_dependent 2; 36 | vod_segment_duration 6000; 37 | include nginx.conf.live.protocols; 38 | } 39 | location ~ ^/ { 40 | include nginx.conf.live.protocols; 41 | } 42 | 43 | } 44 | location ~ /recording/ { 45 | #override default caching expiration for to 1m 46 | vod_expires_live 1m; 47 | vod_expires_live_time_dependent 1m; 48 | 49 | #override to do one continuous recording (no gaps) 50 | vod_force_continuous_timestamps on; 51 | include nginx.conf.live.protocols; 52 | } 53 | 54 | location ~ /clip/ { 55 | vod_force_playlist_type_vod on; 56 | #override to do one continuous recording (no gaps) 57 | vod_force_continuous_timestamps on; 58 | vod_expires 1d; 59 | include nginx.conf.live.protocols; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /packager/config/nginx.conf.live.protocols.template: -------------------------------------------------------------------------------- 1 | location ~ /hls/ { 2 | include nginx.conf.live.bootstrap; 3 | vod hls; 4 | } 5 | 6 | location ~ /ehls/ { 7 | include nginx.conf.live.bootstrap; 8 | vod hls; 9 | vod_secret_key "@LIVE_ENCRYPT_HLS_KEY@ $vod_suburi"; 10 | vod_hls_encryption_method aes-128; 11 | } 12 | 13 | 14 | location ~ /hds/ { 15 | include nginx.conf.live.bootstrap; 16 | vod hds; 17 | } 18 | 19 | location ~ /dash/ { 20 | include nginx.conf.live.bootstrap; 21 | vod dash; 22 | } 23 | 24 | location ~ /mss/ { 25 | include nginx.conf.live.bootstrap; 26 | vod mss; 27 | } 28 | -------------------------------------------------------------------------------- /packager/config/nginx.conf.template: -------------------------------------------------------------------------------- 1 | 2 | #user nobody; 3 | 4 | worker_processes 1; 5 | 6 | #uncomment for debugging 7 | #daemon off; 8 | 9 | #error_log logs/error.log; 10 | #error_log logs/error.log notice; 11 | 12 | error_log logs/error.log debug; 13 | 14 | #pid logs/nginx.pid; 15 | 16 | events { 17 | worker_connections 1024; 18 | } 19 | 20 | http { 21 | #include mime.types; 22 | default_type application/octet-stream; 23 | types { 24 | text/xml xml; 25 | } 26 | 27 | #log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 28 | # '$status $body_bytes_sent "$http_referer" ' 29 | # '"$http_user_agent" "$http_x_forwarded_for"'; 30 | 31 | #access_log logs/access.log main; 32 | 33 | sendfile on; 34 | keepalive_timeout 65; 35 | 36 | #gzip on; 37 | 38 | map $user_type $overrideClipTo{ 39 | "user" ""; 40 | default '{"clipFrom":0, "clipTo":-1}'; 41 | } 42 | 43 | server { 44 | listen @PORT@; 45 | server_name localhost; 46 | 47 | #web server access 48 | #rewrite_log on; 49 | #error_log /usr/local/nginx/logs/example.com.error.log debug; 50 | 51 | location = /serverip { 52 | 53 | expires 1d; 54 | return 200 "Kaltura"; 55 | } 56 | 57 | include "nginx.conf.live.conf"; 58 | 59 | location / { 60 | expires 1d; 61 | root @WWW_DIR@; 62 | } 63 | 64 | # vod status page 65 | location = /vod_status { 66 | vod_status; 67 | access_log off; 68 | 69 | } 70 | } 71 | 72 | 73 | server { 74 | listen 8081; 75 | vod_mode local; 76 | 77 | vod_ignore_edit_list on; 78 | 79 | # vod status page 80 | location = /vod_status { 81 | vod_status; 82 | access_log off; 83 | 84 | } 85 | vod_metadata_cache metadata_cache 512m; 86 | vod_response_cache response_cache 128m; 87 | location ~ ^/manifest/(.*) { 88 | alias /$1; 89 | add_header Access-Control-Allow-Headers '*'; 90 | add_header Access-Control-Expose-Headers 'Server,range,Content-Length,Content-Range'; 91 | add_header Access-Control-Allow-Methods 'GET, HEAD, OPTIONS'; 92 | add_header Access-Control-Allow-Origin '*'; 93 | } 94 | 95 | location ~ ^/serve/(.*)/hls/ { 96 | alias /$1; 97 | vod hls; 98 | 99 | add_header Access-Control-Allow-Headers '*'; 100 | add_header Access-Control-Expose-Headers 'Server,range,Content-Length,Content-Range'; 101 | add_header Access-Control-Allow-Methods 'GET, HEAD, OPTIONS'; 102 | add_header Access-Control-Allow-Origin '*'; 103 | } 104 | } 105 | 106 | } -------------------------------------------------------------------------------- /packager/www/clientaccesspolicy.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /packager/www/crossdomain.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /serviceWrappers/defaults/kLiveController: -------------------------------------------------------------------------------- 1 | # Kaltura liveController defaults file 2 | LOG_DIR=/var/log 3 | PID_DIR=/var/run 4 | USE_NVM="1" 5 | KLIVE_CONTROLLER_PREFIX=/opt/kaltura/liveController/latest 6 | NODE_PATH="${KLIVE_CONTROLLER_PREFIX}/node_modules" -------------------------------------------------------------------------------- /serviceWrappers/defaults/kLiveController.template: -------------------------------------------------------------------------------- 1 | LOG_DIR=@LOG_DIR@ 2 | PID_DIR=@RUN_DIR@ 3 | USE_NVM=@USE_NVM@ 4 | KLIVE_CONTROLLER_PREFIX=@KLIVE_CONTROLLER_PREFIX@ 5 | NODE_PATH=@NODE_PATH@ -------------------------------------------------------------------------------- /tests/recording/recording-spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by lilach.maliniak on 18/08/2016. 3 | */ 4 | var proxyquire = require('proxyquire'); 5 | var chai = require('chai'); 6 | var expect = chai.expect; 7 | var sinon = require('sinon'); 8 | var Q = require('q'); 9 | var should = chai.should(); 10 | var util=require('util'); 11 | var ControllerWrapper = require('./../regression/ControllerWrapper'); 12 | 13 | var testsuite = process.env.TEST_CLASS ? process.env.TEST_CLASS : 'hls-recording'; 14 | 15 | 16 | describe('Live-Recording (for regression)', function() { 17 | 18 | it (testsuite, function(done) { 19 | var controller = new ControllerWrapper('HLS stream recording'); 20 | controller.start() 21 | .then(function(exit_code) { 22 | expect(exit_code).to.equal(0); 23 | }).then(function(){ 24 | done(); 25 | }).done(null, function(err){ 26 | done(err); 27 | }); 28 | }); 29 | }); 30 | 31 | -------------------------------------------------------------------------------- /tests/recording/test.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import threading 3 | import time 4 | 5 | def log_subprocess_output(process, title): 6 | header = "[{}] [pid={}]".format(title, process.pid) 7 | while True: 8 | nextline = process.stdout.readline() 9 | 10 | if nextline == '' and process.poll() is not None: 11 | break 12 | print "["+threading.current_thread().getName()+"] "+ header + nextline 13 | 14 | 15 | #if nextline.find("Input contains NaN/+-Inf")!=-1: 16 | # exit(-5); 17 | 18 | 19 | def koko(): 20 | try: 21 | 22 | convertor = "/Users/guyjacubovski/Library/Developer/Xcode/DerivedData/ts_to_mp4_convertor-cknvyndxytuyqqcmnjvycauwzvax/Build/Products/Debug/ts_to_mp4_convertor" 23 | process = subprocess.Popen(convertor+" 1_bgtg1a6k_1_cmux5n8c_3946546_f32_out.ts f32_out.mp4 eng", cwd=r'/Users/guyjacubovski/Downloads', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 24 | 25 | log_subprocess_output(process, "ffmpeg: ts->mp4") 26 | 27 | output, outerr = process.communicate() 28 | exitcode = process.returncode 29 | 30 | if exitcode is 0: 31 | print "["+threading.current_thread().getName()+"] Successfully finished TS -> MP4 conversion" 32 | else: 33 | status = 'failed' 34 | error = 'Failed to convert TS -> MP4. Convertor process exit code {}, {}'.format(exitcode, outerr) 35 | print "["+threading.current_thread().getName()+"] " + error 36 | 37 | raise subprocess.CalledProcessError(exitcode, "a") 38 | 39 | except (OSError, subprocess.CalledProcessError) as e: 40 | print "["+threading.current_thread().getName()+"] Failed to convert TS -> MP4 {}",format(str(e)) 41 | raise e 42 | except Exception as e: 43 | print "["+threading.current_thread().getName()+"] Failed to convert TS -> MP4 {}".format(str(e)) 44 | raise e 45 | 46 | jobs = [] 47 | 48 | for i in range(0, 15): 49 | thread = threading.Thread(name=i,target=koko) 50 | jobs.append(thread) 51 | 52 | for j in jobs: 53 | j.start() 54 | 55 | stopped = False 56 | 57 | def watch(): 58 | while not stopped: 59 | for j in jobs: 60 | if j.isAlive(): 61 | print j.getName() 62 | time.sleep( 5 ) 63 | 64 | watchdog_thread = threading.Thread(name=i,target=watch) 65 | watchdog_thread.start(); 66 | # Ensure all of the threads have finished 67 | for j in jobs: 68 | j.join() 69 | 70 | 71 | print "haleluja" 72 | stopped=True 73 | watchdog_thread.join(); -------------------------------------------------------------------------------- /tests/regression/regression-spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by lilach.maliniak on 15/08/2016. 3 | */ 4 | 5 | var proxyquire = require('proxyquire'); 6 | var chai = require('chai'); 7 | var expect = chai.expect; 8 | var sinon = require('sinon'); 9 | var should = chai.should(); 10 | var ControllerWrapper = require('./ControllerWrapper'); 11 | 12 | var testsuite = process.env.TEST_CLASS ? process.env.TEST_CLASS : 'hls-regression-test'; 13 | 14 | 15 | 16 | describe('Live-Regression', function() { 17 | 18 | it (testsuite, function(done) { 19 | var controller = new ControllerWrapper("regression test"); 20 | controller.start() 21 | .then(function(exit_code) { 22 | expect(exit_code).to.equal(0); 23 | }).then(function(){ 24 | done(); 25 | }).done(null, function(err){ 26 | done(err); 27 | }); 28 | 29 | }); 30 | 31 | }); 32 | 33 | -------------------------------------------------------------------------------- /tests/resources/crash.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaltura/liveDVR/0f12e06e7ab139cbce4313ddc289311c0056f468/tests/resources/crash.ts -------------------------------------------------------------------------------- /tests/resources/crash2.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaltura/liveDVR/0f12e06e7ab139cbce4313ddc289311c0056f468/tests/resources/crash2.ts -------------------------------------------------------------------------------- /tests/resources/decreasing_pts.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaltura/liveDVR/0f12e06e7ab139cbce4313ddc289311c0056f468/tests/resources/decreasing_pts.ts -------------------------------------------------------------------------------- /tests/resources/erronousManifest.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-VERSION:3 3 | #EXT-X-ALLOW-CACHE:NO 4 | #EXT-X-TARGETDURATION:13 5 | #EXT-X-MEDIA-SEQUENCE:11 6 | #EXTINF:8.9990, 7 | media-uefvqmelj_b1017600_11.ts 8 | #EXTINF:11.9990, 9 | media-uefvqmelj_b1017600_12.ts 10 | #EXTINF **** ERROR *** :9.0000, 11 | media-uefvqmelj_b1017600_13.ts 12 | #EXTINF:8.9990, 13 | media-uefvqmelj_b1017600_14.ts 14 | #EXTINF:11.9990, 15 | media-uefvqmelj_b1017600_15.ts 16 | #EXTINF:8.9990, 17 | media-uefvqmelj_b1017600_16.ts 18 | #EXTINF:9.0000, 19 | media-uefvqmelj_b1017600_17.ts 20 | #EXTINF:11.9990, 21 | media-uefvqmelj_b1017600_18.ts 22 | #EXTINF:8.9990, 23 | media-uefvqmelj_b1017600_19.ts 24 | #EXTINF:8.9990, 25 | media-uefvqmelj_b1017600_20.ts 26 | -------------------------------------------------------------------------------- /tests/resources/flavor-downloader-data/simpleManifest.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-VERSION:3 3 | #EXT-X-ALLOW-CACHE:NO 4 | #EXT-X-TARGETDURATION:13 5 | #EXT-X-MEDIA-SEQUENCE:11 6 | #EXTINF:8.9990, 7 | media-uefvqmelj_b1017600_11.ts 8 | #EXTINF:11.9990, 9 | media-uefvqmelj_b1017600_12.ts 10 | #EXTINF:9.0000, 11 | media-uefvqmelj_b1017600_13.ts 12 | #EXTINF:8.9990, 13 | media-uefvqmelj_b1017600_14.ts 14 | #EXTINF:11.9990, 15 | media-uefvqmelj_b1017600_15.ts 16 | -------------------------------------------------------------------------------- /tests/resources/manifestWithDiscontinuity.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-VERSION:3 3 | #EXT-X-ALLOW-CACHE:NO 4 | #EXT-X-TARGETDURATION:13 5 | #EXT-X-MEDIA-SEQUENCE:11 6 | #EXTINF:8.9990, 7 | media-uefvqmelj_b1017600_11.ts 8 | #EXTINF:11.9990, 9 | media-uefvqmelj_b1017600_12.ts 10 | #EXTINF:9.0000, 11 | media-uefvqmelj_b1017600_13.ts 12 | #EXTINF:8.9990, 13 | media-uefvqmelj_b1017600_14.ts 14 | #EXTINF:11.9990, 15 | media-xxfvqmelj_b1017600_15.ts 16 | #EXT-X-DISCONTINUITY 17 | #EXTINF:8.9990, 18 | media-xxfvqmelj_b1017600_0ts 19 | #EXTINF:9.0000, 20 | media-xxfvqmelj_b1017600_1.ts 21 | #EXTINF:11.9990, 22 | media-xxfvqmelj_b1017600_2.ts 23 | #EXTINF:8.9990, 24 | media-xxfvqmelj_b1017600_3.ts 25 | #EXTINF:8.9990, 26 | media-xxfvqmelj_b1017600_4.ts 27 | -------------------------------------------------------------------------------- /tests/resources/media-u4cc7m30h_b1496000_3383.ts.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaltura/liveDVR/0f12e06e7ab139cbce4313ddc289311c0056f468/tests/resources/media-u4cc7m30h_b1496000_3383.ts.mp4 -------------------------------------------------------------------------------- /tests/resources/simpleManifest.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-VERSION:3 3 | #EXT-X-ALLOW-CACHE:NO 4 | #EXT-X-TARGETDURATION:13 5 | #EXT-X-MEDIA-SEQUENCE:11 6 | #EXTINF:8.9990, 7 | media-uefvqmelj_b1017600_11.ts 8 | #EXTINF:11.9990, 9 | media-uefvqmelj_b1017600_12.ts 10 | #EXTINF:9.0000, 11 | media-uefvqmelj_b1017600_13.ts 12 | #EXTINF:8.9990, 13 | media-uefvqmelj_b1017600_14.ts 14 | #EXTINF:11.9990, 15 | media-uefvqmelj_b1017600_15.ts 16 | #EXTINF:8.9990, 17 | media-uefvqmelj_b1017600_16.ts 18 | #EXTINF:9.0000, 19 | media-uefvqmelj_b1017600_17.ts 20 | #EXTINF:11.9990, 21 | media-uefvqmelj_b1017600_18.ts 22 | #EXTINF:8.9990, 23 | media-uefvqmelj_b1017600_19.ts 24 | #EXTINF:8.9990, 25 | media-uefvqmelj_b1017600_20.ts 26 | -------------------------------------------------------------------------------- /tests/resources/simpleManifestWithEndList.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-VERSION:3 3 | #EXT-X-ALLOW-CACHE:NO 4 | #EXT-X-TARGETDURATION:13 5 | #EXT-X-MEDIA-SEQUENCE:11 6 | #EXT-X-PLAYLIST-TYPE:VOD 7 | #EXTINF:8.9990, 8 | media-uefvqmelj_b1017600_11.ts 9 | #EXTINF:11.9990, 10 | media-uefvqmelj_b1017600_12.ts 11 | #EXTINF:9.0000, 12 | media-uefvqmelj_b1017600_13.ts 13 | #EXTINF:8.9990, 14 | media-uefvqmelj_b1017600_14.ts 15 | #EXTINF:11.9990, 16 | media-uefvqmelj_b1017600_15.ts 17 | #EXTINF:8.9990, 18 | media-uefvqmelj_b1017600_16.ts 19 | #EXTINF:9.0000, 20 | media-uefvqmelj_b1017600_17.ts 21 | #EXTINF:11.9990, 22 | media-uefvqmelj_b1017600_18.ts 23 | #EXTINF:8.9990, 24 | media-uefvqmelj_b1017600_19.ts 25 | #EXTINF:8.9990, 26 | media-uefvqmelj_b1017600_20.ts 27 | #EXT-X-ENDLIST -------------------------------------------------------------------------------- /tests/resources/ufhwdejgz-1.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaltura/liveDVR/0f12e06e7ab139cbce4313ddc289311c0056f468/tests/resources/ufhwdejgz-1.ts -------------------------------------------------------------------------------- /tests/resources/updatedManifest1.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-VERSION:3 3 | #EXT-X-ALLOW-CACHE:NO 4 | #EXT-X-TARGETDURATION:14 5 | #EXT-X-MEDIA-SEQUENCE:21 6 | #EXT-X-DISCONTINUITY 7 | #EXTINF:13.3000, 8 | uriName1 9 | #EXTINF:12.3000, 10 | uriName2 11 | #EXTINF:4.0000, 12 | uriName3 13 | -------------------------------------------------------------------------------- /tests/resources/updatedManifest1_withoutDiscontinuity.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-VERSION:3 3 | #EXT-X-ALLOW-CACHE:NO 4 | #EXT-X-TARGETDURATION:14 5 | #EXT-X-MEDIA-SEQUENCE:21 6 | #EXTINF:13.3000, 7 | uriName1 8 | #EXTINF:12.3000, 9 | uriName2 10 | #EXTINF:4.0000, 11 | uriName3 12 | -------------------------------------------------------------------------------- /tests/streaming_client/streaming_client.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /tests/unit/BackendClientFactory-spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/26/2015. 3 | */ 4 | /*jshint -W030 */ // Ignore "Expected an assignment or function call and instead saw an expression" warning wrongfully raised for chai expectations 5 | 6 | var proxyquire = require('proxyquire'); 7 | var chai = require('chai'); 8 | var expect = chai.expect; 9 | var sinon = require('sinon'); 10 | 11 | describe('Backend client factory spec', function() { 12 | 13 | var createBackendClientFactory = function(customizeMocks){ 14 | 15 | configurationMock = { 16 | get : sinon.stub() 17 | }; 18 | 19 | var mocks = { 20 | './Configuration' : configurationMock 21 | }; 22 | 23 | if (customizeMocks) { 24 | customizeMocks(mocks); 25 | } 26 | 27 | var backendClientFactoryCtor = proxyquire('../../lib/BackendClientFactory', mocks); 28 | return backendClientFactoryCtor; 29 | }; 30 | 31 | it('should return mock in case configuration mandates so', function() 32 | { 33 | var mocks; 34 | var backendClientFactory = createBackendClientFactory(function(m){ 35 | mocks = m; 36 | }); 37 | 38 | mocks['./Configuration'].get.withArgs('mockBackend').returns(true); 39 | var backendClient = backendClientFactory.getBackendClient(); 40 | expect(backendClient.getLiveEntriesForMediaServer).to.not.be.undefined; 41 | }); 42 | 43 | it('should return real client in case configuration mandates so', function() 44 | { 45 | var mocks; 46 | var backendClientFactory = createBackendClientFactory(function(m){ 47 | mocks = m; 48 | }); 49 | 50 | mocks['./Configuration'].get.withArgs('mockBackend').returns(false); 51 | 52 | var backendClient = backendClientFactory.getBackendClient(); 53 | expect(backendClient.getLiveEntriesForMediaServer).to.not.be.undefined; 54 | }); 55 | }); -------------------------------------------------------------------------------- /tests/unit/NetworkClientFactory-spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/26/2015. 3 | */ 4 | /*jshint -W030 */ // Ignore "Expected an assignment or function call and instead saw an expression" warning wrongfully raised for chai expectations 5 | 6 | var proxyquire = require('proxyquire'); 7 | var chai = require('chai'); 8 | var expect = chai.expect; 9 | var sinon = require('sinon'); 10 | 11 | describe('Network client factory spec', function() { 12 | 13 | var createNetworkClientFactory = function(customizeMocks){ 14 | 15 | configurationMock = { 16 | get : sinon.stub() 17 | }; 18 | 19 | var mocks = { 20 | './Configuration' : configurationMock 21 | }; 22 | 23 | if (customizeMocks) { 24 | customizeMocks(mocks); 25 | } 26 | 27 | var networkClientFactoryCtor = proxyquire('../../lib/NetworkClientFactory', mocks); 28 | return networkClientFactoryCtor; 29 | }; 30 | 31 | it('should return mock in case configuration mandates so', function() 32 | { 33 | var mocks; 34 | var networkClientFactory = createNetworkClientFactory(function(m){ 35 | mocks = m; 36 | }); 37 | 38 | mocks['./Configuration'].get.withArgs('mockNetwork').returns(true); 39 | var networkClient = networkClientFactory.getNetworkClient(); 40 | expect(networkClient.read).to.not.be.undefined; 41 | }); 42 | 43 | it('should return real client in case configuration mandates so', function() 44 | { 45 | var mocks; 46 | var networkClientFactory = createNetworkClientFactory(function(m){ 47 | mocks = m; 48 | }); 49 | 50 | mocks['./Configuration'].get.withArgs('mockNetwork').returns(false); 51 | 52 | var networkClient = networkClientFactory.getNetworkClient(); 53 | expect(networkClient.read).to.not.be.undefined; 54 | }); 55 | }); 56 | -------------------------------------------------------------------------------- /tests/unit/PersistenceFormat-spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/23/2015. 3 | */ 4 | 5 | var proxyquire = require('proxyquire'); 6 | var chai = require('chai'); 7 | var expect = chai.expect; 8 | var path = require('path'); 9 | var sinon = require('sinon'); 10 | 11 | describe('PersistenceFormat spec', function() { 12 | 13 | var configMock = { 14 | get : sinon.stub().returns("/home/dev/DVR") 15 | }; 16 | 17 | var mocks = { 18 | './Configuration' : configMock 19 | }; 20 | 21 | it('should get a path for an entry', function(){ 22 | var persistenceFormat = proxyquire('../../common/PersistenceFormat', mocks); 23 | var entryDestPath = persistenceFormat.getEntryFullPath('1_bla'); 24 | expect(entryDestPath).to.equal(path.join('/home/dev/DVR', '1_bla')); 25 | }); 26 | 27 | it('should get a path for a flavor', function(){ 28 | var persistenceFormat = proxyquire('../../common/PersistenceFormat', mocks); 29 | var entryDestPath = persistenceFormat.getFlavorFullPath('1_bla', 400000); 30 | expect(entryDestPath).to.equal(path.join('/home/dev/DVR', '1_bla', '400000')); 31 | }); 32 | }); -------------------------------------------------------------------------------- /tests/unit/promise-m3u8-spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elad.benedict on 8/23/2015. 3 | */ 4 | 5 | var proxyquire = require('proxyquire'); 6 | var fs = require('fs'); 7 | var chai = require('chai'); 8 | var expect = chai.expect; 9 | var path = require('path'); 10 | var sinon = require('sinon'); 11 | 12 | describe('Promise m3u8 spec', function() { 13 | 14 | var createParser = function(customizeMocks){ 15 | 16 | var readStreamMock = { 17 | on : sinon.stub() 18 | }; 19 | 20 | var mocks = { 21 | 'fs' : { 22 | createReadStream : sinon.stub().returns(readStreamMock) 23 | } 24 | }; 25 | 26 | readStreamMock.on.withArgs('error').callsArgWithAsync(1, new Error("Whoops!")); 27 | 28 | if (customizeMocks) { 29 | customizeMocks(mocks); 30 | } 31 | 32 | var parser = proxyquire('../../lib/manifest/promise-m3u8', mocks); 33 | return parser; 34 | }; 35 | 36 | it('should correctly read an M3U8', function(done) 37 | { 38 | var m3u8Parser = require('../../lib/manifest/promise-m3u8'); 39 | var expectedManifest = fs.readFileSync(path.join(__dirname, '/../resources/simpleManifest.m3u8'), 'utf8'); 40 | 41 | m3u8Parser.parseM3U8(path.join(__dirname, '/../resources/simpleManifest.m3u8')).then(function(manifest){ 42 | expect(manifest.toString()).to.eql(expectedManifest.replace(/[\r]/g, '')); 43 | }).done(function(){ 44 | done(); 45 | }); 46 | }); 47 | 48 | it('should correctly read from a stream', function(done) 49 | { 50 | var m3u8Parser = require('../../lib/manifest/promise-m3u8'); 51 | var expectedManifest = fs.readFileSync(path.join(__dirname, '/../resources/simpleManifest.m3u8'), 'utf8'); 52 | 53 | var Readable = require('stream').Readable; 54 | var s = new Readable(); 55 | s.push(expectedManifest); 56 | s.push(null); // Stream end 57 | 58 | m3u8Parser.parseM3U8(s).then(function(manifest){ 59 | expect(manifest.toString()).to.eql(expectedManifest.replace(/[\r]/g, '')); 60 | }).done(function(){ 61 | done(); 62 | }); 63 | }); 64 | 65 | it('should return a failed promise when cannot read the referenced path', function(done) 66 | { 67 | var m3u8Parser = createParser(); 68 | m3u8Parser.parseM3U8("nonExistantPath").then(function(){ 69 | expect.fail(); 70 | done(new Error("Stream should not be created successfully")); 71 | }, function(){ 72 | done(); 73 | }); 74 | }); 75 | }); --------------------------------------------------------------------------------