├── CallUpload.sh ├── README.md ├── checkNodeFailed.sh ├── frontend ├── ajax.php ├── conn.php ├── index.php └── index1.php ├── supervisord.conf ├── transcode-master.sh ├── transcode-nodes.sh └── transcoding.sql /CallUpload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | export PATH=$PATH:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin 4 | echo "${1}" > /tmp/filename 5 | MYSQL="/usr/bin/mysql --skip-column-names -utranscode -ptranscode -h 172.18.10.111 transcoding -e" 6 | 7 | FILE="${1}" 8 | FILENAME=$(basename "${FILE}") 9 | FILEPATH=${FILE%/*} 10 | PRODUCT=$(echo "${FILE}"| cut -d"/" -f4); 11 | 12 | for i in $(ls /srv/workers/ | grep -v $(hostname)); do 13 | rsync --timeout=30 -f"+ */" -f"- *" -rRvz "${FILEPATH}" ${i}:/ 14 | rsync --rsh="ssh -c arcfour256,arcfour128,blowfish-cbc,aes128-ctr,aes192-ctr,aes256-ctr" -Rv "${FILE}" ${i}:/ ; 15 | done 16 | 17 | # The length of each segment is the total length, divided by the total number of worker nodes 18 | TOTALNODES=$(ls /srv/workers/ | wc -l) 19 | #TOTALNODES=4 20 | 21 | # Determine the duration (length) of the video 22 | #echo ORIGINIAL VIDEO LENGTH IS: $TIME 23 | #HHMMSS=$(ffprobe "${FILE}" 2>&1 | /bin/grep Duration: | /bin/sed -e "s/^.*Duration: //" -e "s/\..*$//") 24 | TIME=$(mplayer -identify -frames 0 -vo null -nosound "${FILE}" 2>&1 | awk -F= '/LENGTH/{print $2}') 25 | 26 | ## First MS 27 | MS=$(echo $TIME |cut -d'.' -f2) 28 | SECONDS=$(echo $TIME |cut -d'.' -f1) 29 | 30 | if [ ${MS} -gt 0 ]; then 31 | MS1=$(echo "0.${MS} * 1000000" | bc) 32 | fi 33 | 34 | ## Main Seconds 35 | SEC1=$(echo "scale = 3; ${SECONDS} / ${TOTALNODES}" | bc) 36 | 37 | SEC1MS=$(echo ${SEC1} |cut -d'.' -f2) 38 | SEC1SECONDS=$(echo ${SEC1} |cut -d'.' -f1) 39 | 40 | if [ ${SEC1MS} -gt 0 ]; then 41 | SEC1MS1=$(echo "0.${SEC1MS} * 1000000" | bc |cut -d'.' -f1) 42 | fi 43 | 44 | MSPERNODE=$(echo "${MS1} / ${TOTALNODES}" | bc | cut -d'.' -f1) 45 | 46 | if [ ! -z ${MSPERNODE} ] && [ ! -z ${SEC1MS1} ]; then 47 | FINALMICRO=$(echo "${MSPERNODE} + ${SEC1MS1}" | bc) 48 | else 49 | [ ! -z ${MSPERNODE} ] && FINALMICRO=$(echo ${MSPERNODE}|cut -d'.' -f1) 50 | [ ! -z ${SEC1MS1} ] && FINALMICRO=$(echo ${SEC1MS1}|cut -d'.' -f1) 51 | fi 52 | 53 | NODELENGTH="${SEC1SECONDS}.${FINALMICRO}" 54 | #echo TOTAL VIDEO LENGTH IS: $(echo "${LENGTH} * 4" |bc) 55 | 56 | LENGTH="$(date -d@${SEC1SECONDS} -u +%H:%M:%S).${FINALMICRO}" 57 | 58 | # Convert that HH:MM:SS.xxx to seconds 59 | #SECOND=$(/bin/date -u -d "1970-01-01 ${HHMMSS}" +"%s") 60 | 61 | # Calculate each node's start time 62 | if [ ${SEC1SECONDS} -gt 0 ]; then 63 | JOBID=$($MYSQL "INSERT INTO transcoding.jobs (filename, filepath, duration, nodecount, product) VALUES ('${FILENAME}', '${FILEPATH}/', '${DURATION}', '${TOTALNODES}', '${PRODUCT}');SELECT LAST_INSERT_ID();") 64 | TOTALNODES=$((TOTALNODES-1)) 65 | for i in $(seq 0 $TOTALNODES); do 66 | #STIME=$(echo "${NODELENGTH} * ${i}" | bc) 67 | STARTTIME="$(date -d@$(echo "${NODELENGTH} * ${i}" | bc |cut -d'.' -f1) -u +%H:%M:%S).$(echo "${NODELENGTH} * ${i}" | bc |cut -d'.' -f2)" 68 | # Insert into jobs table 69 | $MYSQL "INSERT INTO transcoding.queue (jobid, starttime, length) VALUES ('${JOBID}', '${STARTTIME}', '${LENGTH}');" 70 | done 71 | fi 72 | echo "${1}" >> /tmp/abrt_filename -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Distributed Multi-bitrate Video Transcoding 2 | Distributed Multi-bitrate Video Transcoding On Centos / Ubuntu / Suse / RedHat (Bash Scripts) 3 | 4 | Multi-bitrate Video processing requires lots of computing power and time to process full movie. There are different open source video transcoding and processing tools freely available in Linux, like libav-tools, ffmpeg, mencoder, and handbrake. However, none of these tools support **PARALLEL** computing easily. 5 | 6 | After some research, I found amazing [solution](http://blog.dustinkirkland.com/2014/07/scalable-parallel-video-transcoding-on.html) designed by '[Dustin Kirkland](http://blog.dustinkirkland.com/2014/07/scalable-parallel-video-transcoding-on.html)' based on Ubuntu JUJU and [avconv](https://libav.org/avconv.html). But our requirement was little bit diffrent from Dustins's solution. Our requirement was to convert single video in Multi-bitrate and in formats like 3gp, flv and upload them to single or multiple CDN(like Akamai or tata). Also we want to build this solution on top of CentOS and ffmpeg. So I decided to develop "Simple Scalable, Parallel, Multi-bitrate Video Transcoding System" by myself. Here is my solution. 7 | 8 | The Algorithm is same as Dustin's solution but with some changes: 9 | 10 | 1. Upload file to FTP. After a successful upload CallUploadScript(pure-ftpd function) will call script: 11 | - Script is responsible for syncing files to all nodes(Disabled ssh encryptions to speed up transfer) 12 | - Divide video duration by number of nodes available to process and add start time, length to MySQL queue table. 13 | - Updating duration, file path, filename of video and number of nodes available for transcoding to MySQL 14 | 2. Transcode Nodes will pick jobs from the queue 15 | 3. Each Node will then process their segments of video and raise a flag when done 16 | 4. Master nodes will wait for each of the all-done flags, and then any master worker will pick the job to concatenate the result 17 | 5. Upload converted files to different CDN 18 | 19 | # Fault Tolerant 20 | Making this process fault tolerant to node failures, I have written small script checkNodeFailed.sh, which will check for failed nodes and will try to reassign that job to another node. We need to add every minute cron run this. 21 | 22 | # Pre-requisites: 23 | 1. bc 24 | 2. nproc 25 | 3. ffmpeg 26 | 4. mysql 27 | 5. mysql-server(For master node) 28 | 6. mplayer 29 | 7. rsync 30 | 8. Password less ssh login 31 | 9. nfs server and client 32 | 10. supervisord 33 | 11. ffprobe 34 | 35 | # Installation: 36 | 37 | 1. Install ffmpeg(Click [here](http://wiki.razuna.com/display/ecp/FFMpeg+Installation+on+CentOS+and+RedHat) for instruction) 38 | 2. Download and copy all scripts(.sh files) to /srv directory 39 | 3. Change file permission to 755 40 | 4. Install Pure-FTPD and change CallUploadscript directive to yes in /etc/pure-ftpd.conf file 41 | 5. Create test user for FTP and set password 42 | 43 | `# useradd -m ftptest; passwd ftptest` 44 | 45 | 6. Run below commands to change pure-ftpd init script 46 | 47 | `# sed -i 's#start() {#start() {\n\t/usr/sbin/pure-uploadscript -B -r /srv/CallUpload.sh#g' /etc/init.d/pure-ftpd` 48 | 49 | `# sed -i 's#stop() {#stop() {\n\tkillall -9 pure-uploadscript#g' /etc/init.d/pure-ftpd` 50 | 51 | 7. restart pure-ftp service 52 | 8. Make sure to Change Database IP in all three scripts (DB_IP variable) 53 | 9. Install mysql-server and import SQL file 'transcoding.sql'. Create 'transcode' user with password same as username. Make sure user is able to connect from all of the worker nodes. 54 | 10. NFS Export /srv directory and mount it on all nodes with NFS client option "lookupcache=none" 55 | 56 | 11. On all servers install supervisord and copy supervisord.conf from download directory to /etc/supervisord.conf. Restart supervisord service. 57 | 58 | 12. Add every minute cron for checkNodeFailed.sh script. 59 | */1 * * * * /srv/checkNodeFailed.sh 60 | 61 | 13. To check the status of jobs you may use the dashboard. Copy frontend folder to your apache DocumentRoot. In my case its /var/www/html/ 62 | 63 | `# cp -a frontend/ /var/www/html/ ` 64 | -------------------------------------------------------------------------------- /checkNodeFailed.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | MYSQL="/usr/bin/mysql --skip-column-names -utranscode -ptranscode -h 172.18.10.111 transcoding -e" 3 | 4 | JOB_ID=$(${MYSQL} "SELECT jobid FROM jobs WHERE jobcomplete> /var/log/failed-node.log 15 | ${MYSQL} "UPDATE queue SET node=NULL,status=0 WHERE id = ${i};" 16 | ${MYSQL} "UPDATE jobs SET jobcount=jobcount-1 WHERE jobid = ${j};" 17 | fi 18 | fi 19 | done 20 | sleep 2 21 | done 22 | fi -------------------------------------------------------------------------------- /frontend/ajax.php: -------------------------------------------------------------------------------- 1 | 0 && !empty($id)){ 32 | $updateQuery = "UPDATE jobs SET job_status = 999 WHERE job_id = ".$id; 33 | $res = mysqli_query($_Link, $updateQuery); 34 | 35 | $queryAppend = ''; 36 | if (isset($p) && $p != '') { 37 | $queryAppend = " AND cp = '" . $p . "'"; 38 | } 39 | 40 | echo failure($_Link, $queryAppend); 41 | } 42 | } 43 | 44 | function in_process($_Link, $queryAppend) 45 | { 46 | /* INPROCESS */ 47 | $sql = "SELECT * FROM jobs WHERE job_status < (no_nodes+1) AND node_failed is null AND job_failed is null" . $queryAppend; 48 | $res = mysqli_query($_Link, $sql); 49 | $html = ""; 50 | while ($rec = mysqli_fetch_array($res)) { 51 | if($rec['curmaster'] != NULL ){ 52 | $html .= "
" . $rec['name'] . " (" . $rec['cp'] . ") : Master is Processing ( ".$rec['curmaster']." )
"; 53 | }elseif($rec['no_nodes'] > $rec['job_status']){ 54 | $html .= "
" . $rec['name'] . " (" . $rec['cp'] . ") : Node is Processing
"; 55 | }else{ 56 | $html .= "
" . $rec['name'] . " (" . $rec['cp'] . ") is Waiting for Master to takeover
"; 57 | } 58 | } 59 | return $html; 60 | } 61 | 62 | 63 | function completed($_Link, $queryAppend) 64 | { 65 | /* COMPLETED */ 66 | $sql = "SELECT * FROM jobs WHERE job_status=(no_nodes+1) AND node_failed is null AND job_failed is null" . $queryAppend." order by job_id desc"; 67 | $html = ""; 68 | $res = mysqli_query($_Link, $sql); 69 | while ($rec = mysqli_fetch_array($res)) { 70 | $html .= "
" . $rec['name'] . " (" . $rec['cp'] . ") is Successfully Completed
"; 71 | } 72 | return $html; 73 | 74 | } 75 | 76 | /* FAILURE */ 77 | function failure($_Link, $queryAppend) 78 | { 79 | $sql = "SELECT * FROM jobs WHERE job_status < (no_nodes+1) AND (node_failed is not null or job_failed is not null)" . $queryAppend; 80 | $html = ""; 81 | $res = mysqli_query($_Link, $sql); 82 | while ($rec = mysqli_fetch_array($res)) { 83 | $html .= "
" . $rec['name'] . " (" . $rec['cp'] . ") is Failed : Clear
"; 84 | $html .= ""; 92 | } 93 | return $html; 94 | } 95 | 96 | ?> 97 | 98 | -------------------------------------------------------------------------------- /frontend/conn.php: -------------------------------------------------------------------------------- 1 | 9 | 10 | 11 | 12 | 13 | Video Transcoding Farm - Network18 14 | 15 | 16 | 17 | 18 | 19 | 58 | 59 | 60 | 61 | 62 |
63 | 68 |
69 | " . $rec['name'] . " (" . $rec['cp'] . ") is in Process
"; 74 | } 75 | */?> 76 |
77 |
78 | " . $rec['name'] . " (" . $rec['cp'] . ") is Successfully Completed
"; 83 | } 84 | */?> 85 | 86 |
87 | " . $rec['name'] . " (" . $rec['cp'] . ") is Failed
"; 92 | echo ""; 100 | } 101 | 102 | 103 | */?> 104 | 105 | 106 | 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /frontend/index1.php: -------------------------------------------------------------------------------- 1 | 8 | 9 | 10 | 11 | 12 | 13 | Video Transcoding Farm - Network18 14 | 15 | 16 | 17 | 18 | 47 | 48 | 49 | 50 | 51 |
52 | 57 |
58 | " . $rec['name'] . " (" . $rec['cp'] . ") is in Process
"; 63 | } 64 | ?> 65 |
66 |
67 | " . $rec['name'] . " (" . $rec['cp'] . ") is Successfully Completed
"; 72 | } 73 | ?> 74 | 75 |
76 | " . $rec['name'] . " (" . $rec['cp'] . ") is Failed
"; 81 | echo ""; 89 | } 90 | 91 | 92 | ?> 93 | 94 | 95 | 96 | 97 | 98 | 99 | -------------------------------------------------------------------------------- /supervisord.conf: -------------------------------------------------------------------------------- 1 | 2 | [supervisord] 3 | ;http_port=/var/tmp/supervisor.sock ; (default is to run a UNIX domain socket server) 4 | http_port=0.0.0.0:9001 ; (alternately, ip_address:port specifies AF_INET) 5 | ;sockchmod=0700 ; AF_UNIX socketmode (AF_INET ignore, default 0700) 6 | ;sockchown=nobody.nogroup ; AF_UNIX socket uid.gid owner (AF_INET ignores) 7 | ;umask=022 ; (process file creation umask;default 022) 8 | logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) 9 | logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) 10 | logfile_backups=10 ; (num of main logfile rotation backups;default 10) 11 | loglevel=info ; (logging level;default info; others: debug,warn) 12 | pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) 13 | nodaemon=false ; (start in foreground if true;default false) 14 | minfds=1024 ; (min. avail startup file descriptors;default 1024) 15 | minprocs=200 ; (min. avail process descriptors;default 200) 16 | 17 | ;nocleanup=true ; (don't clean up tempfiles at start;default false) 18 | http_username=user ; (default is no username (open system)) 19 | http_password=user ; (default is no password (open system)) 20 | ;childlogdir=/tmp ; ('AUTO' child log dir, default $TEMP) 21 | ;user=chrism ; (default is current user, required if root) 22 | ;directory=/tmp ; (default is not to cd during start) 23 | ;environment=KEY=value ; (key value pairs to add to environment) 24 | 25 | [supervisorctl] 26 | ;serverurl=unix:///var/tmp/supervisor.sock ; use a unix:// URL for a unix socket 27 | serverurl=http://0.0.0.0:9001 ; use an http:// url to specify an inet socket 28 | ;username=user ; should be same as http_username if set 29 | ;password=user ; should be same as http_password if set 30 | ;prompt=mysupervisor ; cmd line prompt (default "supervisor") 31 | 32 | ; The below sample program section shows all possible program subsection values, 33 | ; create one or more 'real' program: sections to be able to control them under 34 | ; supervisor. 35 | 36 | [program:Node] 37 | command=/srv/transcode-nodes.sh 38 | autostart=true ; start at supervisord start (default: true) 39 | startsecs=10 ; number of secs prog must stay running (def. 10) 40 | exitcodes=0,1,2 ; 'expected' exit codes for process (default 0,2) 41 | stopsignal=TERM ; signal used to kill process (default TERM) 42 | log_stdout=true ; if true, log program stdout (default true) 43 | log_stderr=true ; if true, log program stderr (def false) 44 | logfile=/home/node.log ; child log path, use NONE for none; default AUTO 45 | logfile_maxbytes=50MB ; max # logfile bytes b4 rotation (default 50MB) 46 | logfile_backups=10 ; # of logfile backups (default 10) 47 | 48 | [program:Master] 49 | command=/srv/transcode-master.sh 50 | autostart=true ; start at supervisord start (default: true) 51 | startsecs=10 ; number of secs prog must stay running (def. 10) 52 | exitcodes=0,1,2 ; 'expected' exit codes for process (default 0,2) 53 | stopsignal=TERM ; signal used to kill process (default TERM) 54 | log_stdout=true ; if true, log program stdout (default true) 55 | log_stderr=true ; if true, log program stderr (def false) 56 | logfile=/home/master.log ; child log path, use NONE for none; default AUTO 57 | logfile_maxbytes=50MB ; max # logfile bytes b4 rotation (default 50MB) 58 | logfile_backups=10 ; # of logfile backups (default 10) -------------------------------------------------------------------------------- /transcode-master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | export PATH=$PATH:/root/bin 6 | 7 | # Exit Traps 8 | function finish { 9 | rm -fv /tmp/master 10 | rm -fv /srv/masters/$(hostname) 11 | echo "Error: Master Service Down" >> /var/log/master.log 12 | } 13 | trap finish EXIT 14 | 15 | CPUNO=$(cat /proc/cpuinfo |grep processor|wc -l) 16 | FORMAT="mp4" 17 | BITRATE="128000 256000 512000 712000" # Used in Rsync command, if changed change in rsync also 18 | DB_IP="172.18.10.111" 19 | MYSQL="mysql --skip-column-names -utranscode -ptranscode -h ${DB_IP} transcoding -e" 20 | 21 | while true; do 22 | mkdir -p /srv/masters 23 | date > /srv/masters/$(hostname) 24 | CURRENT_NODE=$(ls /srv/masters/ | grep -B99999 $(hostname) | wc -l) 25 | 26 | if [ ! -s /tmp/nodes ]; then 27 | echo "Master" > /tmp/master 28 | 29 | # look for work 30 | JOB_ID=$(${MYSQL} "SELECT jobid FROM jobs WHERE jobcomplete=nodecount AND error IS NULL AND master IS NULL LIMIT 1;") 31 | 32 | if [ ! -z ${JOB_ID} ]; then 33 | 34 | IPADDR=$(hostname -i) 35 | ${MYSQL} "UPDATE jobs SET master = concat(ifnull(master,''), '${IPADDR},') where jobid = ${JOB_ID};" 36 | 37 | GETCURMASTER=$(${MYSQL} "SELECT master FROM jobs WHERE jobid = '${JOB_ID}'"| cut -d"," -f1); 38 | if [ ${GETCURMASTER} != ${IPADDR} ]; then 39 | rm -fv /tmp/master 40 | echo "JOB Already In process"; 41 | sleep 5; 42 | else 43 | 44 | # Get Filename with extension and Filename w/o extension 45 | FILEWEXT=$(${MYSQL} "SELECT filename FROM jobs WHERE jobid = ${JOB_ID};") 46 | FILEWOEXT=${FILEWEXT%.*} 47 | CURDATE=$(${MYSQL} "SELECT starttime FROM jobs WHERE jobid = '${JOB_ID}'"| cut -d " " -f1) 48 | 49 | # Set job dirctory 50 | FILEPATH=$(${MYSQL} "SELECT filepath FROM jobs WHERE jobid = ${JOB_ID};") 51 | FTPPATH=$(echo ${FILEPATH} | cut -d"/" -f-4) 52 | #REMOTEPATH=$(echo ${FILEPATH} | cut -d"/" -f5- | sed 's/ /\\ /g') 53 | REMOTEPATH=$(echo ${FILEPATH} | cut -d"/" -f5-) 54 | 55 | # Check for Content Provider Details 56 | CONTPROVIDER=$(echo "${FILEPATH}"| cut -d"/" -f4) 57 | 58 | # Set output directory and create it 59 | #SHOWNAME=$(echo ${FILEPATH} | awk -F'/' {'print $9'}) 60 | 61 | OUTPATH="/video-process/processed/${CONTPROVIDER}/${CURDATE}/${FILEWOEXT}/" 62 | mkdir -p "${OUTPATH}/LOGS/" 63 | 64 | ERROR=0; 65 | 66 | QUEUEID=$(${MYSQL} "SELECT id from queue WHERE jobid = ${JOB_ID} AND status = '2';") 67 | WORKERS=$(${MYSQL} "SELECT node from queue WHERE jobid = ${JOB_ID} AND status = '2';"|grep -v ${IPADDR} |sort -u) 68 | 69 | for i in ${WORKERS}; do 70 | IP=$(echo $i| cut -d',' -f1); 71 | echo "######### Syncing files back to master node from ${i} #########" >> "${OUTPATH}${FILEWOEXT}-sync.log.txt" 2>&1 72 | ESCOUTPATH=$(echo ${OUTPATH}|sed 's/ /\\ /g') 73 | 74 | if ! rsync --rsh="ssh -c arcfour256,arcfour128,blowfish-cbc,aes128-ctr,aes192-ctr,aes256-ctr" -av "${IP}:${ESCOUTPATH}" "${OUTPATH}"/ >> "${OUTPATH}${FILEWOEXT}-sync.log.txt" 2>&1 ; then 75 | ERROR=1 76 | ERRORLOG="File Sync To master Failed ," 77 | break 78 | fi 79 | done 80 | 81 | if [ $ERROR -ne '0' ]; then 82 | ${MYSQL} "UPDATE jobs SET error = '${ERRORLOG}' where jobid = ${JOB_ID};" 83 | echo "Error: Job Failed: ${JOB_ID} Log: ${ERRORLOG}" >> /var/log/master.log 84 | ERRORLOG= 85 | rm -fv /tmp/master 86 | sleep 5; 87 | else 88 | for i in ${WORKERS}; do 89 | IP=$(echo $i| cut -d',' -f1); 90 | ssh ${IP} "rm -fvr ${ESCOUTPATH}" >> "${OUTPATH}${FILEWOEXT}-delete.log.txt" 2>&1 || echo "OUTPATH remove from worker failed" 91 | done 92 | 93 | for b in ${BITRATE}; do 94 | CONCAT=;CONCAT="/dev/null" 95 | for i in ${QUEUEID}; do 96 | if [ -e "${OUTPATH}${FILEWOEXT}.part${i}-${b}.ts" ]; then 97 | CONCAT="${CONCAT}|${OUTPATH}${FILEWOEXT}.part${i}-${b}.ts" 98 | fi 99 | done 100 | 101 | # Write the command to a log file 102 | echo "ffmpeg -i concat:${CONCAT} -c copy -bsf:a aac_adtstoasc -y ${OUTPATH}${FILEWOEXT}-${b}.${FORMAT}" > "${OUTPATH}${FILEWOEXT}-${b}.log.txt" 103 | 104 | # Concatenate the clips together 105 | if ! ffmpeg -i concat:"${CONCAT}" -c copy -bsf:a aac_adtstoasc -y "${OUTPATH}${FILEWOEXT}-${b}.${FORMAT}" >> "${OUTPATH}${FILEWOEXT}-${b}.log.txt" 2>&1; then 106 | ERROR=1 107 | ERRORLOG="${ERRORLOG} Concatenation Failed: ${b} Bitrate " 108 | break; 109 | fi 110 | done 111 | 112 | # .mp4 file 113 | cp -v "${OUTPATH}${FILEWOEXT}-512000.${FORMAT}" "${OUTPATH}${FILEWOEXT}.${FORMAT}" >> "${OUTPATH}${FILEWOEXT}.log.txt" 2>&1 114 | if ! /usr/local/bin/MP4Box -tmp /video-process/tmp/ -hint "${OUTPATH}${FILEWOEXT}.${FORMAT}" >> "${OUTPATH}${FILEWOEXT}.log.txt" 2>&1; then 115 | ERROR=1 116 | ERRORLOG="${ERRORLOG} MP4 Hinting Failed" 117 | fi 118 | 119 | if [ $ERROR -ne '0' ]; then 120 | ${MYSQL} "UPDATE jobs SET error = '${ERRORLOG}' where jobid = ${JOB_ID};" 121 | echo "Error: Job Failed: ${JOB_ID} Log: ${ERRORLOG}" >> /var/log/master.log 122 | ERRORLOG= 123 | rm -fv /tmp/master 124 | else 125 | # 3gp Conversion 126 | if ! ffmpeg -threads ${CPUNO} -i "${OUTPATH}${FILEWOEXT}-128000.${FORMAT}" -r 12.00 -b:v 128k -s 176x144 -vcodec h263 -acodec libfaac -ab 20k -ar 44100 -y "${OUTPATH}${FILEWOEXT}.3gp" >> "${OUTPATH}${FILEWOEXT}-3gp.log.txt" 2>&1; then 127 | ERROR=1 128 | ERRORLOG="${ERRORLOG} Conversion Failed: 3gp " 129 | fi 130 | 131 | # FLV Conversion 132 | echo "ffmpeg -threads ${CPUNO} -i ${OUTPATH}${FILEWOEXT}-128000.${FORMAT} -vcodec copy -acodec copy -y ${OUTPATH}${FILEWOEXT}.flv" > "${OUTPATH}${FILEWOEXT}-flv.log.txt" 133 | 134 | if ! ffmpeg -threads ${CPUNO} -i "${OUTPATH}${FILEWOEXT}-128000.${FORMAT}" -vcodec copy -acodec copy -y "${OUTPATH}${FILEWOEXT}.flv" >> "${OUTPATH}${FILEWOEXT}-flv.log.txt" 2>&1; then 135 | ERROR=1 136 | ERRORLOG="${ERRORLOG} Conversion Failed: flv " 137 | fi 138 | 139 | # Thumbnail generation 140 | TMPVIDEOLEN=$(ffprobe "${OUTPATH}${FILEWOEXT}.${FORMAT}" 2>&1 | /bin/grep Duration: | /bin/sed -e "s/^.*Duration: //" -e "s/\..*$//") 141 | VIDEOLEN=$(/bin/date -u -d "1970-01-01 ${TMPVIDEOLEN}" +"%s") 142 | #VIDEOLEN=$(expr ${VIDEOLEN} - 10) 143 | MODVIDEOLEN=$((${VIDEOLEN} % 10)) 144 | if [ ${MODVIDEOLEN} -ne 0 ]; then 145 | VIDEOLEN=$(((10 - ${VIDEOLEN} % 10) + ${VIDEOLEN})) 146 | fi 147 | 148 | TMPFRAME=$(expr ${VIDEOLEN} / 10) 149 | SNAPFRAME=$(expr ${TMPFRAME} + 1) 150 | 151 | TOTALFRAMES=$(ffprobe -select_streams v -show_streams "${OUTPATH}${FILEWOEXT}.${FORMAT}" 2>/dev/null | grep nb_frames | sed -e 's/nb_frames=//') 152 | THUMBNAILVAL=$(expr "${TOTALFRAMES} / ${SNAPFRAME}" | bc) 153 | 154 | if ! ffmpeg -threads ${CPUNO} -ss 10 -i "${OUTPATH}${FILEWOEXT}-712000.${FORMAT}" -f image2 -vf "thumbnail=${THUMBNAILVAL},scale=120:96,tile=12x10" -pix_fmt yuvj420p -an -vsync 0 -y "${OUTPATH}${FILEWOEXT}-120x69-thumb-%03d.jpg" >> "${OUTPATH}${FILEWOEXT}-snap.log.txt" 2>&1; then 155 | ERROR=1 156 | ERRORLOG="${ERRORLOG} Thumbnail generation Failed: 120x69 " 157 | fi 158 | 159 | if ! ffmpeg -threads ${CPUNO} -ss 10 -i "${OUTPATH}${FILEWOEXT}-712000.${FORMAT}" -f image2 -vf "thumbnail=${THUMBNAILVAL},scale=80:44,tile=12x10" -pix_fmt yuvj420p -an -vsync 0 -y "${OUTPATH}${FILEWOEXT}-80x44-thumb-%03d.jpg" >> "${OUTPATH}${FILEWOEXT}-snap.log.txt" 2>&1; then 160 | ERROR=1 161 | ERRORLOG="${ERRORLOG} Thumbnail generation Failed: 80x44 " 162 | fi 163 | 164 | # Check for error 165 | if [ $ERROR -ne '0' ]; then 166 | ${MYSQL} "UPDATE jobs SET error = '${ERRORLOG}' where jobid = ${JOB_ID};" 167 | echo "Error: Job Failed: ${JOB_ID} Log: ${ERRORLOG}" >> /var/log/master.log 168 | ERRORLOG= 169 | rm -fv /tmp/master 170 | else 171 | # Update coversion end time 172 | ${MYSQL} "UPDATE jobs SET conversiontime = current_timestamp where jobid = ${JOB_ID};" 173 | rm -fv /tmp/master 174 | 175 | # Cleaning all .ts and logs 176 | rm -f "${OUTPATH}"/*.ts >> "${OUTPATH}${FILEWOEXT}-delete.log.txt" 2>&1 || echo "removal of .ts failed" 177 | mv -f "${OUTPATH}"/*.txt "${OUTPATH}/LOGS/" || echo "Move log files to LOGS deirectory" 178 | 179 | # Start upload to akamai storage 180 | SYNCERROR=0; 181 | #while read line; do 182 | # CPHOST=$(echo "$line"|awk '{print $4}'); 183 | # CPUSER=$(echo "$line"|awk '{print $2}'); 184 | # CPPASS=$(echo "$line"|awk '{print $3}'); 185 | # export RSYNC_PASSWORD=${CPPASS}; 186 | # 187 | # echo '######### Syncing folder structure to CDN #########' >> "${OUTPATH}/LOGS/${FILEWOEXT}-sync.log.txt" 2>&1 188 | # # Sync local folder structure to remote location 189 | # if ! rsync --timeout=30 -f"+ */" -f"- *" -avz "${FTPPATH}"/ "${CPUSER}@${CPHOST}::${CPUSER}/" >> "${OUTPATH}/LOGS/${FILEWOEXT}-sync.log.txt" 2>&1; then 190 | # SYNCERROR=1; 191 | # fi 192 | # 193 | # echo '######### Syncing files to CDN #########' >> "${OUTPATH}/LOGS/${FILEWOEXT}-sync.log.txt" 2>&1 194 | # if ! rsync --timeout=30 --progress --exclude=LOGS -avz "${OUTPATH}/" "${CPUSER}@${CPHOST}::${CPUSER}/${REMOTEPATH}/" >> "${OUTPATH}/LOGS/${FILEWOEXT}-sync.log.txt" 2>&1; then 195 | # SYNCERROR=1; 196 | # fi 197 | #done < <(${MYSQL} "SELECT * FROM cpdetails WHERE cp = '${CONTPROVIDER}';") 198 | 199 | if [ $SYNCERROR -ne '0' ]; then 200 | ERRORLOG="${ERRORLOG} Rsync Failed " 201 | ${MYSQL} "UPDATE jobs SET error = '${ERRORLOG}' where jobid = ${JOB_ID};" 202 | echo "Error: Job Failed: ${JOB_ID} Log: ${ERRORLOG}" >> /var/log/master.log 203 | ERRORLOG= 204 | rm -fv /tmp/master 205 | else 206 | #rm -fv "${OUTPATH}"/*.mp4 >> "${OUTPATH}/LOGS/${FILEWOEXT}-delete.log.txt" 2>&1 || echo "Remove .mp4 Failed" 207 | #rm -fv "${OUTPATH}"/*.3gp >> "${OUTPATH}/LOGS/${FILEWOEXT}-delete.log.txt" 2>&1 || echo "Remove .3gp Failed" 208 | #rm -fv "${OUTPATH}"/*.flv >> "${OUTPATH}/LOGS/${FILEWOEXT}-delete.log.txt" 2>&1 || echo "Remove .flv Failed" 209 | #rm -fv "${OUTPATH}"/*.jpg >> "${OUTPATH}/LOGS/${FILEWOEXT}-delete.log.txt" 2>&1 || echo "Remove .jpg Failed" 210 | #rm -fv "${FILEPATH}/${FILEWEXT}" >> "${OUTPATH}/LOGS/${FILEWOEXT}-delete.log.txt" 2>&1 || echo "Remove Source File Failed" 211 | 212 | for i in ${WORKERS}; do 213 | IP=$(echo $i| cut -d',' -f1); 214 | echo "######### Removing source file from ${i} #########" >> "${OUTPATH}/LOGS/${FILEWOEXT}-delete.log.txt" 2>&1 215 | #ssh ${IP} "rm -fv '${FILEPATH}/${FILEWEXT}'" >> "${OUTPATH}/LOGS/${FILEWOEXT}-delete.log.txt" 2>&1 || echo "Source file remove from worker failed" 216 | done 217 | 218 | # Clear previously failed job if exists 219 | #${MYSQL} "UPDATE jobs SET jobcomplete = 999 WHERE name LIKE '${FILEWEXT}' AND (error IS NOT NULL OR error IS NOT NULL);" 220 | # Update job status, so that the other workers know when its done 221 | ${MYSQL} "UPDATE jobs SET jobcomplete=jobcomplete+1 WHERE jobid = ${JOB_ID};" 222 | # Update end time 223 | ${MYSQL} "UPDATE jobs SET totaltime = current_timestamp where jobid = ${JOB_ID};" 224 | #${MYSQL} "UPDATE jobs SET jobcomplete = 999 WHERE name LIKE '${FILEWEXT}' AND (error IS NOT NULL OR error IS NOT NULL);" 225 | rm -fv /tmp/master 226 | fi 227 | fi 228 | fi 229 | fi 230 | fi 231 | else 232 | rm -fv /tmp/master 233 | echo "Hooorrray.. No jobs to Process :)"; 234 | sleep 5; 235 | fi 236 | else 237 | echo "Node process is running.." 238 | rm -fv /tmp/master 239 | sleep 5 240 | fi 241 | done 242 | exit 0 243 | -------------------------------------------------------------------------------- /transcode-nodes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | export PATH=$PATH:/root/bin 6 | 7 | # Exit Traps 8 | function finish { 9 | rm -f "/srv/workers/$(hostname)" 10 | rm -fv /tmp/nodes 11 | echo "Error: Nodes Service Down" >> /var/log/nodes.log 12 | } 13 | trap finish EXIT 14 | 15 | CPUNO=$(cat /proc/cpuinfo |grep processor|wc -l) 16 | FORMAT="mp4" 17 | BITRATE="128 256 512 712" # Used in Rsync command, if changed change in rsync also 18 | DB_IP="172.18.10.111" 19 | MYSQL="mysql --skip-column-names -utranscode -ptranscode -h ${DB_IP} transcoding -e" 20 | IPADDR=$(hostname -i) 21 | 22 | while true; do 23 | mkdir -p /srv/workers 24 | date > /srv/workers/$(hostname) 25 | CURRENT_NODE=$(ls /srv/workers/ | grep -B99999 $(hostname) | wc -l) 26 | CURRENT_NODE=$((CURRENT_NODE-1)) 27 | 28 | if [ ! -s /tmp/master ]; then 29 | # look for work 30 | JOB_ID=$(${MYSQL} "SELECT jobid FROM jobs WHERE jobcomplete /tmp/nodes 37 | # Update current node number to node_id field 38 | #UPDATECNT=${MYSQL} "UPDATE queue SET node = '${IPADDR}',status = '1' WHERE id = ${QUEUEID} AND status = '0' limit 1;SELECT ROW_COUNT();" 39 | UPDATECNT=$(${MYSQL} "UPDATE queue SET node = concat(ifnull(node,''), '${IPADDR},'),status = '1' WHERE id = ${QUEUEID} AND status = '0' limit 1;SELECT ROW_COUNT();") 40 | 41 | if [ ${UPDATECNT} -eq 1 ]; then 42 | # Check if this is first node to pick job for processing, If yes then update start time 43 | TMP1=$(${MYSQL} "SELECT jobcount FROM jobs WHERE jobid = ${JOB_ID};") 44 | if [ "${TMP1}" = "0" ]; then 45 | ${MYSQL} "UPDATE jobs SET starttime = current_timestamp where jobid = ${JOB_ID};" 46 | fi 47 | ${MYSQL} "UPDATE jobs SET jobcount=jobcount+1 where jobid = ${JOB_ID};" 48 | 49 | CURDATE=$(date +%F) 50 | # Get Filename with extension and Filename w/o extension 51 | FILEPATH=$(${MYSQL} "SELECT filepath FROM jobs WHERE jobid = ${JOB_ID};") 52 | FILEWEXT=$(${MYSQL} "SELECT filename FROM jobs WHERE jobid = ${JOB_ID};") 53 | FILEWOEXT=${FILEWEXT%.*} # filename only w/o extension 54 | 55 | # Get Content Provider 56 | CONTPROVIDER=$(echo "${FILEPATH}"| cut -d"/" -f4) 57 | 58 | # Set output directory and create it 59 | if echo ${FILEPATH} | grep -q '/mc_gujrati_videos/'; then 60 | SHOWNAME=$(echo ${FILEPATH} | awk -F'/' {'print $10'}) 61 | else 62 | SHOWNAME=$(echo ${FILEPATH} | awk -F'/' {'print $9'}) 63 | fi 64 | 65 | OUTPATH="/video-process/processed/${CONTPROVIDER}/${CURDATE}/${FILEWOEXT}/" 66 | mkdir -p "${OUTPATH}" 67 | 68 | # Check if source file exists 69 | if [ -f "${FILEPATH}/${FILEWEXT}" ]; then 70 | 71 | # Set source file with full filepath 72 | FILENAME="${FILEPATH}/${FILEWEXT}" 73 | 74 | # Get file duration in seconds 75 | START_TIME=$(${MYSQL} "SELECT starttime FROM queue WHERE id = ${QUEUEID};") 76 | LENGTH=$(${MYSQL} "SELECT length FROM queue WHERE id = ${QUEUEID};") 77 | 78 | # Write the command to a log file 79 | echo "ffmpeg -threads ${CPUNO} -ss ${START_TIME} -i ${FILENAME} -t ${LENGTH} -r 29.97 -vcodec libx264 -acodec aac -bsf:v h264_mp4toannexb -f mpegts -strict experimental -y ${OUTPATH}${FILEWOEXT}.part${QUEUEID}.ts >> ${OUTPATH}${FILEWOEXT}.part${QUEUEID}.log.txt 2>&1" > "${OUTPATH}${FILEWOEXT}.part${QUEUEID}.log.txt" 80 | 81 | ERROR=0; 82 | ERRORLOG=; 83 | 84 | streams_stream_0_width= ; streams_stream_0_height= 85 | 86 | eval $(ffprobe -v error -of flat=s=_ -select_streams v:0 -show_entries stream=height,width "${FILENAME}") 87 | SIZE=${streams_stream_0_width}x${streams_stream_0_height} 88 | 89 | REOLVIDEO=$(echo ${SIZE} |sed 's#x#*#g' | bc) 90 | RESOLUTION=$(echo "scale=1; $streams_stream_0_width/$streams_stream_0_height" | bc) 91 | 92 | transcode() { 93 | declare -a BITRATE=('128' '256' '512' '712'); 94 | ARRAYCNT=0 95 | for i in $@; do 96 | if [ ${REOLVIDEO} -gt $(echo ${i}|sed 's#x#*#g' |bc) ]; then 97 | VDR="$i"; VBR=${BITRATE[$ARRAYCNT]} 98 | if ! ffmpeg -threads ${CPUNO} -ss ${START_TIME} -t ${LENGTH} -i "${FILENAME}" -s ${VDR} -movflags rtphint -b:v ${VBR}k -vcodec libx264 -acodec libfaac -ab 20k -ar 44100 -y "${OUTPATH}${FILEWOEXT}.part${QUEUEID}-${VBR}000.ts" >> "${OUTPATH}${FILEWOEXT}.part${QUEUEID}-${VBR}000.log.txt" 2>&1; then 99 | ERROR=1 100 | ERRORLOG="${ERRORLOG}Failed: ${VBR} Bitrate Conversion" 101 | fi 102 | else 103 | VBR=${BITRATE[$ARRAYCNT]} 104 | if ! ffmpeg -threads ${CPUNO} -ss ${START_TIME} -t ${LENGTH} -i "${FILENAME}" -movflags rtphint -b:v ${VBR}k -vcodec libx264 -acodec libfaac -ab 20k -ar 44100 -y "${OUTPATH}${FILEWOEXT}.part${QUEUEID}-${VBR}000.ts" >> "${OUTPATH}${FILEWOEXT}.part${QUEUEID}-${VBR}000.log.txt" 2>&1; then 105 | ERROR=1 106 | ERRORLOG="${ERRORLOG}Failed: ${VBR} Bitrate Conversion" 107 | fi 108 | fi 109 | ARRAYCNT=$(expr $ARRAYCNT + 1) 110 | done 111 | } 112 | 113 | if [ ${RESOLUTION} = '1.3' ]; then 114 | transcode 320x240 480x360 640x480 1024x768 115 | elif [ ${RESOLUTION} = '1.7' ]; then 116 | transcode 384x216 512x288 640x360 1024x576 117 | else 118 | transcode 384x216 512x288 640x360 1024x576 119 | fi 120 | 121 | if [ $ERROR -ne '0' ]; then 122 | ${MYSQL} "UPDATE jobs SET error = '${ERRORLOG}' where jobid = ${JOB_ID};" 123 | echo "Error: Job Failed: ${JOB_ID} Log: ${ERRORLOG}" >> /var/log/nodes.log 124 | ERRORLOG= 125 | rm -fv /tmp/nodes 126 | sleep 5 127 | else 128 | #rm -fv "${FILENAME}" 129 | # Update job status, so that the other workers know when its done 130 | ${MYSQL} "UPDATE jobs SET jobcomplete=jobcomplete+1 WHERE jobid = ${JOB_ID};" 131 | ${MYSQL} "UPDATE queue SET status=status+1 WHERE id = ${QUEUEID};" 132 | rm -fv /tmp/nodes 133 | sleep 5 134 | fi 135 | else 136 | ${MYSQL} "UPDATE jobs SET error = 'Source file does not exists' where jobid = ${JOB_ID};" 137 | echo "Error: Job Failed: ${JOB_ID} Log: ${ERRORLOG}" >> /var/log/nodes.log 138 | rm -fv /tmp/nodes 139 | sleep 5 140 | fi 141 | else 142 | echo "Job is already in process :("; 143 | rm -fv /tmp/nodes 144 | sleep 5 145 | fi 146 | else 147 | echo "Hooorrray.. No jobs to Process :)"; 148 | rm -fv /tmp/nodes 149 | sleep 5 150 | fi 151 | else 152 | echo "Master process is running.." 153 | rm -fv /tmp/nodes 154 | sleep 5 155 | fi 156 | done 157 | exit 0 158 | -------------------------------------------------------------------------------- /transcoding.sql: -------------------------------------------------------------------------------- 1 | -- phpMyAdmin SQL Dump 2 | -- version 3.5.7 3 | -- http://www.phpmyadmin.net 4 | -- 5 | -- Host: localhost 6 | -- Generation Time: Oct 10, 2015 at 12:46 AM 7 | -- Server version: 5.6.21-70.1 8 | -- PHP Version: 5.3.3 9 | 10 | SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO"; 11 | SET time_zone = "+00:00"; 12 | 13 | 14 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 15 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 16 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 17 | /*!40101 SET NAMES utf8 */; 18 | 19 | -- 20 | -- Database: `transcoding` 21 | -- 22 | 23 | -- -------------------------------------------------------- 24 | 25 | -- 26 | -- Table structure for table `cpdetails` 27 | -- 28 | 29 | DROP TABLE IF EXISTS `cpdetails`; 30 | CREATE TABLE IF NOT EXISTS `cpdetails` ( 31 | `cp` varchar(100) NOT NULL, 32 | `user` varchar(100) NOT NULL, 33 | `password` varchar(100) NOT NULL, 34 | `host` varchar(100) NOT NULL, 35 | UNIQUE KEY `cp` (`cp`) 36 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 37 | 38 | -- -------------------------------------------------------- 39 | 40 | -- 41 | -- Table structure for table `jobs` 42 | -- 43 | 44 | DROP TABLE IF EXISTS `jobs`; 45 | CREATE TABLE IF NOT EXISTS `jobs` ( 46 | `jobid` int(11) NOT NULL AUTO_INCREMENT, 47 | `filename` varchar(255) NOT NULL, 48 | `filepath` varchar(255) NOT NULL, 49 | `duration` int(11) NOT NULL, 50 | `nodecount` int(11) NOT NULL, 51 | `jobcount` int(11) NOT NULL DEFAULT '0', 52 | `jobcomplete` int(11) NOT NULL, 53 | `error` varchar(255) DEFAULT NULL, 54 | `starttime` varchar(100) DEFAULT NULL, 55 | `conversiontime` varchar(100) DEFAULT NULL, 56 | `totaltime` varchar(100) DEFAULT NULL, 57 | `product` varchar(50) DEFAULT NULL, 58 | `master` varchar(100) DEFAULT NULL, 59 | PRIMARY KEY (`jobid`) 60 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1 AUTO_INCREMENT=58 ; 61 | 62 | -- -------------------------------------------------------- 63 | 64 | -- 65 | -- Table structure for table `queue` 66 | -- 67 | 68 | DROP TABLE IF EXISTS `queue`; 69 | CREATE TABLE IF NOT EXISTS `queue` ( 70 | `id` int(11) NOT NULL AUTO_INCREMENT, 71 | `jobid` int(11) NOT NULL, 72 | `starttime` varchar(50) NOT NULL, 73 | `length` varchar(50) NOT NULL, 74 | `node` varchar(255) NOT NULL, 75 | `status` int(11) NOT NULL, 76 | PRIMARY KEY (`id`) 77 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1 AUTO_INCREMENT=222 ; 78 | 79 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 80 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 81 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 82 | --------------------------------------------------------------------------------