├── .gitignore ├── CHANGELOG.markdown ├── LICENSE.txt ├── README.markdown ├── bin └── codem-transcode ├── lib ├── config.js ├── job-handler.js ├── job.js ├── logger.js ├── notify-handler.js ├── probe-handler.js ├── recoverable-stream.js ├── server.js └── transcoder.js ├── migrations ├── 20130305100924-create-jobs.js ├── 20130702120654-add-thumbnails-to-jobs.js └── 20140822085031-add-playlist-and-segments-to-jobs.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ -------------------------------------------------------------------------------- /CHANGELOG.markdown: -------------------------------------------------------------------------------- 1 | ## codem-transcode 0.5.11 (2015/12/02) ## 2 | 3 | * Sanitize null values (@brain64bit) 4 | * Allow HTTPS for notification delivery (@mirion) 5 | 6 | ## codem-transcode 0.5.10 (2015/09/16) ## 7 | 8 | * Increase maxBuffer to 4MB to compensate for older ffmpegs 9 | 10 | ## codem-transcode 0.5.9 (2015/09/03) ## 11 | 12 | * Fix for lingering "processing" jobs. (#42) 13 | * Encoder options can be an array instead of a string as well (to allow complex filters) 14 | 15 | ## codem-transcode 0.5.8 (2015/07/05) ## 16 | 17 | * Prevent transcoder from returning a negative freeSlot count (@larsfi). 18 | 19 | ## codem-transcode 0.5.7 (2015/04/23) ## 20 | 21 | * Add option for database port in config (@brain64bit). 22 | 23 | ## codem-transcode 0.5.6 (2015/04/22) ## 24 | 25 | * Return a job from memory if it is still transcoding. 26 | 27 | ## codem-transcode 0.5.5 (2014/09/08) ## 28 | 29 | * Added basic HLS support (@cutalion) using "segments_options". 30 | 31 | ## codem-transcode 0.5.4 (2014/03/18) ## 32 | 33 | * Updated sqlite3 dependency to 2.2.0. 34 | 35 | ## codem-transcode 0.5.3 (2014/01/27) ## 36 | 37 | * Bugfix: issue #19 and #20, missing or corrupt ffmpeg will no longer put transcoder in weird state. 38 | * Send "X-Codem-Notify-Timestamp" header for notifications with millisecond precision (instead of second). 39 | 40 | ## codem-transcode 0.5.2 (2013/09/10) ## 41 | 42 | * Bugfix: migrations should create "Jobs" table (instead of "jobs") 43 | 44 | ## codem-transcode 0.5.1 (2013/07/08) ## 45 | 46 | * Extra path separator / nul file check for platform-specific issues (Windows/*nix) 47 | * Allow thumbnail-only jobs. See README. 48 | 49 | ## codem-transcode 0.5.0 (2013/07/02) ## 50 | 51 | * All fixes/features from the betas. 52 | * Use async package for some of the complex callback chains. 53 | * Ability to capture thumbnails from the videos after transcoding. 54 | 55 | ## codem-transcode 0.5.0-beta.4 (2013/04/30) ## 56 | 57 | * Fix for #15, cannot open database when using default config. 58 | * sqlite3 dependency updated to 2.1.7. 59 | 60 | ## codem-transcode 0.5.0-beta.3 (2013/04/25) ## 61 | 62 | * Allow purging of old successful jobs. See README for usage. 63 | * Tested against Node.js 0.10. 64 | 65 | ## codem-transcode 0.5.0-beta.2 (2013/03/07) ## 66 | 67 | * Switch to Sequelize for database abstraction. codem-transcode now supports SQLite, MySQL and Postgres. PLEASE NOTE 68 | that this change is *not* backwards compatible. You will need to update your database. If you are coming from an old 69 | version of codem-transcode we advise you to move away your old database and let the software generate a new database 70 | for you. 71 | 72 | ## codem-transcode 0.5.0-beta.1 (2013/02/19) ## 73 | 74 | * The logging system should now be more able to handle unexpected issues, such as when your logging file system is out 75 | of space. In most cases the system should be able to gracefully intercept any issues and resume operation after the 76 | issue has been cleared (ie. more space has been created). 77 | 78 | ## codem-transcode 0.4.4 (2013/01/28) ## 79 | 80 | * Use a "=" instead of a "LIKE" when loading a job. Prevents excessive disk IO. 81 | 82 | ## codem-transcode 0.4.3 (2013/01/22) ## 83 | 84 | * Allow both time formats in the ffmpeg output parsing (xx:xx and xx:xx:xx). 85 | 86 | ## codem-transcode 0.4.2 (2013/01/07) ## 87 | 88 | * Fix for copying a file across partitions. 89 | 90 | ## codem-transcode 0.4.1 (2012/10/16) ## 91 | 92 | * Added ffprobe support. 93 | 94 | ## codem-transcode 0.4.0 (2012/10/10) ## 95 | 96 | * Updated package to Node 0.8. 97 | 98 | ## codem-transcode 0.3.1 (2012/06/27) ## 99 | 100 | * Allow disabling of scratch directory. 101 | 102 | ## codem-transcode 0.3.0 (2012/03/20) ## 103 | 104 | * Updated dependency to a newer version of Node. 105 | * Update ffmpeg time parsing. 106 | * Switched from connect to express. 107 | 108 | ## codem-transcode 0.2.1 (2011/09/26) ## 109 | 110 | * Add additional "X-Codem-Notify-Timestamp" HTTP header. 111 | 112 | ## codem-transcode 0.2.0 (2011/08/23) ## 113 | 114 | * Deleting and cancelling jobs. 115 | * Additional error checking. 116 | 117 | ## codem-transcode 0.1.2 (2011/07/11) ## 118 | 119 | * Additional logging. 120 | * Notify when duration is known. 121 | * Added extra check when creating directories. 122 | 123 | ## codem-transcode 0.1.1 (2011/05/24) ## 124 | 125 | * Fixed license info 126 | 127 | ## codem-transcode 0.1.0 (2011/05/24) ## 128 | 129 | * Initial release -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011-2012 Hiro, Sjoerd Tieleman, Bart Zonneveld 2 | 3 | http://madebyhiro.com/ 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # IMPORTANT: This project is no longer actively maintained! 2 | 3 | --- 4 | 5 | # Codem-transcode 6 | 7 | * http://github.com/madebyhiro/codem-transcode 8 | 9 | ## Description 10 | 11 | Codem-transcode is an offline video transcoder written in node.js. It 12 | 13 | 1. Uses ffmpeg for transcoding 14 | 2. Has a simple HTTP API 15 | 3. Is mostly asynchronous 16 | 17 | Codem-transcode can be used in conjunction with Codem-schedule (https://github.com/madebyhiro/codem-schedule) for robust job scheduling and notifications or it can be used stand-alone, with or without a custom scheduler. 18 | 19 | ## Requirements 20 | 21 | * ffmpeg (at least 0.10 and compiled/configured to your own taste) 22 | * sqlite3/MySQL/PostgreSQL 23 | * node.js version 0.8.x (x>=11), with packages (if you use npm they will be installed automatically): 24 | * sequelize (http://sequelizejs.com/) 25 | * sqlite3 (http://github.com/developmentseed/node-sqlite3) 26 | * express (http://expressjs.com/) 27 | * argsparser (http://github.com/kof/node-argsparser) 28 | * mkdirp (https://github.com/substack/node-mkdirp) 29 | * async (https://github.com/caolan/async) 30 | 31 | ## Installation 32 | 33 | The recommended installation procedure is to just use `npm` (http://npmjs.org/): 34 | 35 | # npm install codem-transcode 36 | 37 | Install it to your preferred location, or use the `-g` option to install it globally. 38 | 39 | ## Upgrading 40 | 41 | Upgrading should most of the times be as simple as shutting down, installing the new package and restarting the transcoder. Unless you're... 42 | 43 | ### Upgrading from earlier versions to 0.5 (IMPORTANT!) 44 | 45 | Codem-transcode switched from using "plain" SQL (using sqlite) to a database abstraction layer (Sequelize). This brings some advantages 46 | (support for multiple database engines, better consistency, easier migrations in the future), but is not backwards compatible. Therefore, we recommend you backup and move away your old database and 47 | start with a fresh one. The procedure for this would be: 48 | 49 | * Shutdown the transcoder; 50 | * Move your database away (or delete if you're not interested in the history); 51 | * Install the new package; 52 | * Start the transcoder. 53 | 54 | This will initialize a new up-to-date database which can be migrated to newer schema's more easily in the future. If you are doing a 55 | clean install you do not need to worry about any of this. 56 | 57 | If you want to keep your history we recommend you follow the above procedure and write a separate import script to import your old data 58 | into the new database. 59 | 60 | ## Starting 61 | 62 | When you install codem-transcode a script will be installed that allows you to start the transcoder. If you install it globally it should be in your `PATH`, otherwise, you can start the transcoder using: 63 | 64 | # /PATH/TO/TRANSCODER/bin/codem-transcode 65 | 66 | Please check for yourself where `npm` installs your packages and script. 67 | 68 | ## Configuration 69 | 70 | Configuration is done by specifying a CLI option (`-c`) and pointing to a file containing a valid JSON object (http://www.json.org/). Note that node.js' JSON parser is fairly strict so make sure you get the syntax right. An example config is: 71 | 72 | { 73 | "port": 8080, 74 | "access_log": "/var/log/access_log", 75 | "database": "/var/db/jobs.db", 76 | "slots": 8, 77 | "interface": "127.0.0.1", 78 | "encoder": "ffmpeg", 79 | "scratch_dir": "/tmp", 80 | "use_scratch_dir": true, 81 | "ffprobe": null 82 | } 83 | 84 | Configuration options: 85 | 86 | * `port`; port to start server on, default `8080` 87 | * `interface`; which network interface to listen on, default `127.0.0.1` (only `localhost`) 88 | * `access_log`; location to store HTTP access log, default `/var/log/access_log` 89 | * `database`; location to store the jobs database, default is SQLite with `/var/db/jobs.db` 90 | * `slots`; number of transcoding slots to use (i.e. the maximum number of ffmpeg child processes), defaults to the number of CPUs/cores in your machine 91 | * `encoder`; path to the ffmpeg binary, if it is in your path specifying only `ffmpeg` is sufficient, defaults to `ffmpeg` 92 | * `scratch_dir`; temporary files are written here and moved into the destination directory after transcoding, defaults to `/tmp` 93 | * `use_scratch_dir`; if set to false temporary files will be written to the output directory of your job, for setups that don't require or are not able to use a separate `scratch_dir`. Defaults to `true` so if you don't want to disable the `scratch_dir` you can also omit this option from your config file. 94 | * `ffprobe`; path to the ffprobe binary, if it is in your path specifying only `ffprobe` is sufficient, defaults to `null`. Set this to a non-null value if you want to enable ffprobe support in the transcoder. 95 | 96 | Note that the default config will put the access_log and job database in `/var/log` and `var/db/` respectively. If you wish to put these in a different location please supply your own config. You can start the transcoder with your custom config using: 97 | 98 | # /PATH/TO/TRANSCODER/bin/codem-transcode -c /PATH/TO/CONFIG/config.json 99 | 100 | ### Advanced database configuration 101 | 102 | codem-transcode supports multiple database backends, courtesy of Sequelize. The default is still to store data in a SQLite database (whenever you specify a string for `database` in the config file). To use MySQL or Postgres, supply a valid object for the database entry. Your configuration will then look like: 103 | 104 | { 105 | "port": 8080, 106 | "access_log": "/var/log/access_log", 107 | "database": { 108 | "dialect": "mysql", 109 | "username": "root", 110 | "database": "codem", 111 | "host": "localhost", 112 | "port": 3306 113 | }, 114 | "slots": 8, 115 | "interface": "127.0.0.1", 116 | "encoder": "ffmpeg", 117 | "scratch_dir": "/tmp", 118 | "use_scratch_dir": true, 119 | "ffprobe": null 120 | } 121 | 122 | Be sure to specify a `dialect` ("mysql", "postgres", "sqlite"), a `username`, a `password` (can be omitted if using a passwordless database) and a `host` (can be omitted for "localhost"). `port` can be omitted for the default port. 123 | 124 | ## Usage 125 | 126 | After starting the server you can control it using most HTTP CLI tools, such as `curl` or `wget`. The HTTP API is as follows: 127 | 128 | * * * 129 | Request: `POST /jobs` 130 | 131 | Parameters (HTTP POST data, should be valid JSON object): 132 | 133 | { 134 | "source_file": "/PATH/TO/INPUT/FILE.wmv", 135 | "destination_file":"/PATH/TO/OUTPUT/FILE.mp4", 136 | "encoder_options": "-acodec libfaac -ab 96k -ar 44100 -vcodec libx264 -vb 416k -s 320x180 -y -threads 0", 137 | "thumbnail_options": { 138 | "percentages": [0.25, 0.5, 0.75], 139 | "size": "160x90", 140 | "format": "png" 141 | }, 142 | "segments_options" : { 143 | "segment_time": 10 144 | }, 145 | "callback_urls": ["http://example.com/notifications"] 146 | } 147 | 148 | Responses: 149 | 150 | * `202 Accepted` - Job accepted 151 | * `400 Bad Request` - Invalid request (format) 152 | * `503 Service Unavailable` - Transcoder not accepting jobs at the moment (all encoding slots are in use) 153 | 154 | 155 | Required options are `source_file`, `destination_file` and `encoder_options`. Input and output files must be *absolute* paths. 156 | 157 | The `callback_urls` array is optional and is a list (array) of HTTP endpoints that should be notified once encoding finishes (due to the job being complete or some error condition). The notification will sent using HTTP PUT to the specified endpoints with the job status. It will also include a custom HTTP header "X-Codem-Notify-Timestamp" that contains the timestamp (in milliseconds) at which the notification was generated and sent. It is best to observe this header to determine the order in which notifications are received at the other end due to network lag or other circumstances that may cause notifications to be received out of order. 158 | 159 | The `thumbnail_options` object is optional and contains a set of thumbnails that should be encoded after the transcoding is complete. Thumbnails are captured from the source file for maximum quality. The options for thumbnails include: 160 | 161 | * Either "percentages" or "seconds" (but not both at the same time), valid options are: 162 | * A single percentage, this will trigger a thumbnail every x%. `"percentages": 0.1` will generate thumbnails at 0%, 10%, 20%, [...], 100%. 163 | * An array of explicit percentages, this will trigger thumbnails only at the specified positions. 164 | `"percentages": [0.25, 0.5, 0.75]` will generate thumbnails at 25%, 50% and 75%. 165 | * A single offset in seconds, this will trigger a thumbnail every x seconds. `"seconds": 10` will generate thumbnails at 0 seconds, 10 seconds, 20 seconds, etc., until the end of the source file. 166 | * An array of explicit offsets, this will trigger thumbnails only at the specified positions. 167 | `"seconds": [30, 60, 90]` will generate thumbnails at 30 seconds, 60 seconds and 90 seconds. 168 | * A size can be specified in pixels (width x height). If omitted it will generate thumbnails the size of the source video. (optional) 169 | * A format for the thumbnails. The format must be supported by your ffmpeg binary. If omitted it will generate thumbnails in the JPEG format. Most people will use either "jpg" or "png". (optional) 170 | 171 | If you specify thumbnails but an error occurs during generation, your job will be marked as failed. If you don't specify a valid `seconds` or `percentages` option thumbnail generation will be skipped but the job can still be completed successfully. 172 | 173 | ### Segmenting / HLS 174 | 175 | The `segments_options` object is optional and contains segment time (duration) in seconds. Segmented videos are used in [HLS](https://en.wikipedia.org/wiki/HTTP_Live_Streaming). These options are applied to the encoded video, thus `encoder_options` are required. Moreover `encoder_options` should prepare video for segmenting, because bitstream 176 | filter [h264_mp4toannexb](https://www.ffmpeg.org/ffmpeg-bitstream-filters.html#h264_005fmp4toannexb) will be applied to the video. Therefore it is recommended to transcode to an MP4 file before segmenting. Segmenting works by taking the output file from your transcoder job and applying segmenting to it. 177 | 178 | The segmenting command looks like: 179 | 180 | ffmpeg -i /tmp/46ee0a404a4b75d85c09d98a7c6b403579ee9f99.mp4 -codec copy -map 0 \ 181 | -f segment -vbsf h264_mp4toannexb -flags -global_header -segment_format mpegts \ 182 | -segment_list /path/to/dest_file_dir/dest_file_name.m3u8 -segment_time 10 \ 183 | /path/to/dest_file_dir/dest_file_name-%06d.ts 184 | 185 | `46ee0a404a4b75d85c09d98a7c6b403579ee9f99.mp4` is a temporary encoded file (generated by Codem). After transcoding and segmenting you end up with the transcoded file, as well as the segments/playlist. 186 | 187 | ### Thumbnail-only job 188 | 189 | It's possible to only generate thumbnails from a video and not do any transcoding at all. This might come in handy if you're transcoding to lots of different formats and want to keep thumbnail generation separate from transcoding. You achieve this by POSTing a job with `"encoder_options": ""` (empty string) and of course specifying your `thumbnail_options`. In this case `destination_file` should be a _prefix_ for the output file, e.g. `"destination_file": "/Users/codem/output/my_video"` results in thumbnails in `/Users/codem/output/` with filenames such as `my_video-$offset.$format` (where `$offset` is the thumbnail offset in the video and `$format` of course the thumbnail format). All other options remain the same. See the examples. 190 | 191 | * * * 192 | Request: `GET /jobs` 193 | 194 | Responses: 195 | 196 | * `200 OK` - Returns status of all active jobs 197 | 198 | * * * 199 | Request: `GET /jobs/$JOB_ID` 200 | 201 | Responses: 202 | 203 | * `200 OK` - Returns status of job 204 | * `404 Not Found` - Job not found 205 | 206 | * * * 207 | Request: `DELETE /jobs/$JOB_ID` 208 | 209 | Cancels the job (if it is running) and deletes it from the database. 210 | 211 | Responses: 212 | 213 | * `200 OK` - Returns last known status of the job that is being deleted 214 | * `404 Not Found` - Job not found 215 | 216 | * * * 217 | Request: `DELETE /jobs/purge/$AGE` 218 | 219 | Purge successfully completed jobs from the database with a certain age. Age is specified in seconds since it was created. So an age of 3600 deletes jobs that were successful and created more than 1 hour ago. 220 | 221 | Responses: 222 | 223 | * `200 OK` - Returns number of jobs that were purged. 224 | 225 | * * * 226 | Request: `POST /probe` 227 | 228 | Probe a source file using `ffprobe` (if you have enabled it in the configuration). Output is a JSON object containing the `ffprobe` output. 229 | 230 | Parameters (HTTP POST data, should be valid JSON object): 231 | 232 | { 233 | "source_file": "/PATH/TO/INPUT/FILE.wmv" 234 | } 235 | 236 | Responses: 237 | 238 | * `200 OK` - Returns `ffprobe` output JSON-formatted 239 | * `400 Bad Request` - Returned if you attempt to probe a file when there is no path set to the `ffprobe` binary 240 | * `500 Internal Server Error` - Returned if there was an error while trying to probe, the output from `ffprobe` will be returned as well 241 | 242 | * * * 243 | ## Examples 244 | 245 | Create a new job, transcode "video.wmv" to "video.mp4" using the specified ffmpeg options (96kbit/s audio, 416kbit/s video, 320x180, use as much threads as possible). Requires libx264 support in your ffmpeg. 246 | 247 | # curl -d '{"source_file": "/tmp/video.wmv","destination_file":"/tmp/video.mp4","encoder_options": "-acodec libfaac -ab 96k -ar 44100 -vcodec libx264 -vb 416k -s 320x180 -y -threads 0"}' http://localhost:8080/jobs 248 | 249 | Output: {"message":"The transcoder accepted your job.","job_id":"d4b1dfebe6860839b2c21b70f35938d870011682"} 250 | 251 | Create a new job, transcode "video.mpg" to "video.webm" using the specified ffmpeg options (total bitrate 512kbit/s, 320x180, use as much threads as possible). Requires libvpx support in your ffmpeg. 252 | 253 | # curl -d '{"source_file": "/tmp/video.mpg","destination_file":"/tmp/video.webm","encoder_options": "-vcodec libvpx -b 512000 -s 320x180 -acodec libvorbis -y -threads 0"}' http://localhost:8080/jobs 254 | 255 | Output: {"message":"The transcoder accepted your job.","job_id":"c26769be0955339db8f98580c212b7611cacf4dd"} 256 | 257 | Get status of all available encoder slots. 258 | 259 | # curl http://localhost:8080/jobs 260 | 261 | Output: {"max_slots":8,"free_slots":8,"jobs":[]} 262 | 263 | or 264 | 265 | Output: {"max_slots":8, "free_slots":7, "jobs":[{"id":"da56da6012bda2ce775fa028f056873bcb29cb3b", "status":"processing", "progress":0.12480252764612954, "duration":633, "filesize":39191346, "message":null}]} 266 | 267 | Get full status of one job with id "da56da6012bda2ce775fa028f056873bcb29cb3b". 268 | 269 | # curl http://localhost:8080/jobs/da56da6012bda2ce775fa028f056873bcb29cb3b 270 | 271 | Output: {"id":"da56da6012bda2ce775fa028f056873bcb29cb3b", "status":"processing", "progress":0.21800947867298578, "duration":633, "filesize":39191346, "opts":"{\"source_file\":\"/shared/videos/asf/video.asf\", \"destination_file\":\"/shared/videos/mp4/journaal.mp4\", \"encoder_options\":\"-acodec libfaac -ab 96k -ar 44100 -vcodec libx264 -vb 416k -s 320x180 -y -threads 0\"}", "message":null, "created_at":1304338160, "updated_at":1304338173} 272 | 273 | Probe a file using `ffprobe`. 274 | 275 | # curl -d '{"source_file": "/tmp/video.wmv"}' http://localhost:8080/probe 276 | 277 | Output: {"ffprobe":{"streams":[ ... stream info ... ],"format":{ ... format info ... }}}} 278 | 279 | Thumbnail-only job (160x90 in PNG format every 10% of the video). 280 | 281 | # curl -d '{"source_file": "/tmp/video.mp4","destination_file":"/tmp/thumbnails/video","encoder_options": "", "thumbnail_options": { "percentages": 0.1, "size": "160x90", "format": "png"} }' http://localhost:8080/jobs 282 | 283 | Output: {"message":"The transcoder accepted your job.","job_id":"d4b1dfebe6860839b2c21b70f35938d870011682"} 284 | 285 | 286 | Segmenting job. 287 | 288 | # curl -d '{"source_file": "/tmp/video.mp4", "destination_file": "/tmp/output/test.mp4", "encoder_options": "-vb 2000k -minrate 2000k -maxrate 2000k -bufsize 2000k -s 1280x720 -acodec aac -strict -2 -ab 192000 -ar 44100 -ac 2 -vcodec libx264 -movflags faststart", "segments_options": {"segment_time": 10} }' http://localhost:8080/jobs 289 | 290 | Output: {"message":"The transcoder accepted your job.","job_id":"7dc3c268783d7f3c737f3a134ccf1d4f15bb8442"} 291 | 292 | Status of finished job: 293 | 294 | # curl http://localhost:8080/jobs/7dc3c268783d7f3c737f3a134ccf1d4f15bb8442 295 | 296 | Output: 297 | { 298 | "id": "7dc3c268783d7f3c737f3a134ccf1d4f15bb8442", 299 | "status": "success", 300 | "progress": 1, 301 | "duration": 1, 302 | "filesize": 783373, 303 | "message": "ffmpeg finished succesfully.", 304 | "playlist": "/tmp/output/test.m3u8", 305 | "segments": [ 306 | "/tmp/output/test-000000.ts", 307 | "/tmp/output/test-000001.ts", 308 | "/tmp/output/test-000002.ts", 309 | "/tmp/output/test-000003.ts", 310 | "/tmp/output/test-000004.ts", 311 | "/tmp/output/test-000005.ts" 312 | ] 313 | } 314 | 315 | Segmenting-only job (you are expected to have a valid MP4 file suitable for segmenting as the input). 316 | 317 | # curl -d '{"source_file": "/tmp/video.mp4","destination_file":"/tmp/segments/video.mp4","encoder_options": "", "segments_options": {"segment_time": 10} }' http://localhost:8080/jobs 318 | 319 | Output: {"message":"The transcoder accepted your job.","job_id":"c7599790527c0bb173cc7a0c44411aaca5c1550a"} 320 | 321 | Status of finished job: 322 | 323 | # curl http://localhost:8080/jobs/c7599790527c0bb173cc7a0c44411aaca5c1550a 324 | 325 | Output: 326 | { 327 | "id":"c7599790527c0bb173cc7a0c44411aaca5c1550a", 328 | "status":"success", 329 | "progress":1, 330 | "duration":26, 331 | "filesize":6734045, 332 | "message":"finished segmenting job.", 333 | "playlist":"/tmp/segments/video.m3u8", 334 | "segments":[ 335 | "/tmp/segments/video-000000.ts", 336 | "/tmp/segments/video-000001.ts", 337 | "/tmp/segments/video-000002.ts" 338 | ] 339 | } 340 | ## Issues and support 341 | 342 | If you run into any issues while using codem-transcode please use the Github issue tracker to see if it is a known problem 343 | or report it as a new one. 344 | 345 | We also provide commercial support for codem-transcode (for bugs, features, configuration, etc.). If you are interested in 346 | commercial support or are already receiving commercial support, feel free to contact us directly at hello@madebyhiro.com. 347 | 348 | ## License 349 | 350 | Codem-transcode is released under the MIT license, see `LICENSE.txt`. 351 | -------------------------------------------------------------------------------- /bin/codem-transcode: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var Transcoder = require('../lib/transcoder.js'); 4 | 5 | new Transcoder().boot(); -------------------------------------------------------------------------------- /lib/config.js: -------------------------------------------------------------------------------- 1 | var os = require('os'), 2 | opts = require('argsparser').parse(), 3 | fs = require('fs'), 4 | logger = require('./logger'); 5 | 6 | var config = { 7 | port: 8080, 8 | access_log: '/var/log/access_log', 9 | database: { dialect: "sqlite", database: '/var/db/jobs.db' }, 10 | slots: os.cpus().length, 11 | interface: "127.0.0.1", 12 | encoder: "ffmpeg", 13 | scratch_dir: "/tmp", 14 | use_scratch_dir: true, 15 | ffprobe: null 16 | }; 17 | 18 | var loadedConfig = null; 19 | 20 | exports.load = function() { 21 | if (opts['-c'] && !loadedConfig) { 22 | try { 23 | loadedConfig = eval('(' + fs.readFileSync(opts['-c'], 'utf8') + ')'); 24 | ConfigUtils.merge(config, loadedConfig); 25 | ConfigUtils.rewriteDatabaseEntry(config); 26 | } catch(err) { 27 | logger.log('Error reading config from ' + opts['-c']); 28 | logger.log(err); 29 | process.exit(1); 30 | } 31 | } 32 | return config; 33 | } 34 | 35 | var ConfigUtils = { 36 | merge: function(obj1,obj2) { 37 | for (key in obj2) { 38 | obj1[key] = obj2[key]; 39 | } 40 | }, 41 | 42 | rewriteDatabaseEntry: function(config) { 43 | if (typeof config.database == 'string') { 44 | config.database = { 45 | dialect: "sqlite", 46 | database: config.database 47 | } 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /lib/job-handler.js: -------------------------------------------------------------------------------- 1 | var os = require('os'), 2 | logger = require('./logger'), 3 | util = require('util'), 4 | config = require('./config').load(), 5 | Job = require('./job'), 6 | notifyHandler = require('./notify-handler'); 7 | 8 | var slots = []; 9 | 10 | exports.find = function(id, callback) { 11 | for (var item in slots) { 12 | if (slots[item].internalId == id) { 13 | callback(null, slots[item]); 14 | return; 15 | } 16 | } 17 | 18 | Job.find({ where: { internalId: id }}).success(function(job) { 19 | callback(null, job); 20 | }).error(function(err) { 21 | // error while finding job 22 | callback('Error while finding the job in the database. Make sure the database is available.', null); 23 | }) 24 | } 25 | 26 | exports.freeSlots = function() { 27 | var freeSlots = config['slots'] - slots.length; 28 | return Math.max(freeSlots, 0); 29 | } 30 | 31 | exports.cancelAndRemove = function(internalId, callback) { 32 | for (var item in slots) { 33 | if (slots[item].internalId == internalId) { 34 | slots[item].cancel(); 35 | } 36 | } 37 | 38 | var job = Job.find({ where: { internalId: internalId }}).success(function(job) { 39 | if (job) { 40 | job.destroy().success(function () { 41 | callback(null, job); 42 | }).error(function (error) { 43 | callback("Error while destroying job from database: " + error, null); 44 | }); 45 | } else { 46 | callback("No such job to delete", null); 47 | } 48 | }).error(function(err) { 49 | callback("Error while cancelling and/or removing job: " + err, null); 50 | }); 51 | } 52 | 53 | exports.slots = slots; 54 | 55 | exports.processJobRequest = function(postData, callback) { 56 | validateJobRequest(postData, function(err, opts) { 57 | if (err) { 58 | callback({ type: 'invalid', message: 'Some required fields are missing.', missingFields: err['missingFields'] }, null) 59 | } else { 60 | if (hasFreeSlots()) { 61 | spawnJob(opts, function onSpawned(err, job) { 62 | if (job) { 63 | slots.push(job); 64 | callback(null, job); 65 | } else { 66 | callback({ type: 'spawn', message: 'An error occurred while spawning a new job.' }, null); 67 | } 68 | }); 69 | } else { 70 | callback({ type: 'full', message: 'The transcoder is at maximum capacity right now.' }, null); 71 | } 72 | } 73 | }); 74 | } 75 | 76 | exports.purgeJobs = function(age, callback) { 77 | var parsedAge = parseInt(age, 10); 78 | var timestamp = (new Date().valueOf()) - parsedAge * 1000; 79 | 80 | if (isNaN(timestamp)) { 81 | callback("Error while parsing age: " + age, null); 82 | } else { 83 | Job.findAll({ where: ["createdAt < ? AND status = 'success'", new Date(timestamp)] }).success(function(jobs) { 84 | var count = jobs.length; 85 | var chainer = new Sequelize.Utils.QueryChainer; 86 | 87 | for (var i=0; i 0) { 137 | callback({ missingFields: missingFields }, null); 138 | } else { 139 | callback(null, opts); 140 | } 141 | } 142 | 143 | function spawnJob(opts, callback) { 144 | Job.create( 145 | opts, 146 | function onCreate(err, job) { 147 | if (job) { 148 | logger.log('Job ' + job.internalId + ' accepted with opts: "' + util.inspect(job.parsedOpts()) + '"'); 149 | callback(null, job); 150 | } else { 151 | callback('Job could not be created.', null); 152 | } 153 | }, 154 | function onComplete(job) { 155 | logger.log('Job ' + this.internalId + ' finished with status: "' + this.status + '" and message: "' + this.message + '"'); 156 | notifyHandler.notify(this); 157 | removeItemFromSlot(this); 158 | } 159 | ); 160 | } 161 | 162 | function hasFreeSlots() { 163 | return config['slots'] - slots.length > 0; 164 | } 165 | 166 | function removeItemFromSlot(item) { 167 | var idx = slots.indexOf(item); 168 | if(idx!=-1) slots.splice(idx, 1); 169 | } 170 | -------------------------------------------------------------------------------- /lib/job.js: -------------------------------------------------------------------------------- 1 | var util = require('util'), 2 | crypto = require('crypto'), 3 | fs = require('fs'), 4 | child_process = require('child_process'), 5 | config = require('./config').load() 6 | mkdirp = require('mkdirp'), 7 | path = require('path'), 8 | notifyHandler = require('./notify-handler'), 9 | logger = require('./logger'), 10 | Sequelize = require('sequelize'), 11 | async = require('async'), 12 | os = require('os'); 13 | 14 | var StatusCodes = { 15 | SUCCESS: "success", 16 | FAILED: "failed", 17 | PROCESSING: "processing" 18 | } 19 | 20 | var JobUtils = { 21 | sql: null, 22 | 23 | generateThumbnailPath: function(destinationFile, offset, format) { 24 | return [path.dirname(destinationFile), path.basename(destinationFile, path.extname(destinationFile))].join(path.sep) + "-" + offset + "." + format; 25 | }, 26 | 27 | generateRangeFromThumbOpts: function(thumbOpts, duration) { 28 | if (thumbOpts['percentages']) { 29 | // percentage based thumbnails 30 | return JobUtils.percentagesToRange(thumbOpts['percentages'], duration); 31 | } else if (thumbOpts['seconds']) { 32 | // seconds based thumbnails 33 | return JobUtils.secondsToRange(thumbOpts['seconds'], duration); 34 | } else { 35 | return null; 36 | } 37 | }, 38 | 39 | getDatabase: function() { 40 | if (JobUtils.sql == null) { 41 | if (config['database']['dialect'] == "sqlite") { 42 | JobUtils.sql = new Sequelize('database', 'username', 'password', { 43 | dialect: 'sqlite', 44 | storage: config['database']['database'], 45 | logging: false 46 | }); 47 | } else { 48 | JobUtils.sql = new Sequelize(config['database']['database'], config['database']['username'], config['database']['password'], { 49 | dialect: config['database']['dialect'], 50 | storage: config['database']['database'], 51 | host: config['database']['host'], 52 | port: config['database']['port'], 53 | omitNull: true, 54 | pool: false, 55 | logging: false 56 | }); 57 | } 58 | } 59 | return JobUtils.sql; 60 | }, 61 | 62 | getMask: function() { 63 | return JobUtils.pad((process.umask() ^ 0777).toString(8), '0', 4); 64 | }, 65 | 66 | markUnknownState: function(callback) { 67 | Job.findAll({ where: { status: StatusCodes.PROCESSING }}).success(function(result) { 68 | if (result.length > 0) { 69 | for (var job in result) { 70 | result[job].status = StatusCodes.FAILED; 71 | result[job].message = "The transcoder quit unexpectedly or the database was unavailable for some period."; 72 | result[job].save(); 73 | } 74 | } 75 | }); 76 | callback(null); 77 | }, 78 | 79 | migrateDatabase: function(callback) { 80 | var migrator = JobUtils.getDatabase().getMigrator({path: __dirname + "/../migrations" }); 81 | migrator.migrate().success(function() { 82 | callback(null); 83 | }).error(function(err) { 84 | callback(err); 85 | }); 86 | }, 87 | 88 | pad: function(orig, padString, length) { 89 | var str = orig; 90 | while (str.length < length) 91 | str = padString + str; 92 | return str; 93 | }, 94 | 95 | percentagesToRange: function(percentages, duration) { 96 | if (Array.isArray(percentages)) { 97 | // explicit percentages 98 | return percentages.map(function (p) { return Math.floor(p * duration); }); 99 | } else { 100 | // single percentage interval 101 | var offsets = []; 102 | for (var p = 0.0; p <= 1.0; p += percentages) { 103 | offsets.push(Math.floor(p * duration)); 104 | } 105 | return offsets; 106 | } 107 | }, 108 | 109 | secondsToRange: function(seconds, duration) { 110 | if (Array.isArray(seconds)) { 111 | // explicit percentages 112 | return seconds; 113 | } else { 114 | // single percentage interval 115 | var offsets = []; 116 | for (var offset = 0; offset <= duration; offset += seconds) { 117 | offsets.push(Math.floor(offset)); 118 | } 119 | return offsets; 120 | } 121 | }, 122 | 123 | verifyDatabase: function(callback) { 124 | JobUtils.getDatabase().getQueryInterface().showAllTables().success(function (tables) { 125 | if (tables.length > 0 && tables.indexOf('SequelizeMeta') == -1) { 126 | logger.log("You appear to be upgrading from an old version of codem-transcode (<0.5). The database handling has " + 127 | "changed, please refer to the upgrade instructions. To prevent data loss the transcoder will now exit."); 128 | callback("Old database schema detected."); 129 | } else { 130 | callback(null); 131 | } 132 | }).error(function (err) { 133 | logger.log(err); 134 | callback(err); 135 | }); 136 | } 137 | } 138 | 139 | // Model definition 140 | var Job = JobUtils.getDatabase().define('Job', { 141 | id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true }, 142 | internalId: { type: Sequelize.STRING, defaultValue: null }, 143 | status: { type: Sequelize.STRING, defaultValue: StatusCodes.PROCESSING }, 144 | progress: { type: Sequelize.FLOAT, defaultValue: 0.0 }, 145 | duration: { type: Sequelize.INTEGER, defaultValue: 0 }, 146 | filesize: { type: Sequelize.INTEGER, defaultValue: 0 }, 147 | opts: { type: Sequelize.TEXT, defaultValue: null }, 148 | thumbnails: { type: Sequelize.TEXT, defaultValue: null }, 149 | message: { type: Sequelize.TEXT, defaultValue: null }, 150 | playlist: { type: Sequelize.STRING, defaultValue: null }, 151 | segments: { type: Sequelize.TEXT, defaultValue: null }, 152 | createdAt: Sequelize.DATE, 153 | updatedAt: Sequelize.DATE 154 | }, { 155 | classMethods: { 156 | prepareDatabase: function(callback) { 157 | JobUtils.verifyDatabase(function (err) { 158 | if (err) { 159 | callback(err); 160 | } else { 161 | JobUtils.migrateDatabase(function (err) { 162 | if (err) { 163 | callback(err); 164 | } else { 165 | JobUtils.markUnknownState(function (err) { 166 | err ? callback(err) : callback(null); 167 | }) 168 | } 169 | }); 170 | } 171 | }) 172 | }, 173 | create: function(opts, callback, completeCallback) { 174 | var job = Job.build({ opts: JSON.stringify(opts), internalId: Job.generateId() }); 175 | job.save().success(function(job) { 176 | job.prepare(function onPrepared(message) { 177 | if (message == "hasInput") job.hasInput = true; 178 | if (message == "hasOutputDir") job.hasOutputDir = true; 179 | if (job.hasInput && job.hasOutputDir) job.spawn(); 180 | }) 181 | job.completeCallback = completeCallback; 182 | callback(null, job); 183 | }).error(function(error) { 184 | /* 185 | Error while saving job, this appears to not always be called(!). 186 | Known issue in either sequelize or mysql package. 187 | */ 188 | logger.log("Could not write job " + job.internalId + " to the database."); 189 | callback('Unable to save new job to the database', null); 190 | }); 191 | }, 192 | generateId: function() { 193 | var hash = crypto.createHash('sha1'); 194 | var date = new Date(); 195 | 196 | hash.update([date, Math.random()].join('')); 197 | 198 | return hash.digest('hex'); 199 | } 200 | }, 201 | instanceMethods: { 202 | parsedOpts: function() { 203 | return JSON.parse(this.opts); 204 | }, 205 | 206 | prepare: function(callback) { 207 | var job = this; 208 | var destination_dir = path.dirname(path.normalize(job.parsedOpts()['destination_file'])); 209 | var mode = JobUtils.getMask(); 210 | 211 | fs.stat(job.parsedOpts()['source_file'], function(err, stats) { 212 | if (err) { 213 | job.exitHandler(-1, "unable to read input file (" + job.parsedOpts()['source_file'] + ")."); 214 | } else { 215 | if (stats.isFile()) { job.filesize = stats.size || Number.NaN }; 216 | callback("hasInput"); 217 | } 218 | }); 219 | 220 | mkdirp.mkdirp(destination_dir, mode, function(err) { 221 | if (err) { 222 | fs.stat(destination_dir, function(err, stats) { 223 | if (err) { 224 | job.exitHandler(-1, "unable to create output directory (" + destination_dir + ")."); 225 | } else { 226 | if (stats.isDirectory()) { 227 | callback("hasOutputDir"); 228 | } else { 229 | job.exitHandler(-1, "unable to create output directory (" + destination_dir + ")."); 230 | } 231 | } 232 | }); 233 | } else { 234 | callback("hasOutputDir"); 235 | } 236 | }); 237 | 238 | }, 239 | spawn: function() { 240 | var job = this; 241 | if (this.hasInput && this.hasOutputDir && !this.the_process) { 242 | var args = []; 243 | args.push('-i', this.parsedOpts()['source_file']); 244 | 245 | var extension = path.extname(this.parsedOpts()['destination_file']); 246 | var outputDir = path.dirname(this.parsedOpts()['destination_file']); 247 | 248 | if (this.parsedOpts()['encoder_options'].length > 0) { 249 | // "proper" transcoding job 250 | if (Array.isArray(this.parsedOpts()['encoder_options'])) { 251 | args = args.concat(this.parsedOpts()['encoder_options']); 252 | } else { 253 | args = args.concat(this.parsedOpts()['encoder_options'].replace(/\s+/g, " ").split(' ')); 254 | } 255 | var tmpFile = outputDir + path.sep + this.internalId + extension; 256 | 257 | if (config['use_scratch_dir'] == true) { 258 | tmpFile = config['scratch_dir'] + path.sep + this.internalId + extension; 259 | } 260 | var ssIndex = args.indexOf('-ss'); 261 | if (ssIndex > -1) { 262 | var ssValueTemp = args[ssIndex + 1]; 263 | args.splice(ssIndex, 2); 264 | Array.prototype.unshift.apply(args, ['-ss', ssValueTemp]); 265 | } 266 | args.push(tmpFile); 267 | 268 | job.tmpFile = tmpFile; 269 | } else { 270 | // thumbnail only job, but still need to find duration, so we start a "null" job 271 | var null_file = (!!os.platform().match(/^win/) ? 'nul' : '/dev/null'); 272 | args.push('-f', 'null', '-acodec', 'copy', '-vcodec', 'copy', '-y', null_file); 273 | } 274 | 275 | var the_process = child_process.spawn(config['encoder'], args); 276 | 277 | the_process.stderr.on('data', function(data) { job.progressHandler(data); }); 278 | the_process.on('error', function(err) { job.lastMessage = 'Unable to execute the ffmpeg binary: ' + err; job.didFinish(1); }); 279 | the_process.on('exit', function(code) { job.didFinish(code); }); 280 | 281 | this.the_process = the_process; 282 | } 283 | }, 284 | cancel: function() { 285 | if (this.the_process) { 286 | this.the_process.kill(); 287 | this.exitHandler(-1, 'job was cancelled'); 288 | } 289 | }, 290 | didFinish: function(code) { 291 | if (code != 0) { 292 | this.finalize(code); 293 | return; 294 | } 295 | 296 | this.processThumbnails({ 297 | error: function(job) { job.finalize(1); }, 298 | success: function(job) { 299 | job.processSegments({ 300 | error: function(job) { job.finalize(1); }, 301 | success: function(job) { job.finalize(0); } 302 | }) 303 | } 304 | }) 305 | }, 306 | 307 | processSegments: function(callbacks){ 308 | if (!this.parsedOpts()['segments_options']) { 309 | callbacks.success(this); 310 | return; 311 | } 312 | 313 | logger.log("Processing segments for job " + this.internalId + "."); 314 | 315 | var job = this; 316 | var args = []; 317 | var segmentsOpts = this.parsedOpts()['segments_options']; 318 | var segmentTime = segmentsOpts['segment_time']; 319 | var destinationFile = this.parsedOpts()['destination_file']; 320 | var playlistName = path.basename(destinationFile, path.extname(destinationFile)); 321 | var playlistDir = path.dirname(destinationFile); 322 | var playlistPath = [playlistDir, playlistName].join(path.sep) + '.m3u8'; 323 | var segmentsFormat = [playlistDir, playlistName].join(path.sep) + '-%06d.ts' 324 | var inputFile = (this.parsedOpts()['encoder_options'].length > 0) ? job.tmpFile : this.parsedOpts()['source_file'] 325 | 326 | args.push('-i', inputFile, 327 | '-codec', 'copy', '-map', '0', '-f', 'segment', 328 | '-vbsf', 'h264_mp4toannexb', '-flags', '-global_header', 329 | '-segment_format', 'mpegts', '-segment_list', playlistPath, 330 | '-segment_time', segmentTime, segmentsFormat); 331 | 332 | child_process.execFile(config['encoder'], args, { maxBuffer: 8192*1024 }, function(error, stdout, stderr) { 333 | if (error) { 334 | job.lastMessage = 'Error while generating segments: ' + error.message; 335 | callbacks.error(job); 336 | return; 337 | } 338 | 339 | fs.readdir(playlistDir, function(error, files) { 340 | if (error) { 341 | job.lastMessage = 'Error while generating segments: ' + error.message; 342 | callbacks.error(job); 343 | return; 344 | } 345 | 346 | job.segments = JSON.stringify(files.filter( 347 | function(file){ return file.match(new RegExp(playlistName + "-\\d+\\.ts")) } 348 | ).map( 349 | function(file){ return path.join(playlistDir, file) } 350 | )); 351 | 352 | job.playlist = playlistPath; 353 | 354 | callbacks.success(job); 355 | }); 356 | }); 357 | }, 358 | 359 | processThumbnails: function(callbacks) { 360 | if (!this.parsedOpts()['thumbnail_options']) { 361 | callbacks.success(this); 362 | return; 363 | } 364 | 365 | logger.log("Processing thumbnails for job " + this.internalId + "."); 366 | var thumbOpts = this.parsedOpts()['thumbnail_options']; 367 | var range = JobUtils.generateRangeFromThumbOpts(thumbOpts, this.duration); 368 | 369 | if (!range) { 370 | // no valid range 371 | logger.log("No valid thumbnails to process for job " + this.internalId + ". Skipping..."); 372 | callbacks.success(this); 373 | return; 374 | } 375 | 376 | var job = this; 377 | async.parallel( 378 | range.map(job.execThumbJob.bind(job)), 379 | function(err, results) { 380 | if (err) { 381 | job.lastMessage = err.message; 382 | callbacks.error(job); 383 | } else { 384 | job.thumbnails = JSON.stringify(results); 385 | callbacks.success(job); 386 | } 387 | } 388 | ); 389 | }, 390 | execThumbJob: function(offset) { 391 | var job = this; 392 | return function(callback) { 393 | var thumbOpts = job.parsedOpts()['thumbnail_options']; 394 | var args = ['-ss', offset, '-i', job.parsedOpts()['source_file'], '-vframes', '1', '-y']; 395 | 396 | // Explicit size provided 397 | if (thumbOpts['size'] && thumbOpts['size'] != 'src') { 398 | args.push('-s', thumbOpts['size']); 399 | } 400 | 401 | var format = thumbOpts['format'] ? thumbOpts['format'] : 'jpg'; 402 | var destinationFile = job.parsedOpts()['destination_file']; 403 | 404 | // Destination file + offset + extension 405 | var outputFile = JobUtils.generateThumbnailPath(destinationFile, offset, format); 406 | args.push(outputFile); 407 | 408 | var thumb_process = child_process.execFile(config['encoder'], args, { maxBuffer: 4096*1024 }, function(error, stdout, stderr) { 409 | if (error) { 410 | callback(new Error('Error while generating thumbnail: ' + error.message), null); 411 | } else { 412 | callback(null, outputFile); 413 | } 414 | }); 415 | } 416 | }, 417 | finalize: function(code) { 418 | var job = this; 419 | 420 | if (code != 0) { 421 | job.exitHandler(code, "ffmpeg finished with an error: '" + job.lastMessage + "' (" + code + ")."); 422 | return; 423 | } 424 | 425 | if (!job.tmpFile) { 426 | // No tmpFile, hence no transcoding, only thumbnails or segmenting 427 | if (job.parsedOpts()['thumbnail_options']) { 428 | job.exitHandler(code, 'finished thumbnail job.'); 429 | } else { 430 | job.exitHandler(code, 'finished segmenting job.'); 431 | } 432 | return; 433 | } 434 | 435 | fs.rename(job.tmpFile, job.parsedOpts()['destination_file'], function (err) { 436 | if (err) { 437 | if ( (err.message).match(/EXDEV/) ) { 438 | /* 439 | EXDEV fix, since util.pump is deprecated, using stream.pipe 440 | example from http://stackoverflow.com/questions/11293857/fastest-way-to-copy-file-in-node-js 441 | */ 442 | try { 443 | logger.log('ffmpeg finished successfully, trying to copy across partitions'); 444 | fs.createReadStream(job.tmpFile).pipe(fs.createWriteStream(job.parsedOpts()['destination_file'])); 445 | job.exitHandler(code, 'ffmpeg finished succesfully.'); 446 | } catch (err) { 447 | logger.log(err); 448 | job.exitHandler(-1, 'ffmpeg finished succesfully, but unable to move file to different partition (' + job.parsedOpts()['destination_file'] + ').'); 449 | } 450 | 451 | } else { 452 | logger.log(err); 453 | job.exitHandler(-1, 'ffmpeg finished succesfully, but unable to move file to destination (' + job.parsedOpts()['destination_file'] + ').'); 454 | } 455 | } else { 456 | job.exitHandler(code, 'ffmpeg finished succesfully.'); 457 | } 458 | }); 459 | }, 460 | toJSON: function() { 461 | var obj = { 462 | 'id': this.internalId, 463 | 'status': this.status, 464 | 'progress': this.progress, 465 | 'duration': this.duration, 466 | 'filesize': this.filesize, 467 | 'message': this.message, 468 | }; 469 | 470 | if (this.thumbnails) { 471 | obj['thumbnails'] = JSON.parse(this.thumbnails); 472 | } 473 | 474 | if (this.playlist) { obj['playlist'] = this.playlist; } 475 | if (this.segments) { 476 | obj['segments'] = JSON.parse(this.segments); 477 | } 478 | 479 | return obj; 480 | }, 481 | progressHandler: function(data) { 482 | if (this.hasExited) return; 483 | 484 | this.lastMessage = data.toString().replace("\n",''); 485 | 486 | (isNaN(this.duration) || this.duration == 0) ? this.extractDuration(data.toString()) : this.extractProgress(data.toString()); 487 | 488 | this.save().success(function() { 489 | // successfull save 490 | }).error(function(err) { 491 | // error while saving job 492 | }); 493 | }, 494 | extractDuration: function(text) { 495 | if (!this.durationBuffer) this.durationBuffer = ""; 496 | 497 | this.durationBuffer += text; 498 | var re = new RegExp(/Duration:\s+(\d{2}):(\d{2}):(\d{2}).(\d{1,2})/); 499 | var m = re.exec(this.durationBuffer); 500 | 501 | if (m != null) { 502 | var hours = parseInt(m[1], 10), minutes = parseInt(m[2], 10), seconds = parseInt(m[3], 10); 503 | 504 | this.duration = hours * 3600 + minutes * 60 + seconds; 505 | notifyHandler.notify(this); 506 | } 507 | }, 508 | extractProgress: function(text) { 509 | // 00:00:00 (hours, minutes, seconds) 510 | var re = new RegExp(/time=(\d{2}):(\d{2}):(\d{2})/); 511 | var m = re.exec(text); 512 | 513 | if (m != null) { 514 | var hours = parseInt(m[1], 10), minutes = parseInt(m[2], 10), seconds = parseInt(m[3], 10); 515 | var current = hours * 3600 + minutes * 60 + seconds; 516 | this.progress = current / this.duration; 517 | } else { 518 | // 00.00 (seconds, hundreds) 519 | re = new RegExp(/time=(\d+).(\d{2})/); 520 | m = re.exec(text); 521 | 522 | if (m != null) { 523 | var current = parseInt(m[1], 10); 524 | this.progress = current / this.duration; 525 | } 526 | } 527 | }, 528 | exitHandler: function(code, message) { 529 | this.hasExited = true 530 | this.message = message; 531 | 532 | if (code == 0) { 533 | this.progress = 1.0; 534 | } 535 | 536 | this.status = (code == 0 ? StatusCodes.SUCCESS : StatusCodes.FAILED); 537 | this.save().success(function() { 538 | // successfull save 539 | }).error(function(err) { 540 | // error while saving job 541 | logger.log("Error while saving job: " + err) 542 | }); 543 | this.completeCallback(); 544 | } 545 | } 546 | }); 547 | 548 | module.exports = Job; 549 | -------------------------------------------------------------------------------- /lib/logger.js: -------------------------------------------------------------------------------- 1 | var buffer = [], 2 | timer = null; 3 | 4 | var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; 5 | 6 | var isWorking = true; 7 | 8 | var Logger = function() { 9 | } 10 | 11 | function setupTimer() { 12 | // Set interval to flush buffer every 250ms 13 | setInterval(function() { 14 | if (buffer.length) { 15 | Logger.flush(); 16 | } 17 | }, 250); 18 | } 19 | 20 | function intPad(n) { 21 | return (n < 10 ? "0" + n.toString(10) : n.toString(10)); 22 | } 23 | 24 | function timestamp() { 25 | var d = new Date(); 26 | var time = [intPad(d.getHours()), intPad(d.getMinutes()), intPad(d.getSeconds())].join(':'); 27 | return [d.getDate(), months[d.getMonth()], time].join(' '); 28 | } 29 | 30 | Logger.log = function(string) { 31 | if (timer == null) { 32 | setupTimer(); 33 | } 34 | 35 | buffer.push(timestamp() + " - " + string); 36 | } 37 | 38 | Logger.isWorking = function() { 39 | return isWorking; 40 | } 41 | 42 | Logger.flush = function() { 43 | try { 44 | process.stdout.write(buffer.join("\n") + "\n"); 45 | buffer.length = 0; 46 | isWorking = true; 47 | } catch(err) { 48 | // Unable to write to stdout due to possible stream errors 49 | isWorking = false; 50 | } 51 | } 52 | 53 | Logger.stop = function() { 54 | if (timer !== null) { 55 | clearTimeout(timer); 56 | timer = null; 57 | } 58 | } 59 | module.exports = Logger; -------------------------------------------------------------------------------- /lib/notify-handler.js: -------------------------------------------------------------------------------- 1 | var url = require('url'), 2 | http = require('http'), 3 | https = require('https'), 4 | logger = require('./logger'); 5 | 6 | exports.notify = function(job) { 7 | var opts = job.parsedOpts(); 8 | var notificationTimestamp = new Date().getTime(); 9 | 10 | if (opts['callback_urls'] instanceof Array) { 11 | for (var u in opts['callback_urls']) { 12 | logger.log('Callback URL: ' + opts['callback_urls'][u]); 13 | try { 14 | var obj = url.parse(opts['callback_urls'][u]); 15 | var data = JSON.stringify(job); 16 | var urlOpts = { 17 | method: 'PUT', 18 | port: obj.port, 19 | host: obj.hostname, 20 | path: [obj.pathname, obj.search].join(''), 21 | headers: { 'Content-Type': 'application/json', 22 | 'Accept': 'application/json', 23 | 'Content-Length': data.length, 24 | 'X-Codem-Notify-Timestamp': notificationTimestamp 25 | } 26 | }; 27 | var requestHandler = obj.protocol == "https:" ? https : http; 28 | var req = requestHandler.request(urlOpts, function(res) { 29 | logger.log('Notification completed with HTTP status code: ' + res.statusCode); 30 | }).on('error', function(err) { 31 | logger.log("Failed delivering notification due to connection error: " + err); 32 | }); 33 | req.write(data); 34 | req.end(); 35 | } catch (error) { 36 | logger.log("Failed delivering notification: " + error); 37 | } 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /lib/probe-handler.js: -------------------------------------------------------------------------------- 1 | var os = require('os'), 2 | config = require('./config').load(), 3 | child_process = require('child_process'); 4 | 5 | exports.doProbe = function(postData, callback) { 6 | try { 7 | var obj = JSON.parse(postData); 8 | } catch(e) { 9 | callback("HTTP POST data contains no valid JSON object.", null); 10 | return; 11 | } 12 | 13 | var source_file = obj['source_file']; 14 | 15 | if (source_file) { 16 | var the_process = child_process.execFile( 17 | config['ffprobe'], 18 | ['-print_format', 'json', '-show_format', '-show_streams', source_file], 19 | function didFinishProbing(error, stdout, stderr) { 20 | if (error) { 21 | var lastMsg = error.message.trim().split("\n").pop(); 22 | callback(lastMsg, null); 23 | } else { 24 | try { 25 | var probe = JSON.parse(stdout); 26 | callback(null, probe); 27 | } catch(e) { 28 | callback("Error while parsing ffprobe JSON output.", null); 29 | } 30 | } 31 | } 32 | ); 33 | } else { 34 | callback("No source file was specified to probe.", null); 35 | } 36 | } -------------------------------------------------------------------------------- /lib/recoverable-stream.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'), 2 | logger = require('./logger'); 3 | 4 | var stream = null; 5 | var path = null; 6 | var opts = null; 7 | 8 | var RecoverableStream = function(streamPath, streamOpts) { 9 | path = streamPath; 10 | opts = streamOpts; 11 | reopen(); 12 | 13 | return this; 14 | } 15 | 16 | RecoverableStream.prototype.write = function(string) { 17 | stream.write(string); 18 | } 19 | 20 | RecoverableStream.prototype.end = function() { 21 | if (stream) { 22 | stream.end(); 23 | } 24 | } 25 | 26 | function reopen() { 27 | stream = fs.createWriteStream(path, opts); 28 | stream.on('error', function(err) { logger.log('Error while logging to access_log. ' + err); stream.end(); stream.destroy(); reopen(); }); 29 | } 30 | 31 | module.exports = RecoverableStream; -------------------------------------------------------------------------------- /lib/server.js: -------------------------------------------------------------------------------- 1 | var config = require('./config').load(), 2 | jobHandler = require('./job-handler'), 3 | probeHandler = require('./probe-handler'), 4 | logger = require('./logger'), 5 | express = require('express'), 6 | RecoverableStream = require('./recoverable-stream'); 7 | 8 | var rejectMessage = 'The transcoder is not accepting jobs right now. Please try again later.'; 9 | var rejectLoggerMessage = 'The transcoder is not accepting jobs due to a logger error. Please try again later.'; 10 | var acceptedMessage = 'The transcoder accepted your job.'; 11 | var notImplementedMessage = 'This method is not yet implemented.'; 12 | var badRequestMessage = 'The data supplied was not valid.'; 13 | var notFoundMessage = 'The specified job could not be found.'; 14 | var probeNotSupportedMessage = 'Probing files is not supported. Make sure you add the \'ffprobe\' configuration option.'; 15 | var probeErrorMessage = 'An error occurred while probing the file.'; 16 | var unableToDeleteMessage = 'The transcoder was not able to cancel and/or delete the job.'; 17 | var unableToSpawnMessage = 'The transcoder was not able to spawn a new job.'; 18 | var unableToPurgeMessage = 'The transcoder was not able to purge jobs.'; 19 | var purgeMessage = 'The transcoder successfully purged old jobs.'; 20 | 21 | var logfile = null; 22 | var server = null; 23 | 24 | exports.launch = function() { 25 | server = express.createServer(); 26 | logfile = new RecoverableStream(config['access_log'], { flags: 'a' }); 27 | server.use(express.logger({stream: logfile})); 28 | 29 | server.get( '/jobs', getJobs); 30 | server.post( '/jobs', postNewJob); 31 | server.get( '/jobs/:id', getJobStatus); 32 | server.delete('/jobs/purge/:age', purgeJobs); 33 | server.delete('/jobs/:id', removeJob); 34 | server.post( '/probe', probeFile); 35 | 36 | server.listen(config['port'], config['interface']); 37 | 38 | logger.log("Started server on interface " + config['interface'] + " port " + config['port'] + " with pid " + process.pid + "."); 39 | } 40 | 41 | exports.relaunch = function() { 42 | logger.log("Restarting..."); 43 | server.close(); 44 | logfile.end(); 45 | exports.launch(); 46 | } 47 | 48 | // POST /probe 49 | function probeFile(request, response) { 50 | var postData = ""; 51 | 52 | request.on('data', function(data) { postData += data; }) 53 | request.on('end', function() { processProbe(postData, response); } ); 54 | } 55 | 56 | // POST /jobs 57 | function postNewJob(request, response) { 58 | var postData = ""; 59 | 60 | request.on('data', function(data) { postData += data; }) 61 | request.on('end', function() { processPostedJob(postData, response); } ); 62 | } 63 | 64 | // GET /jobs 65 | function getJobs(request, response) { 66 | var content = { max_slots: config['slots'], free_slots: jobHandler.freeSlots(), jobs: jobHandler.slots }; 67 | 68 | try { 69 | response.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'}); 70 | response.end(JSON.stringify(content), 'utf8'); 71 | } catch(e) { 72 | logger.log("Error while returning all jobs: " + e + e.stack); 73 | } 74 | } 75 | 76 | // GET /jobs/:id 77 | function getJobStatus(request, response) { 78 | jobHandler.find(request.params.id, function onResult(err, job) { 79 | var body = {}; 80 | 81 | if (err || !job) { 82 | response.statusCode = 404; 83 | body['message'] = notFoundMessage; 84 | } else { 85 | response.statusCode = 200; 86 | body = job; 87 | } 88 | 89 | try { 90 | response.setHeader('Content-Type', 'application/json; charset=utf-8'); 91 | response.end(JSON.stringify(body), 'utf8'); 92 | } catch(e) { 93 | logger.log("Error while returning job status: " + e + e.stack); 94 | } 95 | }); 96 | } 97 | 98 | // DELETE /jobs/:id 99 | function removeJob(request, response) { 100 | jobHandler.cancelAndRemove(request.params.id, function onResult(err, job) { 101 | var body = {}; 102 | 103 | if (err) { 104 | if (job) { 105 | response.statusCode = 500; 106 | body['message'] = unableToDeleteMessage; 107 | } else { 108 | response.statusCode = 404; 109 | body['message'] = notFoundMessage; 110 | } 111 | } else { 112 | response.statusCode = 200; 113 | body = job; 114 | } 115 | 116 | try { 117 | response.setHeader('Content-Type', 'application/json; charset=utf-8'); 118 | response.end(JSON.stringify(body), 'utf8'); 119 | } catch(e) { 120 | logger.log("Error while returning deleted job: " + e + e.stack); 121 | } 122 | }); 123 | } 124 | 125 | // DELETE /jobs/purge 126 | function purgeJobs(request, response) { 127 | jobHandler.purgeJobs(request.params.age, function onPurge(err, count) { 128 | var body = {}; 129 | if (err) { 130 | response.statusCode = 500; 131 | body['message'] = unableToPurgeMessage + " " + err; 132 | } else { 133 | response.statusCode = 200; 134 | body['message'] = purgeMessage + ' (' + count + ')'; 135 | } 136 | 137 | try { 138 | response.setHeader('Content-Type', 'application/json; charset=utf-8'); 139 | response.end(JSON.stringify(body), 'utf8'); 140 | } catch(e) { 141 | logger.log("Error while returning purged jobs: " + e + e.stack); 142 | } 143 | }); 144 | } 145 | 146 | function rejectRequestDueToLogger(response) { 147 | var body = {}; 148 | body['message'] = rejectLoggerMessage; 149 | response.statusCode = 503; 150 | 151 | try { 152 | response.end(JSON.stringify(body), 'utf8'); 153 | } catch(e) { 154 | logger.log("Error while sending response: " + e + e.stack); 155 | } 156 | } 157 | 158 | function processPostedJob(postData, response) { 159 | response.setHeader('Content-Type', 'application/json; charset=utf-8'); 160 | 161 | if (!logger.isWorking()) { 162 | rejectRequestDueToLogger(response); 163 | } else { 164 | jobHandler.processJobRequest(postData, function onProcessed(err, job) { 165 | var body = {}; 166 | 167 | if (err) { 168 | switch(err.type) { 169 | case 'invalid': 170 | body['message'] = badRequestMessage; 171 | if (err['missingFields'] && err['missingFields'].length > 0) { 172 | body['message'] += " Missing fields: " + err['missingFields'].join(", ") + "."; 173 | } 174 | response.statusCode = 400; 175 | break; 176 | case 'spawn': 177 | body['message'] = unableToSpawnMessage; 178 | response.statusCode = 500; 179 | break; 180 | case 'full': 181 | case 'shutdown': 182 | body['message'] = rejectMessage; 183 | response.statusCode = 503; 184 | break; 185 | } 186 | } else { 187 | body['message'] = acceptedMessage; 188 | body['job_id'] = job.internalId; 189 | response.statusCode = 202; 190 | } 191 | 192 | try { 193 | response.end(JSON.stringify(body), 'utf8'); 194 | } catch(e) { 195 | logger.log("Error while posting new job: " + e + e.stack); 196 | } 197 | }); 198 | } 199 | } 200 | 201 | function processProbe(postData, response) { 202 | response.setHeader('Content-Type', 'application/json; charset=utf-8'); 203 | var body = {}; 204 | 205 | if (!logger.isWorking()) { 206 | rejectRequestDueToLogger(response); 207 | } else if (config['ffprobe']) { 208 | logger.log("Starting probe for: " + postData); 209 | 210 | probeHandler.doProbe(postData, function onProbed(err, probe) { 211 | if (err) { 212 | response.statusCode = 500; 213 | body['message'] = probeErrorMessage + " " + err; 214 | logger.log("Error while probing: " + body['message']); 215 | } else { 216 | response.statusCode = 200; 217 | body['ffprobe'] = probe; 218 | } 219 | 220 | try { 221 | response.end(JSON.stringify(body), 'utf8'); 222 | } catch(e) { 223 | logger.log("Error while sending probe response: " + e + e.stack); 224 | } 225 | 226 | }); 227 | } else { 228 | // ffprobe not supported 229 | response.statusCode = 400; 230 | body['message'] = probeNotSupportedMessage; 231 | logger.log("An attempt was made to probe a file, but ffprobe support was not configured."); 232 | 233 | try { 234 | response.end(JSON.stringify(body), 'utf8'); 235 | } catch(e) { 236 | logger.log("Error while sending probe response: " + e + e.stack); 237 | } 238 | } 239 | } -------------------------------------------------------------------------------- /lib/transcoder.js: -------------------------------------------------------------------------------- 1 | var logger = require('./logger'), 2 | server = require('./server'), 3 | config = require('./config').load(), 4 | Job = require('./job'); 5 | 6 | var Transcoder = function() { 7 | } 8 | 9 | Transcoder.prototype.boot = function() { 10 | this.addSignalHandlers(); 11 | Job.prepareDatabase(function (err) { 12 | if (err) { 13 | // something went wrong 14 | logger.log("Error while preparing database for the transcoder."); 15 | process.exit(-1); 16 | } 17 | }); 18 | server.launch(); 19 | } 20 | 21 | Transcoder.prototype.addSignalHandlers = function() { 22 | var transcoder = this; 23 | 24 | process.on('uncaughtException', function onException(err) { logger.log('Caught exception: ' + err); }); 25 | 26 | process.on('SIGTERM', function onSigTerm() { transcoder.shutdown('SIGTERM'); }); 27 | process.on('SIGINT', function onSigInt() { transcoder.shutdown('SIGINT'); }); 28 | process.on('SIGUSR1', function onSigUsr1() { server.relaunch(); }); 29 | 30 | process.on('exit', function onExit() { logger.flush(); logger.stop(); }); 31 | } 32 | 33 | Transcoder.prototype.shutdown = function(signal) { 34 | logger.log('Received ' + signal); 35 | process.exit(0); 36 | } 37 | 38 | module.exports = Transcoder; -------------------------------------------------------------------------------- /migrations/20130305100924-create-jobs.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | up: function(migration, DataTypes, done) { 3 | // add altering commands here 4 | migration.createTable('Jobs', 5 | { 6 | id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true }, 7 | internalId: { type: DataTypes.STRING, defaultValue: null }, 8 | status: { type: DataTypes.STRING, defaultValue: "processing" }, 9 | progress: { type: DataTypes.FLOAT, defaultValue: 0.0 }, 10 | duration: { type: DataTypes.INTEGER, defaultValue: 0 }, 11 | filesize: { type: DataTypes.INTEGER, defaultValue: 0 }, 12 | opts: { type: DataTypes.TEXT, defaultValue: null }, 13 | message: { type: DataTypes.TEXT, defaultValue: null }, 14 | createdAt: DataTypes.DATE, 15 | updatedAt: DataTypes.DATE 16 | }, 17 | { 18 | charset: 'utf8', 19 | collate: 'utf8_general_ci' 20 | } 21 | ) 22 | migration.addIndex('Jobs', ['internalId']).complete(done); 23 | }, 24 | down: function(migration, DataTypes, done) { 25 | // add reverting commands here 26 | migration.dropTable('Jobs').complete(done); 27 | } 28 | } -------------------------------------------------------------------------------- /migrations/20130702120654-add-thumbnails-to-jobs.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | up: function(migration, DataTypes, done) { 3 | // add altering commands here 4 | migration.addColumn('Jobs', 'thumbnails', { type: DataTypes.TEXT, defaultValue: null }).complete(done); 5 | }, 6 | down: function(migration, DataTypes, done) { 7 | // add reverting commands here 8 | migration.removeColumn('Jobs', 'thumbnails').complete(done); 9 | } 10 | } -------------------------------------------------------------------------------- /migrations/20140822085031-add-playlist-and-segments-to-jobs.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | up: function(migration, DataTypes, done) { 3 | // add altering commands here 4 | migration.addColumn('Jobs', 'playlist', { type: DataTypes.STRING, defaultValue: null }).complete(done); 5 | migration.addColumn('Jobs', 'segments', { type: DataTypes.TEXT, defaultValue: null }).complete(done); 6 | }, 7 | down: function(migration, DataTypes, done) { 8 | // add reverting commands here 9 | migration.removeColumn('Jobs', 'playlist').complete(done); 10 | migration.removeColumn('Jobs', 'segments').complete(done); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "codem-transcode", 3 | "description": "Offline video transcoding using ffmpeg, with a small HTTP API.", 4 | "version": "0.5.11", 5 | "keywords": ["transcoding", "ffmpeg", "video"], 6 | "homepage": "https://github.com/madebyhiro/codem-transcode", 7 | "repository": { 8 | "type": "git", 9 | "url": "https://github.com/madebyhiro/codem-transcode.git" 10 | }, 11 | "bin": { "codem-transcode": "./bin/codem-transcode" }, 12 | "engines": { 13 | "node": ">=0.8.11" 14 | }, 15 | "dependencies": { 16 | "sequelize": "~1.6.0", 17 | "sqlite3": "~2.2.0", 18 | "mkdirp": "~0.3.4", 19 | "express": "~2.5.11", 20 | "argsparser": "~0.0.6", 21 | "async": "~0.2.9" 22 | }, 23 | "license": "MIT" 24 | } --------------------------------------------------------------------------------