├── .gitignore ├── README.md ├── config.js ├── config.local.js.orig ├── controllers ├── api.js ├── debug.js └── fingerprinter.js ├── docs └── node-echoprint-debug01.png ├── index.js ├── logs └── .keep ├── models └── mysql.js ├── mutex.js ├── mysql.sql ├── package.json ├── server.js └── views └── debug.jade /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .DS_Store 3 | node_modules/ 4 | logs/*.log 5 | 6 | config.local.js 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # node-echoprint-server # 2 | 3 | A node.js music identification server that is compatible with the 4 | [Echoprint](http://echoprint.me/) music fingerprinting client. This server is 5 | based on the original 6 | [echoprint-server](https://github.com/echonest/echoprint-server) but attempts 7 | to improve on ease of installation, speed, matching accuracy, and ease of 8 | development/debugging. 9 | 10 | ## Dependencies ## 11 | 12 | * [node.js](http://nodejs.org/) - Tested with 0.6.10 13 | * [MySQL](http://mysql.com/) - Tested with 5.5.20 14 | 15 | To generate audio fingerprints you will need the 16 | [echoprint-codegen](https://github.com/echonest/echoprint-codegen) client. 17 | 18 | ## Installation ## 19 | 20 | Clone this repository, enter the `node-echoprint-server` directory and run 21 | `npm install` to fetch the required dependencies. Import the `mysql.sql` file 22 | into your MySQL database. Next, copy `config.local.js.orig` to 23 | `config.local.js` and modify it to suit your environment. Make sure the 24 | database settings point to the MySQL database you just imported. Finally, run 25 | `node index.js` to start the server. 26 | 27 | ## Usage ## 28 | 29 | The server will listen on the port configured in `config.local.js` and exposes 30 | two API endpoints. 31 | 32 | ### POST /ingest 33 | 34 | Adds a new music fingerprint to the database if the given fingerprint is 35 | unique, otherwise the existing track information is returned. 36 | 37 | Required fields: 38 | 39 | * `code` - The code string output by echoprint-codegen 40 | * `version` - metadata.version field output by echoprint-codegen 41 | * `length` - Length in seconds of the track. duration field output by 42 | echoprint-codegen 43 | 44 | Optional fields: 45 | 46 | * `track` - Name of the track 47 | * `artist` - Track artist 48 | 49 | The response is a JSON object containing `track_id`, `track`, `artist_id`, 50 | `artist` on success or `error` string on failure. 51 | 52 | ### GET /query?code=...&version=... 53 | 54 | Queries for a track matching the given fingerprint. `code` and `version` 55 | query parameters are both required. The response is a JSON object 56 | containing a `success` boolean, `status` string, `match` object on 57 | successful match, or `error` string if something went wrong. 58 | 59 | ### GET /debug 60 | 61 | The /debug endpoint can be visited in a browser and provides a human-friendly 62 | way of querying for a match and observing results. Here is a screenshot of the 63 | debug interface in action: 64 | 65 | ![](https://github.com/jhurliman/node-echoprint-server/raw/master/docs/node-echoprint-debug01.png) 66 | 67 | ## Sponsors ## 68 | 69 | This server has been released as open source by 70 | [John Hurliman](http://jhurliman.org/) at [cull.tv](http://cull.tv). 71 | 72 | ## License ## 73 | 74 | Uses code from 75 | [echoprint-server](https://github.com/echonest/echoprint-server), which is 76 | released by The Echo Nest Corporation under the 77 | [Apache 2 License](https://github.com/echonest/echoprint-server/blob/master/LICENSE). 78 | 79 | (The MIT License) 80 | 81 | Copyright (c) 2012 Cull TV, Inc. <jhurliman@cull.tv> 82 | 83 | Permission is hereby granted, free of charge, to any person obtaining 84 | a copy of this software and associated documentation files (the 85 | 'Software'), to deal in the Software without restriction, including 86 | without limitation the rights to use, copy, modify, merge, publish, 87 | distribute, sublicense, and/or sell copies of the Software, and to 88 | permit persons to whom the Software is furnished to do so, subject to 89 | the following conditions: 90 | 91 | The above copyright notice and this permission notice shall be 92 | included in all copies or substantial portions of the Software. 93 | 94 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 95 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 96 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 97 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 98 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 99 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 100 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 101 | -------------------------------------------------------------------------------- /config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Configuration variables. These can be overridden in the per-system config file 3 | */ 4 | 5 | var log = require('winston'); 6 | 7 | var settings = { 8 | // Port that the web server will bind to 9 | web_port: 37760, 10 | 11 | // Database settings 12 | db_user: 'root', 13 | db_pass: '', 14 | db_database: 'echoprint', 15 | db_host: 'localhost', 16 | 17 | // Set this to a system username to drop root privileges 18 | run_as_user: '', 19 | 20 | // Filename to log to 21 | log_path: __dirname + '/logs/echoprint.log', 22 | // Log level. Valid values are debug, info, warn, error 23 | log_level: 'debug', 24 | 25 | // Minimum number of codes that must be matched to consider a fingerprint 26 | // match valid 27 | code_threshold: 10, 28 | 29 | // Supported version of echoprint-codegen codes 30 | codever: '4.12' 31 | }; 32 | 33 | // Override default settings with any local settings 34 | try { 35 | var localSettings = require('./config.local'); 36 | 37 | for (var property in localSettings) { 38 | if (localSettings.hasOwnProperty(property)) 39 | settings[property] = localSettings[property]; 40 | } 41 | 42 | log.info('Loaded settings from config.local.js. Database is ' + 43 | settings.db_database + '@' + settings.db_host); 44 | } catch (err) { 45 | log.warn('Using default settings from config.js. Database is ' + 46 | settings.db_database + '@' + settings.db_host); 47 | } 48 | 49 | module.exports = settings; 50 | -------------------------------------------------------------------------------- /config.local.js.orig: -------------------------------------------------------------------------------- 1 | /** 2 | * Local configuration variables 3 | */ 4 | 5 | module.exports = { 6 | // Port that the web server will bind to 7 | web_port: 37760, 8 | 9 | // Database settings 10 | db_user: 'root', 11 | db_pass: '', 12 | db_database: 'echoprint', 13 | db_host: 'localhost', 14 | 15 | // Set this to a system username to drop root privileges 16 | run_as_user: '', 17 | 18 | // Filename to log to 19 | log_path: __dirname + '/logs/echoprint.log', 20 | // Log level. Valid values are debug, info, warn, error 21 | log_level: 'debug', 22 | 23 | // Minimum number of codes that must be matched to consider a fingerprint 24 | // match valid 25 | code_threshold: 10, 26 | 27 | // Supported version of echoprint-codegen codes 28 | codever: '4.12' 29 | }; 30 | -------------------------------------------------------------------------------- /controllers/api.js: -------------------------------------------------------------------------------- 1 | var urlParser = require('url'); 2 | var log = require('winston'); 3 | var fingerprinter = require('./fingerprinter'); 4 | var server = require('../server'); 5 | var config = require('../config'); 6 | 7 | /** 8 | * Querying for the closest matching track. 9 | */ 10 | exports.query = function(req, res) { 11 | var url = urlParser.parse(req.url, true); 12 | var code = url.query.code; 13 | if (!code) 14 | return server.respond(req, res, 500, { error: 'Missing code' }); 15 | 16 | var codeVer = url.query.version; 17 | if (codeVer != config.codever) 18 | return server.respond(req, res, 500, { error: 'Missing or invalid version' }); 19 | 20 | fingerprinter.decodeCodeString(code, function(err, fp) { 21 | if (err) { 22 | log.error('Failed to decode codes for query: ' + err); 23 | return server.respond(req, res, 500, { error: 'Failed to decode codes for query: ' + err }); 24 | } 25 | 26 | fp.codever = codeVer; 27 | 28 | fingerprinter.bestMatchForQuery(fp, config.code_threshold, function(err, result) { 29 | if (err) { 30 | log.warn('Failed to complete query: ' + err); 31 | return server.respond(req, res, 500, { error: 'Failed to complete query: ' + err }); 32 | } 33 | 34 | var duration = new Date() - req.start; 35 | log.debug('Completed lookup in ' + duration + 'ms. success=' + 36 | !!result.success + ', status=' + result.status); 37 | 38 | return server.respond(req, res, 200, { success: !!result.success, 39 | status: result.status, match: result.match || null }); 40 | }); 41 | }); 42 | }; 43 | 44 | /** 45 | * Adding a new track to the database. 46 | */ 47 | exports.ingest = function(req, res) { 48 | var code = req.body.code; 49 | var codeVer = req.body.version; 50 | var length = req.body.length; 51 | var track = req.body.track; 52 | var artist = req.body.artist; 53 | 54 | if (!code) 55 | return server.respond(req, res, 500, { error: 'Missing "code" field' }); 56 | if (!codeVer) 57 | return server.respond(req, res, 500, { error: 'Missing "version" field' }); 58 | if (codeVer != config.codever) 59 | return server.respond(req, res, 500, { error: 'Version "' + codeVer + '" does not match required version "' + config.codever + '"' }); 60 | if (isNaN(parseInt(length, 10))) 61 | return server.respond(req, res, 500, { error: 'Missing or invalid "length" field' }); 62 | if (!track) 63 | return server.respond(req, res, 500, { error: 'Missing "track" field' }); 64 | if (!artist) 65 | return server.respond(req, res, 500, { error: 'Missing "artist" field' }); 66 | 67 | fingerprinter.decodeCodeString(code, function(err, fp) { 68 | if (err || !fp.codes.length) { 69 | log.error('Failed to decode codes for ingest: ' + err); 70 | return server.respond(req, res, 500, { error: 'Failed to decode codes for ingest: ' + err }); 71 | } 72 | 73 | fp.codever = codeVer; 74 | fp.track = track; 75 | fp.length = length; 76 | fp.artist = artist; 77 | 78 | fingerprinter.ingest(fp, function(err, result) { 79 | if (err) { 80 | log.error('Failed to ingest track: ' + err); 81 | return server.respond(req, res, 500, { error: 'Failed to ingest track: ' + err }); 82 | } 83 | 84 | var duration = new Date() - req.start; 85 | log.debug('Ingested new track in ' + duration + 'ms. track_id=' + 86 | result.track_id + ', artist_id=' + result.artist_id); 87 | 88 | result.success = true; 89 | return server.respond(req, res, 200, result); 90 | }); 91 | }); 92 | }; 93 | -------------------------------------------------------------------------------- /controllers/debug.js: -------------------------------------------------------------------------------- 1 | var async = require('async'); 2 | var urlParser = require('url'); 3 | var log = require('winston'); 4 | var fingerprinter = require('./fingerprinter'); 5 | var server = require('../server'); 6 | var config = require('../config'); 7 | 8 | /** 9 | * Browser-friendly query debugging endpoint. 10 | */ 11 | exports.debugQuery = function(req, res) { 12 | if (!req.body || !req.body.json) 13 | return server.renderView(req, res, 200, 'debug.jade', {}); 14 | 15 | var json, code, codeVer; 16 | try { 17 | json = JSON.parse(req.body.json)[0]; 18 | code = json.code; 19 | codeVer = json.metadata.version.toString(); 20 | } catch (err) { 21 | log.warn('Failed to parse JSON debug input: ' + err); 22 | } 23 | 24 | if (!code || !codeVer || codeVer.length !== 4) { 25 | return server.renderView(req, res, 500, 'debug.jade', 26 | { err: 'Unrecognized input' }); 27 | } 28 | 29 | if (req.body.Ingest) { 30 | delete req.body.json; 31 | req.body.code = code; 32 | req.body.version = codeVer; 33 | req.body.track = json.metadata.title; 34 | req.body.length = json.metadata.duration; 35 | req.body.artist = json.metadata.artist; 36 | return require('./api').ingest(req, res); 37 | } 38 | 39 | fingerprinter.decodeCodeString(code, function(err, fp) { 40 | if (err) { 41 | log.error('Failed to decode codes for debug query: ' + err); 42 | return server.renderView(req, res, 500, 'debug.jade', 43 | { err: 'Failed to decode codes for debug query: ' + err }); 44 | } 45 | 46 | fp.codever = codeVer; 47 | fp = fingerprinter.cutFPLength(fp); 48 | 49 | fingerprinter.bestMatchForQuery(fp, config.code_threshold, 50 | function(err, result, allMatches) 51 | { 52 | if (err) { 53 | log.warn('Failed to complete debug query: ' + err); 54 | return server.renderView(req, res, 500, 'debug.jade', 55 | { err: 'Failed to complete debug query: ' + err, input: req.body.json }); 56 | } 57 | 58 | var duration = new Date() - req.start; 59 | log.debug('Completed debug lookup in ' + duration + 'ms. success=' + 60 | !!result.success + ', status=' + result.status); 61 | 62 | // TODO: Determine a useful set of data to return about the query and 63 | // each match and return it in an HTML view 64 | if (allMatches) { 65 | async.forEach(allMatches, 66 | function(match, done) { 67 | fingerprinter.getTrackMetadata(match, null, null, function(err) { 68 | match.codeLength = Math.ceil(match.length * fingerprinter.SECONDS_TO_TIMESTAMP); 69 | // Find each match that contributed to ascore 70 | getContributors(fp, match); 71 | delete match.codes; 72 | delete match.times; 73 | 74 | done(err); 75 | }); 76 | }, 77 | function(err) { 78 | if (err) { 79 | return server.renderView(req, res, 500, 'debug.jade', 80 | { err: 'Metadata lookup failed:' + err }); 81 | } 82 | 83 | renderView(); 84 | } 85 | ); 86 | } else { 87 | renderView(); 88 | } 89 | 90 | function renderView() { 91 | var json = JSON.stringify({ success: !!result.success, status: result.status, 92 | queryLen: fp.codes.length, matches: allMatches, queryTime: duration }); 93 | return server.renderView(req, res, 200, 'debug.jade', { res: json, 94 | input: req.body.json }); 95 | } 96 | }); 97 | }); 98 | }; 99 | 100 | /** 101 | * Attach an array called contributors to the match object that contains one 102 | * entry for each matched code that is contributing to the final match score. 103 | * Used by the client-side JS to draw pretty pictures. 104 | */ 105 | function getContributors(fp, match) { 106 | var MAX_DIST = 32767; 107 | var i, j, k; 108 | 109 | match.contributors = []; 110 | 111 | if (match.codes.length < config.code_threshold) 112 | return; 113 | 114 | // Find the top two entries in the match histogram 115 | var keys = Object.keys(match.histogram); 116 | var array = new Array(keys.length); 117 | for (i = 0; i < keys.length; i++) 118 | array[i] = [ parseInt(keys[i], 10), match.histogram[keys[i]] ]; 119 | array.sort(function(a, b) { return b[1] - a[1]; }); 120 | var topOffsets = array.splice(0, 2); 121 | 122 | var matchCodesToTimes = fingerprinter.getCodesToTimes(match, fingerprinter.MATCH_SLOP); 123 | 124 | // Iterate over each {code,time} tuple in the query 125 | for (i = 0; i < fp.codes.length; i++) { 126 | var code = fp.codes[i]; 127 | var time = Math.floor(fp.times[i] / fingerprinter.MATCH_SLOP) * fingerprinter.MATCH_SLOP; 128 | 129 | var matchTimes = matchCodesToTimes[code]; 130 | if (matchTimes) { 131 | for (j = 0; j < matchTimes.length; j++) { 132 | var dist = Math.abs(time - matchTimes[j]); 133 | 134 | // If dist is in topOffsets, add a contributor object 135 | for (k = 0; k < topOffsets.length; k++) { 136 | if (dist === topOffsets[k][0]) { 137 | match.contributors.push({ 138 | code: code, 139 | time: matchTimes[j], 140 | dist: dist 141 | }); 142 | break; 143 | } 144 | } 145 | } 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /controllers/fingerprinter.js: -------------------------------------------------------------------------------- 1 | var zlib = require('zlib'); 2 | var log = require('winston'); 3 | var Mutex = require('../mutex'); 4 | var config = require('../config'); 5 | var database = require('../models/mysql'); 6 | 7 | // Constants 8 | var CHARACTERS = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'; 9 | var SECONDS_TO_TIMESTAMP = 43.45; 10 | var MAX_ROWS = 30; 11 | var MIN_MATCH_PERCENT = 0.1; 12 | var MATCH_SLOP = 2; 13 | 14 | // Exports 15 | exports.decodeCodeString = decodeCodeString; 16 | exports.cutFPLength = cutFPLength; 17 | exports.getCodesToTimes = getCodesToTimes; 18 | exports.bestMatchForQuery = bestMatchForQuery; 19 | exports.getTrackMetadata = getTrackMetadata; 20 | exports.ingest = ingest; 21 | exports.SECONDS_TO_TIMESTAMP = SECONDS_TO_TIMESTAMP; 22 | exports.MATCH_SLOP = MATCH_SLOP; 23 | 24 | // Globals 25 | var gTimestamp = +new Date(); 26 | var gMutex = Mutex.getMutex(); 27 | 28 | /** 29 | * Takes a base64 encoded representation of a zlib-compressed code string 30 | * and passes a fingerprint object to the callback. 31 | */ 32 | function decodeCodeString(codeStr, callback) { 33 | // Fix url-safe characters 34 | codeStr = codeStr.replace(/-/g, '+').replace(/_/g, '/'); 35 | 36 | // Expand the base64 data into a binary buffer 37 | var compressed = new Buffer(codeStr, 'base64'); 38 | 39 | // Decompress the binary buffer into ascii hex codes 40 | zlib.inflate(compressed, function(err, uncompressed) { 41 | if (err) return callback(err, null); 42 | // Convert the ascii hex codes into codes and time offsets 43 | var fp = inflateCodeString(uncompressed); 44 | log.debug('Inflated ' + codeStr.length + ' byte code string into ' + 45 | fp.codes.length + ' codes'); 46 | 47 | callback(null, fp); 48 | }); 49 | } 50 | 51 | /** 52 | * Takes an uncompressed code string consisting of zero-padded fixed-width 53 | * sorted hex integers and converts it to the standard code string. 54 | */ 55 | function inflateCodeString(buf) { 56 | // 5 hex bytes for hash, 5 hex bytes for time (40 bits per tuple) 57 | var count = Math.floor(buf.length / 5); 58 | var endTimestamps = count / 2; 59 | var i; 60 | 61 | var codes = new Array(count / 2); 62 | var times = new Array(count / 2); 63 | 64 | for (i = 0; i < endTimestamps; i++) { 65 | times[i] = parseInt(buf.toString('ascii', i * 5, i * 5 + 5), 16); 66 | } 67 | for (i = endTimestamps; i < count; i++) { 68 | codes[i - endTimestamps] = parseInt(buf.toString('ascii', i * 5, i * 5 + 5), 16); 69 | } 70 | 71 | // Sanity check 72 | for (i = 0; i < codes.length; i++) { 73 | if (isNaN(codes[i]) || isNaN(times[i])) { 74 | log.error('Failed to parse code/time index ' + i); 75 | return { codes: [], times: [] }; 76 | } 77 | } 78 | 79 | return { codes: codes, times: times }; 80 | } 81 | 82 | /** 83 | * Clamp this fingerprint to a maximum N seconds worth of codes. 84 | */ 85 | function cutFPLength(fp, maxSeconds) { 86 | if (!maxSeconds) maxSeconds = 60; 87 | 88 | var newFP = {}; 89 | for(var key in fp) { 90 | if (fp.hasOwnProperty(key)) 91 | newFP[key] = fp[key]; 92 | } 93 | 94 | var firstTimestamp = fp.times[0]; 95 | var sixtySeconds = maxSeconds * SECONDS_TO_TIMESTAMP + firstTimestamp; 96 | 97 | for (var i = 0; i < fp.times.length; i++) { 98 | if (fp.times[i] > sixtySeconds) { 99 | log.debug('Clamping ' + fp.codes.length + ' codes to ' + i + ' codes'); 100 | 101 | newFP.codes = fp.codes.slice(0, i); 102 | newFP.times = fp.times.slice(0, i); 103 | return newFP; 104 | } 105 | } 106 | 107 | newFP.codes = fp.codes.slice(0); 108 | newFP.times = fp.times.slice(0); 109 | return newFP; 110 | } 111 | 112 | /** 113 | * Finds the closest matching track, if any, to a given fingerprint. 114 | */ 115 | function bestMatchForQuery(fp, threshold, callback) { 116 | fp = cutFPLength(fp); 117 | 118 | if (!fp.codes.length) 119 | return callback('No valid fingerprint codes specified', null); 120 | 121 | log.debug('Starting query with ' + fp.codes.length + ' codes'); 122 | 123 | database.fpQuery(fp, MAX_ROWS, function(err, matches) { 124 | if (err) return callback(err, null); 125 | 126 | if (!matches || !matches.length) { 127 | log.debug('No matched tracks'); 128 | return callback(null, { status: 'NO_RESULTS' }); 129 | } 130 | 131 | log.debug('Matched ' + matches.length + ' tracks, top code overlap is ' + 132 | matches[0].score); 133 | 134 | // If the best result matched fewer codes than our percentage threshold, 135 | // report no results 136 | if (matches[0].score < fp.codes.length * MIN_MATCH_PERCENT) 137 | return callback(null, { status: 'MULTIPLE_BAD_HISTOGRAM_MATCH' }); 138 | 139 | // Compute more accurate scores for each track by taking time offsets into 140 | // account 141 | var newMatches = []; 142 | for (var i = 0; i < matches.length; i++) { 143 | var match = matches[i]; 144 | match.ascore = getActualScore(fp, match, threshold, MATCH_SLOP); 145 | if (match.ascore && match.ascore >= fp.codes.length * MIN_MATCH_PERCENT) 146 | newMatches.push(match); 147 | } 148 | matches = newMatches; 149 | 150 | if (!matches.length) { 151 | log.debug('No matched tracks after score adjustment'); 152 | return callback(null, { status: 'NO_RESULTS_HISTOGRAM_DECREASED' }); 153 | } 154 | 155 | // Sort the matches based on actual score 156 | matches.sort(function(a, b) { return b.ascore - a.ascore; }); 157 | 158 | // If we only had one track match, just use the threshold to determine if 159 | // the match is good enough 160 | if (matches.length === 1) { 161 | if (matches[0].ascore / fp.codes.length >= MIN_MATCH_PERCENT) { 162 | // Fetch metadata for the single match 163 | log.debug('Single good match with actual score ' + matches[0].ascore + 164 | '/' + fp.codes.length); 165 | return getTrackMetadata(matches[0], matches, 166 | 'SINGLE_GOOD_MATCH_HISTOGRAM_DECREASED', callback); 167 | } else { 168 | log.debug('Single bad match with actual score ' + matches[0].ascore + 169 | '/' + fp.codes.length); 170 | return callback(null, { status: 'SINGLE_BAD_MATCH' }); 171 | } 172 | } 173 | 174 | var origTopScore = matches[0].ascore; 175 | 176 | // Sort by the new adjusted score 177 | matches.sort(function(a, b) { return b.ascore - a.score; }); 178 | 179 | var topMatch = matches[0]; 180 | var newTopScore = topMatch.ascore; 181 | 182 | log.debug('Actual top score is ' + newTopScore + ', next score is ' + 183 | matches[1].ascore); 184 | 185 | // If the best result actually matched fewer codes than our percentage 186 | // threshold, report no results 187 | if (newTopScore < fp.codes.length * MIN_MATCH_PERCENT) 188 | return callback(null, { status: 'MULTIPLE_BAD_HISTOGRAM_MATCH' }); 189 | 190 | // If the actual score was not close enough, then no match 191 | if (newTopScore <= origTopScore / 2) 192 | return callback(null, { status: 'MULTIPLE_BAD_HISTOGRAM_MATCH' }); 193 | 194 | // If the difference in actual scores between the first and second matches 195 | // is not significant enough, then no match 196 | if (newTopScore - matches[1].ascore < newTopScore / 2) 197 | return callback(null, { status: 'MULTIPLE_BAD_HISTOGRAM_MATCH' }); 198 | 199 | // Fetch metadata for the top track 200 | getTrackMetadata(topMatch, matches, 201 | 'MULTIPLE_GOOD_MATCH_HISTOGRAM_DECREASED', callback); 202 | }); 203 | } 204 | 205 | /** 206 | * Attach track metadata to a query match. 207 | */ 208 | function getTrackMetadata(match, allMatches, status, callback) { 209 | database.getTrack(match.track_id, function(err, track) { 210 | if (err) return callback(err, null); 211 | if (!track) 212 | return callback('Track ' + match.track_id + ' went missing', null); 213 | 214 | match.track = track.name; 215 | match.artist = track.artist_name; 216 | match.artist_id = track.artist_id; 217 | match.length = track.length; 218 | match.import_date = track.import_date; 219 | 220 | callback(null, { success: true, status: status, match: match }, 221 | allMatches); 222 | }); 223 | } 224 | 225 | /** 226 | * Build a mapping from each code in the given fingerprint to an array of time 227 | * offsets where that code appears, with the slop factor accounted for in the 228 | * time offsets. Used to speed up getActualScore() calculation. 229 | */ 230 | function getCodesToTimes(match, slop) { 231 | var codesToTimes = {}; 232 | 233 | for (var i = 0; i < match.codes.length; i++) { 234 | var code = match.codes[i]; 235 | var time = Math.floor(match.times[i] / slop) * slop; 236 | 237 | if (codesToTimes[code] === undefined) 238 | codesToTimes[code] = []; 239 | codesToTimes[code].push(time); 240 | } 241 | 242 | return codesToTimes; 243 | } 244 | 245 | /** 246 | * Computes the actual match score for a track by taking time offsets into 247 | * account. 248 | */ 249 | function getActualScore(fp, match, threshold, slop) { 250 | var MAX_DIST = 32767; 251 | 252 | if (match.codes.length < threshold) 253 | return 0; 254 | 255 | var timeDiffs = {}; 256 | var i, j; 257 | 258 | var matchCodesToTimes = getCodesToTimes(match, slop); 259 | 260 | // Iterate over each {code,time} tuple in the query 261 | for (i = 0; i < fp.codes.length; i++) { 262 | var code = fp.codes[i]; 263 | var time = Math.floor(fp.times[i] / slop) * slop; 264 | var minDist = MAX_DIST; 265 | 266 | var matchTimes = matchCodesToTimes[code]; 267 | if (matchTimes) { 268 | for (j = 0; j < matchTimes.length; j++) { 269 | var dist = Math.abs(time - matchTimes[j]); 270 | 271 | // Increment the histogram bucket for this distance 272 | if (timeDiffs[dist] === undefined) 273 | timeDiffs[dist] = 0; 274 | timeDiffs[dist]++; 275 | } 276 | } 277 | } 278 | 279 | match.histogram = timeDiffs; 280 | 281 | // Convert the histogram into an array, sort it, and sum the top two 282 | // frequencies to compute the adjusted score 283 | var keys = Object.keys(timeDiffs); 284 | var array = new Array(keys.length); 285 | for (i = 0; i < keys.length; i++) 286 | array[i] = [ keys[i], timeDiffs[keys[i]] ]; 287 | array.sort(function(a, b) { return b[1] - a[1]; }); 288 | 289 | if (array.length > 1) 290 | return array[0][1] + array[1][1]; 291 | else if (array.length === 1) 292 | return array[0][1]; 293 | return 0; 294 | } 295 | 296 | /** 297 | * Takes a track fingerprint (includes codes and time offsets plus any 298 | * available metadata), adds it to the database and returns a track_id, 299 | * artist_id, and artist name if available. 300 | */ 301 | function ingest(fp, callback) { 302 | var MAX_DURATION = 60 * 60 * 4; 303 | 304 | fp.codever = fp.codever || fp.version; 305 | 306 | log.info('Ingesting track "' + fp.track + '" by artist "' + fp.artist + 307 | '", ' + fp.length + ' seconds, ' + fp.codes.length + ' codes (' + fp.codever + ')'); 308 | 309 | if (!fp.codes.length) 310 | return callback('Missing "codes" array', null); 311 | if (typeof fp.length !== 'number') 312 | return callback('Missing or invalid "length" field', null); 313 | if (!fp.codever) 314 | return callback('Missing or invalid "version" field', null); 315 | if (!fp.track) 316 | return callback('Missing or invalid "track" field', null); 317 | if (!fp.artist) 318 | return callback('Missing or invalid "artist" field', null); 319 | 320 | fp = cutFPLength(fp, MAX_DURATION); 321 | 322 | // Acquire a lock while modifying the database 323 | gMutex.lock(function() { 324 | // Check if this track already exists in the database 325 | bestMatchForQuery(fp, config.code_threshold, function(err, res) { 326 | if (err) { 327 | gMutex.release(); 328 | return callback('Query failed: ' + err, null); 329 | } 330 | 331 | if (res.success) { 332 | var match = res.match; 333 | log.info('Found existing match with status ' + res.status + 334 | ', track ' + match.track_id + ' ("' + match.track + '") by "' + 335 | match.artist + '"'); 336 | 337 | var checkUpdateArtist = function() { 338 | if (!match.artist) { 339 | // Existing artist is unnamed but we have a name now. Check if this 340 | // artist name already exists in the database 341 | log.debug('Updating track artist'); 342 | database.getArtistByName(fp.artist, function(err, artist) { 343 | if (err) { gMutex.release(); return callback(err, null); } 344 | 345 | if (artist) { 346 | log.debug('Setting track artist_id to ' + artist.artist_id); 347 | 348 | // Update the track to point to the existing artist 349 | database.updateTrack(match.track_id, match.track, 350 | artist.artist_id, function(err) 351 | { 352 | if (err) { gMutex.release(); return callback(err, null); } 353 | match.artist_id = artist.artist_id; 354 | match.artist = artist.name; 355 | finished(match); 356 | }); 357 | } else { 358 | log.debug('Setting artist ' + artist.artist_id + ' name to "' + 359 | artist.name + '"'); 360 | 361 | // Update the artist name 362 | database.updateArtist(match.artist_id, fp.artist, 363 | function(err) 364 | { 365 | if (err) { gMutex.release(); return callback(err, null); } 366 | match.artist = fp.artist; 367 | finished(match); 368 | }); 369 | } 370 | }); 371 | } else { 372 | if (match.artist != fp.artist) { 373 | log.warn('New artist name "' + fp.artist + '" does not match ' + 374 | 'existing artist name "' + match.artist + '" for track ' + 375 | match.track_id); 376 | } 377 | log.debug('Skipping artist update'); 378 | finished(match); 379 | } 380 | }; 381 | 382 | var finished = function(match) { 383 | // Success 384 | log.info('Track update complete'); 385 | gMutex.release(); 386 | callback(null, { track_id: match.track_id, track: match.track, 387 | artist_id: match.artist_id, artist: match.artist }); 388 | }; 389 | 390 | if (!match.track && fp.track) { 391 | // Existing track is unnamed but we have a name now. Update the track 392 | log.debug('Updating track name to "' + fp.track + '"'); 393 | database.updateTrack(match.track_id, fp.track, match.artist_id, 394 | function(err) 395 | { 396 | if (err) { gMutex.release(); return callback(err, null); } 397 | match.track = fp.track; 398 | checkUpdateArtist(); 399 | }); 400 | } else { 401 | log.debug('Skipping track name update'); 402 | checkUpdateArtist(); 403 | } 404 | } else { 405 | // Track does not exist in the database yet 406 | log.debug('Track does not exist in the database yet, status ' + 407 | res.status); 408 | 409 | // Does this artist already exist in the database? 410 | database.getArtistByName(fp.artist, function(err, artist) { 411 | if (err) { gMutex.release(); return callback(err, null); } 412 | 413 | if (!artist) 414 | createArtistAndTrack(); 415 | else 416 | createTrack(artist.artist_id, artist.name); 417 | }); 418 | } 419 | 420 | // Function for creating a new artist and new track 421 | function createArtistAndTrack() { 422 | log.debug('Adding artist "' + fp.artist + '"') 423 | database.addArtist(fp.artist, function(err, artistID) { 424 | if (err) { gMutex.release(); return callback(err, null); } 425 | 426 | // Success 427 | log.info('Created artist ' + artistID + ' ("' + fp.artist + '")'); 428 | createTrack(artistID, fp.artist); 429 | }); 430 | } 431 | 432 | // Function for creating a new track given an artistID 433 | function createTrack(artistID, artist) { 434 | log.debug('Adding track "' + fp.track + '" for artist "' + artist + '" (' + artistID + ')'); 435 | database.addTrack(artistID, fp, function(err, trackID) { 436 | if (err) { gMutex.release(); return callback(err, null); } 437 | 438 | // Success 439 | log.info('Created track ' + trackID + ' ("' + fp.track + '")'); 440 | gMutex.release(); 441 | callback(null, { track_id: trackID, track: fp.track, 442 | artist_id: artistID, artist: artist }); 443 | }); 444 | } 445 | }); 446 | }); 447 | } 448 | -------------------------------------------------------------------------------- /docs/node-echoprint-debug01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jhurliman/node-echoprint-server/da695b6537b9a7d118b87d0b5be03a672b39e62c/docs/node-echoprint-debug01.png -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var http = require('http'); 2 | var urlParser = require('url'); 3 | var qs = require('querystring'); 4 | var log = require('winston'); 5 | var config = require('./config'); 6 | var server = require('./server'); 7 | 8 | // Make sure we have permission to bind to the requested port 9 | if (config.web_port < 1024 && process.getuid() !== 0) 10 | throw new Error('Binding to ports less than 1024 requires root privileges'); 11 | 12 | // Start listening for web requests 13 | server.init(); 14 | 15 | // If run_as_user is set, try to switch users 16 | if (config.run_as_user) { 17 | try { 18 | process.setuid(config.run_as_user); 19 | log.info('Changed to running as user ' + config.run_as_user); 20 | } catch (err) { 21 | log.error('Failed to change to user ' + config.run_as_user + ': ' + err); 22 | } 23 | } 24 | 25 | // Now that we've dropped root privileges (if requested), setup file logging 26 | // NOTE: Any messages logged before this will go to the console only 27 | if (config.log_path) 28 | log.add(log.transports.File, { level: config.log_level, filename: config.log_path }); 29 | -------------------------------------------------------------------------------- /logs/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jhurliman/node-echoprint-server/da695b6537b9a7d118b87d0b5be03a672b39e62c/logs/.keep -------------------------------------------------------------------------------- /models/mysql.js: -------------------------------------------------------------------------------- 1 | /** 2 | * MySQL database backend. An alternative database backend can be created 3 | * by implementing all of the methods exported by this module 4 | */ 5 | 6 | var fs = require('fs'); 7 | var mysql = require('mysql'); 8 | var temp = require('temp'); 9 | var log = require('winston'); 10 | var config = require('../config'); 11 | 12 | exports.fpQuery = fpQuery; 13 | exports.getTrack = getTrack; 14 | exports.getTrackByName = getTrackByName; 15 | exports.getArtist = getArtist; 16 | exports.getArtistByName = getArtistByName; 17 | exports.addTrack = addTrack; 18 | exports.addArtist = addArtist; 19 | exports.updateTrack = updateTrack; 20 | exports.updateArtist = updateArtist; 21 | exports.disconnect = disconnect; 22 | 23 | // Initialize the MySQL connection 24 | var client = mysql.createConnection({ 25 | user: config.db_user, 26 | password: config.db_pass, 27 | database: config.db_database, 28 | host: config.db_host 29 | }); 30 | client.connect(); 31 | 32 | /** 33 | * 34 | */ 35 | function fpQuery(fp, rows, callback) { 36 | var fpCodesStr = fp.codes.join(','); 37 | 38 | // Get the top N matching tracks sorted by score (number of matched codes) 39 | var sql = 'SELECT track_id,COUNT(track_id) AS score ' + 40 | 'FROM codes ' + 41 | 'WHERE code IN (' + fpCodesStr + ') ' + 42 | 'GROUP BY track_id ' + 43 | 'ORDER BY score DESC ' + 44 | 'LIMIT ' + rows; 45 | client.query(sql, [], function(err, matches) { 46 | if (err) return callback(err, null); 47 | if (!matches || !matches.length) return callback(null, []); 48 | 49 | var trackIDs = new Array(matches.length); 50 | var trackIDMap = {}; 51 | for (var i = 0; i < matches.length; i++) { 52 | var trackID = matches[i].track_id; 53 | trackIDs[i] = trackID; 54 | trackIDMap[trackID] = i; 55 | } 56 | var trackIDsStr = trackIDs.join('","'); 57 | 58 | // Get all of the matching codes and their offsets for the top N matching 59 | // tracks 60 | sql = 'SELECT code,time,track_id ' + 61 | 'FROM codes ' + 62 | 'WHERE code IN (' + fpCodesStr + ') ' + 63 | 'AND track_id IN ("' + trackIDsStr + '")'; 64 | client.query(sql, [], function(err, codeMatches) { 65 | if (err) return callback(err, null); 66 | 67 | for (var i = 0; i < codeMatches.length; i++) { 68 | var codeMatch = codeMatches[i]; 69 | var idx = trackIDMap[codeMatch.track_id]; 70 | if (idx === undefined) continue; 71 | 72 | var match = matches[idx]; 73 | if (!match.codes) { 74 | match.codes = []; 75 | match.times = []; 76 | } 77 | match.codes.push(codeMatch.code); 78 | match.times.push(codeMatch.time); 79 | } 80 | 81 | callback(null, matches); 82 | }); 83 | }); 84 | } 85 | 86 | function getTrack(trackID, callback) { 87 | var sql = 'SELECT tracks.*,artists.name AS artist_name ' + 88 | 'FROM tracks,artists ' + 89 | 'WHERE tracks.id=? ' + 90 | 'AND artists.id=artist_id'; 91 | client.query(sql, [trackID], function(err, tracks) { 92 | if (err) return callback(err, null); 93 | if (tracks.length === 1) 94 | return callback(null, tracks[0]); 95 | else 96 | return callback(null, null); 97 | }); 98 | } 99 | 100 | function getTrackByName(track, artistID, callback) { 101 | var sql = 'SELECT tracks.*,artists.name AS artist_name ' + 102 | 'FROM tracks,artists ' + 103 | 'WHERE tracks.name LIKE ? ' + 104 | 'AND artist_id=? ' + 105 | 'AND artists.id=artist_id'; 106 | client.query(sql, [track, artistID], function(err, tracks) { 107 | if (err) return callback(err, null); 108 | if (tracks.length > 0) 109 | return callback(null, tracks[0]); 110 | else 111 | return callback(null, null); 112 | }); 113 | } 114 | 115 | function getArtist(artistID, callback) { 116 | var sql = 'SELECT * FROM artists WHERE id=?'; 117 | client.query(sql, [artistID], function(err, artists) { 118 | if (err) return callback(err, null); 119 | if (artists.length === 1) { 120 | artists[0].artist_id = artists[0].id; 121 | return callback(null, artists[0]); 122 | } else { 123 | return callback(null, null); 124 | } 125 | }); 126 | } 127 | 128 | function getArtistByName(artistName, callback) { 129 | var sql = 'SELECT * FROM artists WHERE name LIKE ?'; 130 | client.query(sql, [artistName], function(err, artists) { 131 | if (err) return callback(err, null); 132 | if (artists.length > 0) { 133 | artists[0].artist_id = artists[0].id; 134 | return callback(null, artists[0]); 135 | } else { 136 | return callback(null, null); 137 | } 138 | }); 139 | } 140 | 141 | function addTrack(artistID, fp, callback) { 142 | var length = fp.length; 143 | if (typeof length === 'string') 144 | length = parseInt(length, 10); 145 | 146 | // Sanity checks 147 | if (!artistID) 148 | return callback('Attempted to add track with missing artistID', null); 149 | if (isNaN(length) || !length) 150 | return callback('Attempted to add track with invalid duration "' + length + '"', null); 151 | if (!fp.codever) 152 | return callback ('Attempted to add track with missing code version (codever field)', null); 153 | 154 | var sql = 'INSERT INTO tracks ' + 155 | '(codever,name,artist_id,length,import_date) ' + 156 | 'VALUES (?,?,?,?,NOW())'; 157 | client.query(sql, [fp.codever, fp.track, artistID, length], 158 | function(err, info) 159 | { 160 | if (err) return callback(err, null); 161 | if (info.affectedRows !== 1) return callback('Track insert failed', null); 162 | 163 | var trackID = info.insertId; 164 | var tempName = temp.path({ prefix: 'echoprint-' + trackID, suffix: '.csv' }); 165 | 166 | // Write out the codes to a file for bulk insertion into MySQL 167 | log.debug('Writing ' + fp.codes.length + ' codes to temporary file ' + tempName); 168 | writeCodesToFile(tempName, fp, trackID, function(err) { 169 | if (err) return callback(err, null); 170 | 171 | // Bulk insert the codes 172 | log.debug('Bulk inserting ' + fp.codes.length + ' codes for track ' + trackID); 173 | sql = 'LOAD DATA LOCAL INFILE ? IGNORE INTO TABLE codes'; 174 | client.query(sql, [tempName], function(err, info) { 175 | // Remove the temporary file 176 | log.debug('Removing temporary file ' + tempName); 177 | fs.unlink(tempName, function(err2) { 178 | if (!err) err = err2; 179 | callback(err, trackID); 180 | }); 181 | }); 182 | }); 183 | }); 184 | } 185 | 186 | function writeCodesToFile(filename, fp, trackID, callback) { 187 | var i = 0; 188 | var keepWriting = function() { 189 | var success = true; 190 | while (success && i < fp.codes.length) { 191 | success = file.write(fp.codes[i]+'\t'+fp.times[i]+'\t'+trackID+'\n'); 192 | i++; 193 | } 194 | if (i === fp.codes.length) 195 | file.end(); 196 | }; 197 | 198 | var file = fs.createWriteStream(filename); 199 | file.on('drain', keepWriting); 200 | file.on('error', callback); 201 | file.on('close', callback); 202 | 203 | keepWriting(); 204 | } 205 | 206 | function addArtist(name, callback) { 207 | var sql = 'INSERT INTO artists (name) VALUES (?)'; 208 | client.query(sql, [name], function(err, info) { 209 | if (err) return callback(err, null); 210 | callback(null, info.insertId); 211 | }); 212 | } 213 | 214 | function updateTrack(trackID, name, artistID, callback) { 215 | var sql = 'UPDATE tracks SET name=?, artist_id=? WHERE id=?'; 216 | client.query(sql, [name, artistID, trackID], function(err, info) { 217 | if (err) return callback(err, null); 218 | callback(null, info.affectedRows === 1 ? true : false); 219 | }); 220 | } 221 | 222 | function updateArtist(artistID, name, callback) { 223 | var sql = 'UPDATE artists SET name=? WHERE id=?'; 224 | client.query(sql, [name, artistID], function(err, info) { 225 | if (err) return callback(err, null); 226 | callback(null, info.affectedRows === 1 ? true : false); 227 | }); 228 | } 229 | 230 | function disconnect(callback) { 231 | client.end(callback); 232 | } 233 | -------------------------------------------------------------------------------- /mutex.js: -------------------------------------------------------------------------------- 1 | /** 2 | * A simple asynchronous mutex for node.js. 3 | * Based on code by 4 | */ 5 | 6 | var EventEmitter = require('events').EventEmitter; 7 | 8 | var Mutex = function() { 9 | var queue = new EventEmitter(); 10 | var locked = false; 11 | 12 | this.lock = function lock(fn) { 13 | if (locked) { 14 | queue.once('ready', function() { 15 | lock(fn); 16 | }); 17 | } else { 18 | locked = true; 19 | fn(); 20 | } 21 | }; 22 | 23 | this.release = function release() { 24 | locked = false; 25 | queue.emit('ready'); 26 | }; 27 | }; 28 | 29 | exports.getMutex = function() { 30 | var m = new Mutex(); 31 | return { 32 | lock: m.lock, 33 | release: m.release 34 | }; 35 | }; 36 | -------------------------------------------------------------------------------- /mysql.sql: -------------------------------------------------------------------------------- 1 | -- -------------------------------------------------------- 2 | -- 3 | -- Table structure for table `artists` 4 | -- 5 | 6 | CREATE TABLE IF NOT EXISTS `artists` ( 7 | `id` MEDIUMINT NOT NULL AUTO_INCREMENT, 8 | `name` varchar(255) DEFAULT NULL, 9 | PRIMARY KEY (`id`), 10 | UNIQUE KEY `name` (`name`) 11 | ) DEFAULT CHARSET=utf8; 12 | 13 | -- -------------------------------------------------------- 14 | -- 15 | -- Table structure for table `tracks` 16 | -- 17 | 18 | CREATE TABLE IF NOT EXISTS `tracks` ( 19 | `id` MEDIUMINT NOT NULL AUTO_INCREMENT, 20 | `codever` char(4) NOT NULL, 21 | `name` varchar(255) DEFAULT NULL, 22 | `artist_id` MEDIUMINT NOT NULL, 23 | `length` int(5) NOT NULL, 24 | `import_date` datetime NOT NULL, 25 | PRIMARY KEY (`id`,`codever`), 26 | FOREIGN KEY (`artist_id`) REFERENCES `artists`(`id`) ON DELETE CASCADE 27 | ) DEFAULT CHARSET=utf8; 28 | 29 | -- -------------------------------------------------------- 30 | -- 31 | -- Table structure for table `codes` 32 | -- 33 | 34 | CREATE TABLE IF NOT EXISTS `codes` ( 35 | `code` int(7) NOT NULL, 36 | `time` int(7) NOT NULL, 37 | `track_id` MEDIUMINT NOT NULL, 38 | PRIMARY KEY (`code`,`time`,`track_id`), 39 | FOREIGN KEY (`track_id`) REFERENCES `tracks`(`id`) ON DELETE CASCADE 40 | ) DEFAULT CHARSET=utf8; 41 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "echoprint-server", 3 | "version": "0.1.0", 4 | "dependencies": { 5 | "mysql": "2.0.0-alpha8", 6 | "winston": "0.6.2", 7 | "jade": "0.28.2", 8 | "async": "0.2.6", 9 | "temp": "0.5.0" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Simple HTTP server module 3 | */ 4 | 5 | var http = require('http'); 6 | var urlParser = require('url'); 7 | var qs = require('querystring'); 8 | var log = require('winston'); 9 | var jade = require('jade'); 10 | var config = require('./config'); 11 | var api = require('./controllers/api'); 12 | var debug = require('./controllers/debug'); 13 | 14 | exports.init = init; 15 | exports.renderView = renderView; 16 | exports.respond = respond; 17 | 18 | var TIMEOUT = 1000 * 60; 19 | 20 | /** 21 | * Initialize the HTTP endpoints. 22 | */ 23 | function init() { 24 | http.createServer(function(req, res) { 25 | req.start = new Date(); 26 | req.timer = setTimeout(function() { timeout(req, res); }, TIMEOUT); 27 | 28 | var url = urlParser.parse(req.url, true); 29 | var path = url.pathname.split('/', 16); 30 | 31 | if (req.method === 'GET') { 32 | if (path[1] === 'query') 33 | return api.query(req, res); 34 | else if (path[1] === 'debug') 35 | return debug.debugQuery(req, res); 36 | } else if (req.method === 'POST') { 37 | req.body = ''; 38 | req.on('data', function(data) { req.body += data; }); 39 | req.on('end', function() { 40 | req.body = qs.parse(req.body); 41 | 42 | if (path[1] === 'ingest') 43 | return api.ingest(req, res); 44 | else if (path[1] === 'debug') 45 | return debug.debugQuery(req, res); 46 | 47 | respond(req, res, 404, { error: 'Invalid API endpoint' }); 48 | }); 49 | return; 50 | } 51 | 52 | respond(req, res, 404, { error: 'Invalid API endpoint' }); 53 | }).addListener('clientError', function(ex) { 54 | log.warn('Client error: ' + ex); 55 | }).listen(config.web_port); 56 | 57 | log.info('HTTP listening on port ' + config.web_port); 58 | } 59 | 60 | /** 61 | * Render a view template and send it as an HTTP response. 62 | */ 63 | function renderView(req, res, statusCode, view, options, headers) { 64 | jade.renderFile(__dirname + '/views/' + view, options, function(err, html) { 65 | if (err) { 66 | log.error('Failed to render ' + view + ': ' + err); 67 | return respond(req, res, 500, 'Internal server error', headers); 68 | } 69 | 70 | respond(req, res, 200, html, headers); 71 | }); 72 | } 73 | 74 | /** 75 | * Send an HTTP response to a client. 76 | */ 77 | function respond(req, res, statusCode, body, headers) { 78 | // Destroy the response timeout timer 79 | clearTimeout(req.timer); 80 | 81 | statusCode = statusCode || 200; 82 | 83 | if (!headers) 84 | headers = {}; 85 | 86 | if (typeof body !== 'string') { 87 | body = JSON.stringify(body); 88 | headers['Content-Type'] = 'application/json'; 89 | } else { 90 | headers['Content-Type'] = 'text/html'; 91 | } 92 | 93 | var contentLength = body ? Buffer.byteLength(body, 'utf8') : '-'; 94 | if (body) 95 | headers['Content-Length'] = contentLength; 96 | 97 | var remoteAddress = 98 | (req.socket && (req.socket.remoteAddress || (req.socket.socket && req.socket.socket.remoteAddress))); 99 | 100 | var referrer = req.headers.referer || req.headers.referrer || ''; 101 | if (referrer.length > 128) 102 | referrer = referrer.substr(0, 128) + ' ...'; 103 | 104 | var url = req.url; 105 | if (url.length > 128) 106 | url = url.substr(0, 128) + ' ...'; 107 | 108 | log.info( 109 | remoteAddress + 110 | ' - - [' + (new Date()).toUTCString() + ']' + 111 | ' "' + req.method + ' ' + url + 112 | ' HTTP/' + req.httpVersionMajor + '.' + req.httpVersionMinor + '" ' + 113 | statusCode + ' ' + contentLength + 114 | ' "' + referrer + 115 | '" "' + (req.headers['user-agent'] || '') + '"'); 116 | 117 | try { 118 | res.writeHead(200, headers); 119 | res.end(body); 120 | } catch (ex) { 121 | log.error('Error sending response to ' + remoteAddress + ': ' + ex); 122 | } 123 | } 124 | 125 | /** 126 | * Handles server timeouts by logging an error and responding with a 503. 127 | */ 128 | function timeout(req, res) { 129 | var remoteAddress = 130 | (req.socket && (req.socket.remoteAddress || (req.socket.socket && req.socket.socket.remoteAddress))); 131 | log.error('Timed out while responding to a request from ' + remoteAddress); 132 | 133 | try { 134 | res.writeHead(503); 135 | res.end(); 136 | } catch (ex) { } 137 | } 138 | -------------------------------------------------------------------------------- /views/debug.jade: -------------------------------------------------------------------------------- 1 | !!! 5 2 | html(lang="en") 3 | head 4 | meta(charset='utf-8') 5 | title Echoprint Debug 6 | link(href='http://ajax.googleapis.com/ajax/libs/jqueryui/1.8/themes/base/jquery-ui.css', rel='stylesheet', type='text/css') 7 | style(type='text/css') 8 | .ui-progressbar { 9 | height: 15px; 10 | background: #ffffff; 11 | background: -moz-linear-gradient(top, #ffffff 0%, #f7f7f7 44%, #ededed 100%); 12 | background: -webkit-linear-gradient(top, #ffffff 0%,#f7f7f7 44%,#ededed 100%); 13 | background: -o-linear-gradient(top, #ffffff 0%,#f7f7f7 44%,#ededed 100%); 14 | background: -ms-linear-gradient(top, #ffffff 0%,#f7f7f7 44%,#ededed 100%); 15 | background: linear-gradient(top, #ffffff 0%,#f7f7f7 44%,#ededed 100%); 16 | } 17 | .ui-widget-header { 18 | background: #7abcff; 19 | background: -moz-linear-gradient(top, #7abcff 0%, #60abf8 44%, #4096ee 100%); 20 | background: -webkit-linear-gradient(top, #7abcff 0%,#60abf8 44%,#4096ee 100%); 21 | background: -o-linear-gradient(top, #7abcff 0%,#60abf8 44%,#4096ee 100%); 22 | background: -ms-linear-gradient(top, #7abcff 0%,#60abf8 44%,#4096ee 100%); 23 | background: linear-gradient(top, #7abcff 0%,#60abf8 44%,#4096ee 100%); 24 | } 25 | body 26 | h3 Echoprint Debug 27 | 28 | - if (typeof(err) !== 'undefined') 29 | h4 30 | font(color='red')= err 31 | 32 | form(action='/debug', method='POST') 33 | span Paste the JSON output from echoprint-codegen 34 | br 35 | textarea(name='json', rows=10, cols=80) 36 | - if (typeof(input) !== 'undefined') 37 | | !{input} 38 | br 39 | input(type='submit', name='Lookup', value='Lookup') 40 | input(type='submit', name='Ingest', value='Ingest') 41 | 42 | h2#status 43 | table#matches(border=1) 44 | 45 | script(src='http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js') 46 | script(src='http://ajax.googleapis.com/ajax/libs/jqueryui/1.8.16/jquery-ui.min.js') 47 | - if (typeof(res) !== 'undefined') 48 | script var res = !{res}; 49 | - else 50 | script var res = null; 51 | script 52 | if (res) { 53 | $('#status').text(res.status + ' - ' + res.queryTime + 'ms'); 54 | createMatchesTable(); 55 | } 56 | 57 | function createMatchesTable() { 58 | if (!res.success) return; 59 | 60 | for (var i = 0; i < res.matches.length; i++) { 61 | var match = res.matches[i]; 62 | var trackTD = $(document.createElement('td')); 63 | 64 | var artistName = match.artist || '[Unnamed Artist]'; 65 | var trackName = match.track || '[Unnamed Track]'; 66 | 67 | trackTD.html(match.track_id + '
"' + artistName + ' - ' + trackName + '"'); 68 | 69 | console.log(match.contributors.length + ' / ' + res.queryLen); 70 | var pctMatched = match.contributors.length / res.queryLen; 71 | 72 | var graphsTD = $(document.createElement('td')); 73 | $("
").progressbar({ value: pctMatched * 100 }).appendTo(graphsTD); 74 | $('', { 75 | name: match.track_id + '-lines', 76 | css: { 77 | margin: '0', 78 | background: '#ffffff', 79 | border: '1px solid black', 80 | width: (Math.min(match.length, 600) * 1.3) + 'px', 81 | height: '25px' 82 | } 83 | }).appendTo(graphsTD); 84 | 85 | var tr = $(document.createElement('tr')); 86 | tr.append(trackTD); 87 | tr.append(graphsTD); 88 | 89 | $('#matches').append(tr); 90 | 91 | drawCodeMatchGraph(match); 92 | } 93 | } 94 | 95 | function drawCodeMatchGraph(match, graphsTD) { 96 | var canvas = document.getElementsByName(match.track_id + '-lines')[0]; 97 | var ctx = canvas.getContext('2d'); 98 | 99 | // Find the most frequently occurring time offset in the histogram 100 | var topOffset = 0; 101 | var topOffsetCount = 0; 102 | for (var offset in match.histogram) { 103 | if (match.histogram.hasOwnProperty(offset)) { 104 | if (match.histogram[offset] > topOffsetCount) { 105 | topOffsetCount = match.histogram[offset]; 106 | topOffset = parseInt(offset, 10); 107 | } 108 | } 109 | } 110 | 111 | for (var i = 0; i < match.contributors.length; i++) { 112 | var contrib = match.contributors[i]; 113 | var color = (contrib.dist == topOffset) ? 'blue' : 'red'; 114 | drawCodeMatch(ctx, canvas, contrib.time, match.codeLength, color); 115 | } 116 | 117 | ctx.save(); 118 | } 119 | 120 | function drawCodeMatch(ctx, canvas, offset, duration, color) { 121 | // Find the canvas offset 122 | var x = (offset / duration) * canvas.width; 123 | 124 | ctx.strokeStyle = color; 125 | ctx.beginPath(); 126 | ctx.moveTo(x, 0); 127 | ctx.lineTo(x, canvas.height) 128 | ctx.stroke(); 129 | } 130 | --------------------------------------------------------------------------------