├── .gitmodules ├── LICENSE ├── README ├── diffable.js ├── fileResourceManager.js ├── requestHandler.js └── resources ├── DJSBootstrap.js ├── DJSBootstrap.min.js ├── DeltaBootstrap.js ├── DeltaBootstrap.min.js ├── JsDictionaryBootstrap.js └── JsDictionaryBootstrap.min.js /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vcdiff.js"] 2 | path = vcdiff.js 3 | url = git://github.com/plotnikoff/vcdiff.js.git 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2010 Konstantin Plotnikov. All rights reserved. 2 | Permission is hereby granted, free of charge, to any person obtaining a copy 3 | of this software and associated documentation files (the "Software"), to 4 | deal in the Software without restriction, including without limitation the 5 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 6 | sell copies of the Software, and to permit persons to whom the Software is 7 | furnished to do so, subject to the following conditions: 8 | 9 | The above copyright notice and this permission notice shall be included in 10 | all copies or substantial portions of the Software. 11 | 12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 13 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 14 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 15 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 16 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 17 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 18 | IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Diffable. 2 | "Diffable is a method for reducing page load latency by transmitting 3 | differential encodings of static files. It works by sending deltas between 4 | versions of files that reside in a browser's cache. In web applications, files 5 | such as Javascript, HTML, and CSS are often updated every week or more, but 6 | change very little between versions. This method significantly reduces the 7 | bandwidth and transmission time for these files and is implementable on all 8 | legacy browsers that support Javascript." 9 | How it works and reference Java implementation: http://code.google.com/p/diffable/ 10 | 11 | Connect-diffable. 12 | Middleware for Sencha's Connect (http://github.com/senchalabs/connect/) stack 13 | that implements algorithm described in previous section. 14 | 15 | Installation. 16 | git clone git://github.com/plotnikoff/connect-diffable.git 17 | cd connect-diffable 18 | git submodule init 19 | git submodule update 20 | 21 | Usage. 22 | require('../connect-diffable/diffable') - Module exports constructor for 23 | Diffable. 24 | 25 | Constuctor accepts configuration object: 26 | { 27 | 'diffableDir' : './diffable', 28 | 'resourceDir' : './static' 29 | } 30 | where 'diffableDir' points to directory where versions and deltas will be kept 31 | and 'resourceDir' is directory where static files reside. 32 | 33 | serve(fileName...) - method returns middleware function for Connect stack and 34 | adds files passed as parameters to diffable control, so when you change static 35 | file changes are tracked by middleware and appropriate delta and version files 36 | are created. Path is relative to 'resourceDir' 37 | 38 | Also it is possible to generate docs by running jsdoc-toolkit on diffable.js 39 | 40 | Example. 41 | Say we'd like to serve file app.js with diffable. 42 | 43 | Simple server that serves static files. 44 | 45 | server.js 46 | var Connect = require('connect'), 47 | diffable = require('../connect-diffable/diffable'), 48 | diff = new diffable({ 49 | 'diffableDir' : './diffable', 50 | 'resourceDir' : './static' 51 | }); 52 | 53 | module.exports = Connect.createServer( 54 | diff.serve('/js/app.js'), 55 | Connect.staticProvider('./static') 56 | ); 57 | 58 | 59 | Also we need to change index.html in which we use this secondary resource: 60 | 61 | index.html 62 | 63 | 64 | 65 | 66 | New Web Project 67 | 68 | 69 |

New Web Project Page

70 | {{DIFFABLE res="/js/app.js"}} 71 | 72 | 73 | 74 | Pattern {{DIFFABLE res="/js/app.js"}} will be replaced with javascript that 75 | will load appropriate version file, delta if necessary and will eval the source. 76 | Html file can contain references to several diffable controled files. 77 | 78 | -------------------------------------------------------------------------------- /diffable.js: -------------------------------------------------------------------------------- 1 | /*global module, require */ 2 | 3 | var fs = require('fs'), 4 | crypto = require('crypto'), 5 | requestHandler = require('./requestHandler'), 6 | FileResourceManager = require('./fileResourceManager'); 7 | 8 | module.exports = (function () { 9 | var that = null, 10 | frm = new FileResourceManager(), 11 | 12 | /** 13 | * Instantiates diffable object 14 | * @class 15 | * @constructs 16 | * @param {Object} config configuration object 17 | * 18 | * 
 19 |      *     {
 20 |      *         "diffableDir" : '/path/to/directory'
 21 |      *         "resourceDir" : '/path/to/directory'
 22 |      *     }
 23 |      *
24 | * 25 | * diffableDir - directory where delta and version files will 26 | * be stored
27 | * resourceDir - directory where diffable can find static files. 28 | */ 29 | diffable = function (config) { 30 | this.dir = fs.realpathSync(config.diffableDir); 31 | this.resourceDir = config.resourceDir; 32 | this.provider = requestHandler({ 33 | 'resourceDir': config.resourceDir, 34 | 'frm': frm, 35 | 'diffableRoot': this.dir, 36 | 'logger' : config.logger 37 | }); 38 | that = this; 39 | }; 40 | 41 | /** 42 | * Method adds resources that should be served with diffable. 43 | * @private 44 | * @param {String} path Path to resource that should be served by diffable. 45 | * Path is relative to resourceDir. 46 | */ 47 | diffable.prototype.watch = function (path) { 48 | //resolving absolute path to resource being tracked 49 | fs.realpath(this.resourceDir + path, function (err, resolvedPath) { 50 | if (err) { 51 | throw err; 52 | } 53 | 54 | var hash = crypto.createHash('md5').update(resolvedPath), 55 | resourceDir = that.dir + '/' + hash.digest('hex'); 56 | 57 | //Create resource directory (if doesn't exist) name of directory is 58 | // md5 hash of resource's absolute path 59 | fs.mkdir(resourceDir, 0755, function (err) { 60 | 61 | //create first version of resource 62 | frm.putResource(resolvedPath, resourceDir); 63 | 64 | //add callback to track file changes 65 | fs.watchFile(resolvedPath, function (curr, prev) { 66 | 67 | //if changes were made add new version of resource 68 | frm.putResource(resolvedPath, resourceDir); 69 | }); 70 | }); 71 | }); 72 | }; 73 | 74 | /** 75 | * Method adds files to diffable control, and returns Connect middleware. 76 | * Returns connect stack interface, if request contains data that is 77 | * relevant to diffable this middleware will serve appropriate version 78 | * and/or delta files. 79 | * @public 80 | * @param {String} filename... varargs names of the files to look for. Paths 81 | * are relative to resourceDir in configuration object 82 | * @returns {Function} 83 | */ 84 | diffable.prototype.serve = function () { 85 | var i = 0, len = arguments.length 86 | if (len >= 1) { 87 | for (; i < len; i += 1) { 88 | that.watch(arguments[i]); 89 | } 90 | return that.provider; 91 | } else { 92 | console.log('Diffable: There are no files under control') 93 | return function (req, res, next) { 94 | next(); 95 | } 96 | } 97 | } 98 | 99 | return diffable; 100 | 101 | }()); 102 | -------------------------------------------------------------------------------- /fileResourceManager.js: -------------------------------------------------------------------------------- 1 | /*global require, module*/ 2 | 3 | var fs = require('fs'), 4 | crypto = require('crypto'), 5 | vcdiff = require('./vcdiff.js/vcdiff'); 6 | 7 | /** 8 | * Class keeps track of resources and thier versions/deltas. 9 | */ 10 | var FileResourceManager = function () { 11 | this.resources = {}; 12 | this.versions = {}; 13 | this.currentVersion = null; 14 | }; 15 | 16 | FileResourceManager.prototype = { 17 | 18 | /** 19 | * Method returns resource path hash if resource is tracked by 20 | * FileResourceManager 21 | * @param {String} path 22 | */ 23 | getResourceHash : function (path) { 24 | return this.resources[path]; 25 | }, 26 | 27 | /** 28 | * Method returns most recent version hash for given resource. 29 | * @param {String} resourceHash 30 | */ 31 | getVersionHash : function (resourceHash) { 32 | return this.versions[resourceHash]; 33 | }, 34 | 35 | /** 36 | * Method is responsible for version and delta files createion. 37 | * @param {String} resolvedPath 38 | * @param {String} resourceDir 39 | */ 40 | putResource : function (resolvedPath, resourceDir) { 41 | var that = this; 42 | 43 | //read resource file 44 | fs.readFile(resolvedPath, function (err, data) { 45 | if (err) { 46 | throw err; 47 | } 48 | 49 | //generate md5 hash from resource's data 50 | var hash = crypto.createHash('md5').update(data), 51 | resourceName = hash.digest('hex') + '.version', 52 | resourceHash = resourceDir.split('/').reverse()[0]; 53 | 54 | that.resources[resolvedPath] = resourceHash; 55 | 56 | //create version file 57 | fs.writeFile(resourceDir + '/' + resourceName, data, function (err) { 58 | if (err) { 59 | throw err; 60 | } 61 | that.currentVersion = resourceName; 62 | that.versions[resourceHash] = resourceName.split('.')[0]; 63 | 64 | //read resource directory to fetch older versions of resource 65 | fs.readdir(resourceDir, function (err, files) { 66 | if (err) { 67 | throw err; 68 | } 69 | var i, len = files.length, vcd = new vcdiff.Vcdiff(); 70 | 71 | //loop over older versions to create diffs 72 | for (i = 0;i < len; i += 1) { 73 | 74 | //check that this is not current version and isn't a diff file 75 | if (files[i] !== that.currentVersion && 76 | files[i].split('.')[1] !== 'diff') { 77 | (function (fileName) { 78 | //read file with older version and generate diff file 79 | fs.readFile(resourceDir + '/' + fileName, 'utf8', 80 | function (err, dictData) { 81 | var diff = vcd.encode(dictData.toString(), 82 | data.toString()), 83 | diffName = fileName.split('.')[0] + '_' + 84 | that.currentVersion.split('.')[0] + '.diff'; 85 | fs.writeFile(resourceDir + '/' + diffName, 86 | JSON.stringify(diff), function (err) { 87 | if (err) { 88 | throw err; 89 | } 90 | } 91 | ); 92 | } 93 | ); 94 | }(files[i])); 95 | } 96 | } 97 | }); 98 | }); 99 | }); 100 | } 101 | 102 | }; 103 | 104 | module.exports = FileResourceManager; -------------------------------------------------------------------------------- /requestHandler.js: -------------------------------------------------------------------------------- 1 | /*jslint nomen:false, regexp:false*/ 2 | 3 | /*global module, require, __dirname, process*/ 4 | 5 | var fs = require('fs'), 6 | Path = require('path'), 7 | parseUrl = require('url').parse, 8 | queryString = require('querystring'), that; 9 | /** 10 | * Function sends response with aggresive caching 11 | * @param {Object} res 12 | * @param {Object} string 13 | */ 14 | function sendForCaching(res, str) { 15 | var headers = { 16 | "Content-Type": 'text/javascript', 17 | "Content-Length": str.length, 18 | "Cache-Control": "public, max-age=63072000", 19 | "Last-Modified": new Date(2000, 1, 1).toUTCString(), 20 | "Expires": new Date().toUTCString() 21 | }; 22 | 23 | res.writeHead(200, headers); 24 | res.end(str); 25 | } 26 | 27 | function sendConditional(res) { 28 | res.writeHead(304, { 29 | "Content-Type": 'text/javascript', 30 | "Last-Modified": new Date(2000, 1, 1).toUTCString(), 31 | "Expires": new Date().toUTCString(), 32 | "Cache-Control": "public, max-age=63072000" 33 | }); 34 | res.end(); 35 | } 36 | 37 | /** 38 | * @param {Object} options 39 | * @cfg {String} resourceDir 40 | * @cfg {FileResourceManager} frm 41 | * @cfg {String} diffableRoot 42 | */ 43 | module.exports = function requestHandler(config) { 44 | var root = config.resourceDir, 45 | frm = config.frm, 46 | diffableRoot = config.diffableRoot, 47 | log = config.logger ? config.logger : function () {}, 48 | suffix = process.env.NODE_ENV === 'production' ? '.min' : '', 49 | bootScript = fs.readFileSync(__dirname + 50 | '/resources/DJSBootstrap' + suffix + '.js', 'utf8'), 51 | deltaScript = fs.readFileSync(__dirname + 52 | '/resources/DeltaBootstrap' + suffix + '.js', 'utf8'), 53 | versionScript = fs.readFileSync(__dirname + 54 | '/resources/JsDictionaryBootstrap' + suffix + '.js', 'utf8'); 55 | 56 | return function (req, res, next) { 57 | if (req.method !== 'GET' && req.method !== 'HEAD') { 58 | return next(); 59 | } 60 | 61 | var filename, url = parseUrl(req.url), hashes, resHash, dictVerHash, 62 | targetVerHash; 63 | 64 | //Delta request handler 65 | function onDiffRead(err, diffData) { 66 | if (err) { 67 | return next(); 68 | } 69 | var script = deltaScript; 70 | script = script.replace('{{DJS_RESOURCE_IDENTIFIER}}', 71 | resHash); 72 | script = script.replace('{{DJS_DIFF_CONTENT}}', diffData.toString()); 73 | log({ 74 | 'type' : 'delta', 75 | 'resource' : resHash, 76 | 'deltaFile' : dictVerHash + '_' + targetVerHash + '.diff' 77 | }); 78 | sendForCaching(res, script); 79 | } 80 | 81 | //Version request handler 82 | function onJsRead(err, versionData) { 83 | if (err) { 84 | return next(); 85 | } 86 | var script = versionScript; 87 | script = script.replace('{{DJS_RESOURCE_IDENTIFIER}}', resHash); 88 | script = script.replace('{{DJS_CODE}}', 89 | JSON.stringify(versionData.toString())); 90 | script = script.replace('{{DJS_BOOTSTRAP_VERSION}}', dictVerHash); 91 | script = script.replace('{{DJS_DIFF_URL}}', 92 | '/diffable/' + resHash + '/'); 93 | log({ 94 | 'type' : 'version', 95 | 'resource' : resHash, 96 | 'versionFile' : dictVerHash + '.version' 97 | }); 98 | sendForCaching(res, script); 99 | } 100 | 101 | //HTML request handler 102 | function onHtmlRead(err, data) { 103 | if (err) { 104 | return next(err); 105 | } 106 | var strData = data.toString(), 107 | matches = strData.match(/\{\{DIFFABLE(.)*\}\}/gi), 108 | i, len, resource, script = "", counter = 0; 109 | 110 | //file doesn't contain diffable template. Pass through to 111 | //next middleware 112 | if (matches === null) { 113 | return next(); 114 | } 115 | 116 | for (i = 0, len = matches.length; i < len; i += 1) { 117 | resource = matches[i].split('"')[1]; 118 | (function (counter) { 119 | fs.realpath(root + resource, function (err, resolvedPath) { 120 | var resHash = frm.getResourceHash(resolvedPath), 121 | verHash = frm.getVersionHash(resHash), headers; 122 | 123 | if (counter === 0) { 124 | script += ""; 125 | } 126 | 127 | script += '"; 134 | strData = strData.replace(matches[counter], script); 135 | script = ''; 136 | counter += 1; 137 | if (counter === len) { 138 | headers = { 139 | "Content-Type": 'text/html', 140 | "Content-Length": strData.length, 141 | "Cache-Control": "private, max-age=0", 142 | "Expires": "-1" 143 | }; 144 | log({ 145 | 'type' : 'html', 146 | 'resource' : resHash 147 | }); 148 | res.writeHead(200, headers); 149 | res.end(strData); 150 | } 151 | 152 | }); 153 | }(i)); 154 | } 155 | } 156 | 157 | filename = Path.join(root, queryString.unescape(url.pathname)); 158 | 159 | if (filename[filename.length - 1] === '/') { 160 | filename += "index.html"; 161 | } 162 | 163 | if (filename.match(/\/diffable\/(.)*\.diff/gi)) { 164 | //request for delta data 165 | if (!req.headers['if-modified-since']) { 166 | hashes = filename.split('/').reverse()[0].split('_'); 167 | resHash = hashes[0]; 168 | dictVerHash = hashes[1]; 169 | targetVerHash = hashes[2].split('.')[0]; 170 | fs.readFile(diffableRoot + '/' + resHash + '/' + 171 | dictVerHash + '_' + targetVerHash + '.diff', onDiffRead); 172 | } else { 173 | sendConditional(res); 174 | } 175 | } else if (filename.match(/\/diffable\/(.)/gi)) { 176 | //request for versioned file 177 | if (!req.headers['if-modified-since']) { 178 | resHash = filename.split('/').reverse()[0]; 179 | dictVerHash = frm.getVersionHash(resHash); 180 | if (dictVerHash) { 181 | fs.readFile(diffableRoot + '/' + resHash + 182 | '/' + 183 | dictVerHash + 184 | '.version', onJsRead); 185 | } 186 | else { 187 | return next(); 188 | } 189 | } else { 190 | sendConditional(res); 191 | } 192 | } else if (filename.match(/(.)*\.html/gi)) { 193 | //request for html page 194 | fs.readFile(filename, onHtmlRead); 195 | } else { 196 | //pass through to next middleware 197 | return next(); 198 | } 199 | 200 | }; 201 | }; 202 | -------------------------------------------------------------------------------- /resources/DJSBootstrap.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2010 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | function DJSBootstrap(identifier, code) { 18 | this.code_ = code; 19 | this.identifier_ = identifier; 20 | if (window.localStorage) { 21 | if (!localStorage["diffable"]) { 22 | localStorage["diffable"] = JSON.stringify({}); 23 | } 24 | this.ls_ = JSON.parse(localStorage["diffable"]); 25 | } 26 | } 27 | 28 | DJSBootstrap.prototype.bootstrap = function(bootstrap_version, 29 | diff_url) { 30 | if (window['deltajs'][this.identifier_]['cv'] == bootstrap_version) { 31 | this.applyAndExecute(); 32 | } else { 33 | var me = this; 34 | window['deltajs'][this.identifier_]['load'] = function() { 35 | me.applyAndExecute.apply(me, arguments); 36 | }; 37 | var diffScript = document.createElement('script'); 38 | diffScript.src = diff_url + this.identifier_ + "_" + 39 | bootstrap_version + "_" + 40 | window['deltajs'][this.identifier_]['cv'] + ".diff"; 41 | document.getElementsByTagName("head")[0].appendChild(diffScript); 42 | } 43 | }; 44 | 45 | DJSBootstrap.prototype.applyAndExecute = function(opt_delta) { 46 | var output = this.code_; 47 | if (opt_delta) { 48 | output = DJSBootstrap.apply_(this.code_, opt_delta); 49 | } 50 | if (this.ls_) { 51 | this.ls_[this.identifier_] = { 52 | 'v': window['deltajs'][this.identifier_]['cv'], 53 | 'c': output 54 | } 55 | localStorage['diffable'] = JSON.stringify(this.ls_); 56 | } 57 | DJSBootstrap.globalEval(output); 58 | }; 59 | 60 | DJSBootstrap.apply_ = function(dict, diff) { 61 | var output = []; 62 | for (var i = 0; i < diff.length; i++) { 63 | if (typeof diff[i] == 'number') { 64 | output.push( 65 | dict.substring(diff[i], diff[i] + diff[i + 1])); 66 | ++i; 67 | } else if (typeof diff[i] == 'string') { 68 | output.push(diff[i]); 69 | } 70 | } 71 | return output.join(''); 72 | }; 73 | 74 | DJSBootstrap.globalEval = (function() { 75 | var isIndirectEvalGlobal = (function (original, Object) { 76 | try { 77 | return (1, eval)('Object') === original; 78 | } catch(err) { 79 | return false; 80 | } 81 | }(Object, 123)); 82 | 83 | if (isIndirectEvalGlobal) { 84 | return function(expression) { 85 | return (1,eval)(expression); 86 | }; 87 | } else if (typeof window.execScript !== 'undefined') { 88 | return function(expression) { 89 | return window.execScript(expression); 90 | }; 91 | } 92 | }()); 93 | 94 | DJSBootstrap.loadVersion = function (identifier) { 95 | var script = document.createElement('script'); 96 | script.src = '/diffable/' + identifier; 97 | document.getElementsByTagName('head')[0].appendChild(script); 98 | } 99 | 100 | DJSBootstrap.checkStorage = function (resHash, verHash) { 101 | if (!window.localStorage || !window.localStorage['diffable']) { 102 | DJSBootstrap.loadVersion(resHash); 103 | } else { 104 | var ls = JSON.parse(localStorage['diffable']); 105 | if (ls[resHash] && ls[resHash]['v'] === verHash) { 106 | DJSBootstrap.globalEval(ls[resHash]['c']); 107 | } else { 108 | DJSBootstrap.loadVersion(resHash); 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /resources/DJSBootstrap.min.js: -------------------------------------------------------------------------------- 1 | function DJSBootstrap(a,b){this.code_=b;this.identifier_=a;if(window.localStorage){if(!localStorage.diffable){localStorage.diffable=JSON.stringify({})}this.ls_=JSON.parse(localStorage.diffable)}}DJSBootstrap.prototype.bootstrap=function(c,a){if(window.deltajs[this.identifier_]["cv"]==c){this.applyAndExecute()}else{var b=this;window.deltajs[this.identifier_]["load"]=function(){b.applyAndExecute.apply(b,arguments)};var d=document.createElement("script");d.src=a+this.identifier_+"_"+c+"_"+window.deltajs[this.identifier_]["cv"]+".diff";document.getElementsByTagName("head")[0].appendChild(d)}};DJSBootstrap.prototype.applyAndExecute=function(b){var a=this.code_;if(b){a=DJSBootstrap.apply_(this.code_,b)}if(this.ls_){this.ls_[this.identifier_]={v:window.deltajs[this.identifier_]["cv"],c:a};localStorage.diffable=JSON.stringify(this.ls_)}DJSBootstrap.globalEval(a)};DJSBootstrap.apply_=function(d,c){var a=[];for(var b=0;b