├── sync ├── client │ ├── a │ │ └── a-synced.txt │ └── synced.txt └── server │ ├── a │ └── a-synced.txt │ └── synced.txt ├── .gitignore ├── .npmignore ├── index.js ├── History.md ├── test ├── vanilla-usage.js ├── performance.js ├── server.js ├── rename.js ├── rsync.js ├── helper.js └── tests.js ├── package.json ├── credentials ├── certrequest.csr ├── certificate.pem └── privatekey.pem ├── Readme.md └── lib ├── hash.js ├── parser.js ├── anchor.js ├── client.js └── node-rsync.js /sync/client/a/a-synced.txt: -------------------------------------------------------------------------------- 1 | sdfas -------------------------------------------------------------------------------- /sync/client/synced.txt: -------------------------------------------------------------------------------- 1 | abcdefgh -------------------------------------------------------------------------------- /sync/server/a/a-synced.txt: -------------------------------------------------------------------------------- 1 | asfasdf -------------------------------------------------------------------------------- /sync/server/synced.txt: -------------------------------------------------------------------------------- 1 | abcdefgh -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | *.sock -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | support 2 | test 3 | examples 4 | *.sock 5 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = require('./lib/node-rsync'); -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- 1 | 2 | 0.0.1 / 2010-01-03 3 | ================== 4 | 5 | * Initial release 6 | -------------------------------------------------------------------------------- /test/vanilla-usage.js: -------------------------------------------------------------------------------- 1 | var Anchor = require('../lib/anchor') 2 | 3 | var clientAnchor = new Anchor({ 4 | host: '127.0.0.1' 5 | , localPort: 8080 6 | , remotePort: 8081 7 | , roots: { 8 | '../sync/client': '/home/ttezel/development/anchor/sync/server' 9 | } 10 | }) 11 | 12 | var serverAnchor = new Anchor({ 13 | host: '127.0.0.1' 14 | , localPort: 8081 15 | , remotePort: 8080 16 | , roots: { 17 | '../sync/server': '/home/ttezel/development/anchor/sync/client' 18 | } 19 | }) -------------------------------------------------------------------------------- /test/performance.js: -------------------------------------------------------------------------------- 1 | var rsync = require('../lib/node-rsync.js'); 2 | 3 | var sync = rsync.createRSync('./performance', 750); 4 | 5 | var t1 = Date.now(); 6 | sync.checksum('/libruby190.a', function (err, checksums) { 7 | var t2 = Date.now(); 8 | sync.diff('/libruby191.a', checksums, function (err, diffs) { 9 | var t3 = Date.now(); 10 | console.log('checksums - time: %d, num: %d', t2 - t1, checksums.length); 11 | console.log('diff - time: %d, num: %d', t3 - t2, diffs.length); 12 | }); 13 | }); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-rsync" 3 | , "version": "0.0.1" 4 | , "description": "rsync implemented in node over http" 5 | , "keywords": [] 6 | , "author": "Tolga Tezel " 7 | , "contributors": ["Mihai Tomescu "] 8 | , "dependencies": { 9 | "formidable": ">=1.0.6" 10 | , "restify": ">=0.5.4" 11 | , "node-static": ">=0.5.9" 12 | , "findit": ">=0.1.2" 13 | } 14 | , "devDependencies": { 15 | "vows": ">=0.5.13" 16 | , "colors": ">=0.5.1" 17 | } 18 | , "main": "index" 19 | , "engines": { "node": "0.5 || 0.6" } 20 | } -------------------------------------------------------------------------------- /credentials/certrequest.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIIBdzCB4QIBADA4MRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl 3 | cm5ldCBXaWRnaXRzIFB0eSBMdGQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGB 4 | ANodRf3hWLkyFOU9gqsX4WScNkTji7nsYme4aO+QEYF2NERpjn9ecWV7i57Mjade 5 | Qz+32B/IkidVihP7YSd9HDqUMIThd0uwDtQrGJzz3UKTcUgArXIABx1Ca7BDXfKc 6 | x8I4TX80EJiFXslqvG5u1CGStT+0hVh17yKX0tB/lXexAgMBAAGgADANBgkqhkiG 7 | 9w0BAQUFAAOBgQDB9Vlrkq/EbpcSTyptxXJyMlVe4+JSIxqqBU3jFolC7kDzDVPW 8 | 6wbKuXO0jaqbOikd+0z+PuCUJ+VJiDtXSv2dEgjhEHOO1KtDutymjHNk1E4aGXx+ 9 | 4eaWNGqGtUJXkK9Ys6j2Gy6hGzXE1vJ5i9cUOuuWjDlkok2AzbfBxAobAQ== 10 | -----END CERTIFICATE REQUEST----- 11 | -------------------------------------------------------------------------------- /test/server.js: -------------------------------------------------------------------------------- 1 | var server = require('../lib/server.js').createServer().listen(8000); 2 | var client1 = require('../lib/client.js').createClient(8000); 3 | var client2 = require('../lib/client.js').createClient(8000); 4 | 5 | server.checksum(function (client, data) { 6 | var path = data[1].toString(); 7 | }); 8 | 9 | server.sync(function (buffers) { 10 | console.log('SERVER: sync request'); 11 | }); 12 | 13 | server.error(function (buffers) { 14 | console.log('SERVER: error'); 15 | }) 16 | 17 | client1.on('ready', function () { 18 | client1.checksum('/client.txt', function (result) { 19 | console.log('CLIENT: received checksum: %s', result); 20 | }); 21 | }); 22 | 23 | client1.close(); 24 | client2.close(); -------------------------------------------------------------------------------- /credentials/certificate.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIB5zCCAVACCQC/bInZ4isV+DANBgkqhkiG9w0BAQUFADA4MRMwEQYDVQQIDApT 3 | b21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcN 4 | MTIwMzA1MDMxOTQ1WhcNMTIwNDA0MDMxOTQ1WjA4MRMwEQYDVQQIDApTb21lLVN0 5 | YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwgZ8wDQYJKoZI 6 | hvcNAQEBBQADgY0AMIGJAoGBANodRf3hWLkyFOU9gqsX4WScNkTji7nsYme4aO+Q 7 | EYF2NERpjn9ecWV7i57MjadeQz+32B/IkidVihP7YSd9HDqUMIThd0uwDtQrGJzz 8 | 3UKTcUgArXIABx1Ca7BDXfKcx8I4TX80EJiFXslqvG5u1CGStT+0hVh17yKX0tB/ 9 | lXexAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEA0N+gYJNO3bEpax6AP+L5h1BoUOf2 10 | p5Pcn+0TXzHY2B3+0i1OmbsoIR+xyu/o1jNq3qMm20KbjUecQ5yoQYQJqJKpMavk 11 | 5RqIdZtCd7gdYIUAdSXAbCekVXhTYdUAWc0Mg1/hthR1w8xutxoo60ppZ2qYzreA 12 | 0jGRsGolBvpx68s= 13 | -----END CERTIFICATE----- 14 | -------------------------------------------------------------------------------- /credentials/privatekey.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIICXQIBAAKBgQDaHUX94Vi5MhTlPYKrF+FknDZE44u57GJnuGjvkBGBdjREaY5/ 3 | XnFle4uezI2nXkM/t9gfyJInVYoT+2EnfRw6lDCE4XdLsA7UKxic891Ck3FIAK1y 4 | AAcdQmuwQ13ynMfCOE1/NBCYhV7JarxubtQhkrU/tIVYde8il9LQf5V3sQIDAQAB 5 | AoGBAIPPTv3nEcwRROloK7AjpVU8xdsJu+XmwW0210t2v+2BtoJlW/UC6PpQGAcm 6 | TLCJWZCHFHfYqOJWYjRDJOpNRCHQnXw8vjTV6Ykz6FL9J/+u+fjJc9siiS1NQ0L3 7 | obLyy5qzXQktVuhpNF/88nNY21tTyMmqJcwJVkSzrb/YNx6xAkEA9ZdeJztY9ytZ 8 | DOx3LrUGVeMjgu8NmLI1hn8QnYoiiW6xqqH8rKCdCeU4NKsdy85K1jdwjBQ3+eUk 9 | HS7j+ECAnQJBAONbyN1eYjj3jOWhQof1LA4gOgT7dxM6j0e6EzIleRumfd6yHm8o 10 | ot5zTxqAxn8Nspiaf870InYq/DXp+b2uFSUCQEHThpsBX0EwzzeVkgRk5QPUU8pe 11 | hMhuy8X2/N8dDVDE6L4RmQY0LqNeWwhS7TOZYZm1VmdVbAOBFYL09Imv0EkCQH9k 12 | IVMwHFcRFehgh/fH/wxXMEs0X07t4/RrpW/WoVpF2ocaRIVCPqfn8i3Gc/IiyaxJ 13 | /U0Ha/vMMA9Bb4bBcOUCQQDnNBQowkdQD84QyKnz8EAAK/B6RLqOQwf3LU0FG+ZM 14 | ll2UJaE/OsmgSHavCrkaDGRGI0odxD7zJkGg4dqwymMH 15 | -----END RSA PRIVATE KEY----- 16 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | 2 | # node-rsync 3 | 4 | rsync implemented in node over http 5 | 6 | ## License 7 | 8 | (The MIT License) 9 | 10 | Copyright (c) 2011 Mihai Tomescu <matomesc@gmail.com> & Tolga Tezel <tolgatezel11@gmail.com> 11 | 12 | Permission is hereby granted, free of charge, to any person obtaining 13 | a copy of this software and associated documentation files (the 14 | 'Software'), to deal in the Software without restriction, including 15 | without limitation the rights to use, copy, modify, merge, publish, 16 | distribute, sublicense, and/or sell copies of the Software, and to 17 | permit persons to whom the Software is furnished to do so, subject to 18 | the following conditions: 19 | 20 | The above copyright notice and this permission notice shall be 21 | included in all copies or substantial portions of the Software. 22 | 23 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 24 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 26 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 27 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 28 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 29 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /lib/hash.js: -------------------------------------------------------------------------------- 1 | var crypto = require('crypto'); 2 | 3 | module.exports = { 4 | md5: function (data) { 5 | return crypto.createHash('md5') 6 | .update(data) 7 | .digest('hex'); 8 | }, 9 | weak32: function (data, prev, start, end) { 10 | var a = 0 11 | , b = 0 12 | , sum = 0 13 | , M = 1 << 16; 14 | 15 | if (!prev) { 16 | var len = start >= 0 && end >= 0 ? end - start : data.length 17 | , i = 0; 18 | 19 | for (; i < len; i++) { 20 | a += data[i]; 21 | b += a; 22 | } 23 | 24 | a %= M; 25 | b %= M; 26 | } else { 27 | var k = start 28 | , l = end - 1 29 | , prev_k = k - 1 30 | , prev_l = l - 1 31 | , prev_first = data[prev_k] 32 | , prev_last = data[prev_l] 33 | , curr_first = data[k] 34 | , curr_last = data[l]; 35 | 36 | a = (prev.a - prev_first + curr_last) % M 37 | b = (prev.b - (prev_l - prev_k + 1) * prev_first + a) % M 38 | } 39 | return { a: a, b: b, sum: a + b * M }; 40 | }, 41 | weak16: function (data) { 42 | return 0xffff & (data >> 16 ^ data*1009); 43 | } 44 | }; -------------------------------------------------------------------------------- /test/rename.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | , fs = require('fs') 3 | , path = require('path') 4 | , Anchor = require('../lib/anchor') 5 | , Client = require('../lib/client') 6 | 7 | , Seq = require('seq') 8 | 9 | //paths to rename 10 | var from = path.resolve(__dirname, '../sync/client/synced.txt') 11 | , to = path.resolve(__dirname, '../sync/client/renamed.txt') 12 | 13 | var fromNest = path.resolve(__dirname, '../sync/client/a/a-synced.txt') 14 | , toNest = path.resolve(__dirname, '../sync/client/a/a-renamed.txt') 15 | 16 | //test cases 17 | var cases = [ 18 | //{ fn: rename, from: from, to: to } 19 | //, { fn: rename, from: fromNest, to: toNest } 20 | { fn: renameDelay, from: from, to: to } 21 | //, { fn: renameDelay, from: fromNest, to: toNest } 22 | ] 23 | 24 | /* 25 | describe('#Rename', function () { 26 | before(function (done) { 27 | var anchor = new Anchor({ 28 | host: '127.0.0.1' 29 | , roots: { 30 | '../sync/client': '/sync/server' 31 | } 32 | }) 33 | 34 | anchor.on('ready', function () { done() }) 35 | }) 36 | 37 | cases.forEach(function (test) { 38 | it('does not throw', function(done) { 39 | test.fn.call(null, test.from, test.to, function (err) { 40 | assert.equal(null, err) 41 | done() 42 | }) 43 | }) 44 | }) 45 | }) 46 | */ 47 | 48 | //non-mocha test 49 | var clientAnchor = new Anchor({ 50 | host: '127.0.0.1' 51 | , localPort: 8080 52 | , remotePort: 8081 53 | , roots: { 54 | '../sync/client': '/home/ttezel/development/anchor/sync/server' 55 | } 56 | }) 57 | 58 | var serverAnchor = new Anchor({ 59 | host: '127.0.0.1' 60 | , localPort: 8081 61 | , remotePort: 8080 62 | , roots: { 63 | '../sync/server': '/home/ttezel/development/anchor/sync/client' 64 | } 65 | }) 66 | 67 | clientAnchor.on('ready', function () { 68 | var test = cases[0] 69 | 70 | test.fn.call(null, test.from, test.to, function (err) { 71 | if(err) throw err 72 | }) 73 | }) 74 | 75 | // 76 | // helpers 77 | // 78 | 79 | //rename @from to @to, then change it back 80 | function rename (from, to, cb) { 81 | fs.rename(from, to, function (err) { 82 | if(err) return cb(err) 83 | 84 | fs.rename(to, from, function (err) { 85 | if(err) return cb(err) 86 | return cb(null) 87 | }) 88 | }) 89 | } 90 | 91 | //rename @from to @to, then change it back after a timeout 92 | function renameDelay (from, to, cb) { 93 | fs.rename(from, to, function (err) { 94 | if(err) return cb(err) 95 | 96 | setTimeout(function () { 97 | fs.rename(to, from, function (err) { 98 | if(err) return cb(err) 99 | 100 | return cb(null) 101 | }) 102 | }, 1000) 103 | }) 104 | } -------------------------------------------------------------------------------- /test/rsync.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | , vows = require('vows') 3 | , assert = require('assert') 4 | , colors = require('colors') 5 | , helper = require('./helper') 6 | , rsync = require('../lib/node-rsync'); 7 | 8 | //modify modes 9 | var modes = [ 10 | 'prepend' 11 | , 'append' 12 | , 'middle' 13 | , 'remove' 14 | , 'hybrid' 15 | ]; 16 | 17 | function runRsync(blocksize, callback) { 18 | var sync = rsync.createRSync('../lib/files', blocksize) 19 | , t0 = Date.now(); 20 | 21 | sync.checksum('/server.txt', function (err, results) { 22 | if(err) { callback(err); } 23 | 24 | var t1 = Date.now(); 25 | sync.diff('/client.txt', results, function (err, diff) { 26 | if(err) { callback(err); } 27 | 28 | var t2 = Date.now(); 29 | sync.sync('/server.txt', diff, function(err, synced) { 30 | if(err) { callback(err); } 31 | 32 | var t3 = Date.now(); 33 | console.log( 'tot. time:' , t3-t0 34 | , 'sync time:' , t3-t2 35 | , 'diff time:' , t2-t1 36 | , 'chksum time:' , t1-t0 37 | ); 38 | console.log('---------------------------------' 39 | + '---------------------------------'); 40 | 41 | return callback(null, synced); 42 | }); 43 | }); 44 | }); 45 | }; 46 | 47 | // 48 | // test batch creation 49 | // 50 | var tests = vows.describe('Rsync tests') 51 | , numlevels = helper.levels.length 52 | , numModes = modes.length 53 | , blk = 750; 54 | 55 | for(var level = 0; level < numlevels; level++) { 56 | 57 | for(mode = 0; mode < numModes; mode++) { 58 | (function() { 59 | var lvl = level 60 | , m = mode; 61 | 62 | tests.addBatch({ 63 | 'file test': { 64 | topic: function() { 65 | var self = this 66 | , cur = lvl+m 67 | , tot = numlevels*numModes; 68 | 69 | console.log('Test %d of %d', cur, tot); 70 | console.log('mode: %s'.green, modes[m]); 71 | console.log('blocksize: %d', blk); 72 | 73 | helper.oldFile(lvl, function(err) { 74 | if (err) { throw err; } 75 | 76 | helper.newFile(m, function(err) { 77 | if (err) { throw err; } 78 | 79 | runRsync(blk, self.callback); 80 | }); 81 | }); 82 | }, 83 | 'sync matches newFile': function(err, result) { 84 | if(err) { throw err; } 85 | 86 | var client = fs.readFileSync(helper.allow[0], 'utf-8'); 87 | assert.equal(client, result.toString()); 88 | } 89 | } 90 | }); 91 | }).call(this); 92 | } 93 | } 94 | tests.export(module); 95 | -------------------------------------------------------------------------------- /test/helper.js: -------------------------------------------------------------------------------- 1 | // 2 | // File Helpers for Test Suite 3 | // 4 | var colors = require('colors') 5 | , fs = require('fs'); 6 | 7 | // inclusive max 8 | function rand (max) { 9 | return Math.floor(Math.random()*(max+1)); 10 | }; 11 | 12 | // range, inclusive on both ends 13 | function randRange (min, max) { 14 | return Math.floor(min + Math.random()*(max+1 - min)); 15 | }; 16 | 17 | // random string 18 | function randString (length) { 19 | var str = '' 20 | , min = 40 21 | , max = 126; 22 | 23 | for(var i = 0; i < length; i++) { 24 | var index = randRange(min, max); 25 | str += String.fromCharCode(index); 26 | } 27 | return str; 28 | }; 29 | 30 | function populatelevels () { 31 | levels = []; 32 | for(var i = 0; i < 24; i++) { 33 | levels.push(1 << i); 34 | } 35 | return levels; 36 | }; 37 | 38 | // 39 | // Helper 40 | // 41 | Helper = function() { 42 | this.allow = [ 43 | '../lib/files/client.txt' 44 | , '../lib/files/server.txt' 45 | ]; 46 | this.levels = populatelevels(); 47 | }; 48 | 49 | Helper.prototype = { 50 | // 51 | // file modification helpers 52 | // 53 | prepend: function (old, size) { 54 | return randString(size) + old; 55 | }, 56 | append: function (old, size) { 57 | return old + randString(size); 58 | }, 59 | middle: function (old, size) { 60 | var mods = randString(size) 61 | , k = randRange(1, old.length-2); 62 | 63 | return old.substr(0,k) + mods + old.substr(k+1); 64 | }, 65 | remove: function (old) { 66 | var min = randRange(0, old.length) 67 | , max = randRange(min, old.length); 68 | 69 | return old.substr(0,min) + old.substr(max); 70 | }, 71 | randomMode: function (modified, size) { 72 | var choices = [ 73 | this.prepend 74 | , this.append 75 | , this.middle 76 | , this.remove 77 | ]; 78 | 79 | var fn = choices[rand(choices.length-1)]; 80 | return fn.call(null, modified, size); 81 | }, 82 | //writes to server.txt 83 | oldFile: function (level, callback) { 84 | var path = this.allow[1] //server.txt 85 | , min = level === 0 ? 1 : this.levels[level - 1] 86 | , max = this.levels[level]; 87 | 88 | var size = randRange(min, max) 89 | , contents = randString(size); 90 | 91 | fs.writeFileSync(path, contents); 92 | return callback(null); 93 | }, 94 | //writes modified file contents to client.txt 95 | newFile: function (mode, callback) { 96 | var oldpath = this.allow[1] //server.txt 97 | , newpath = this.allow[0] //client.txt 98 | , raw = fs.readFileSync(oldpath); 99 | 100 | console.log('old filesize: %d Bytes'.bold, raw.length); 101 | 102 | var old = raw.toString() 103 | , modsize = rand(old.length) 104 | , numChanges = rand(old.length/2); 105 | 106 | if(mode > 1) { 107 | console.log('# of changes: %d'.cyan, numChanges); 108 | } 109 | 110 | var modified = old; 111 | 112 | switch(mode) { 113 | case 0: 114 | modified = this.prepend(modified, modsize); 115 | break; 116 | case 1: 117 | modified = this.append(modified, modsize); 118 | break; 119 | case 2: 120 | for(var i = 0; i < numChanges; i++) { 121 | modified = this.middle(modified, modsize); 122 | } 123 | break; 124 | case 3: 125 | for(var i = 0; i < numChanges; i++) { 126 | modified = this.remove(modified); 127 | } 128 | break; 129 | case 4: //hybrid 130 | for(var i = 0; i < numChanges; i++) { 131 | modified = this.randomMode(modified, modsize); 132 | } 133 | break; 134 | default: 135 | callback(new Error('mode not supported: '+mode.toString())); 136 | } 137 | 138 | raw = new Buffer(modified); 139 | console.log('new filesize: %d Bytes'.bold, raw.length); 140 | 141 | fs.writeFileSync(newpath, raw); 142 | 143 | return callback(null); 144 | } 145 | }; 146 | 147 | module.exports = new Helper(); -------------------------------------------------------------------------------- /lib/parser.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Parser 3 | * 4 | * Copyright(c) 2011 Mihai Tomescu 5 | * Copyright(c) 2011 Tolga Tezel 6 | * 7 | * MIT Licensed 8 | */ 9 | 10 | var util = require('util') 11 | , EventEmitter = require('events').EventEmitter; 12 | 13 | var Parser = exports.Parser = function () { 14 | this.states = { 15 | START: 0 16 | , LENGTH: 1 17 | , DATA: 2 18 | , DATA_END: 3 19 | }; 20 | this.reset(); 21 | EventEmitter.call(this); 22 | } 23 | 24 | util.inherits(Parser, EventEmitter); 25 | 26 | Parser.prototype.reset = function () { 27 | // data buffers 28 | this.buffers = []; 29 | 30 | // current buffer length and write offset 31 | this.length = ''; 32 | this.offset = 0; 33 | 34 | // initial state 35 | this.state = this.states.START; 36 | }; 37 | 38 | Parser.prototype.parse = function (data) { 39 | var states = this.states 40 | , index = 0; 41 | 42 | while (index < data.length) { 43 | var input = this.input = data[index]; 44 | 45 | switch (this.state) { 46 | case states.START: 47 | switch (input) { 48 | case 36: // $ 49 | this.state = states.LENGTH; 50 | break; 51 | default: 52 | this.syntaxError(); 53 | this.reset(); 54 | } 55 | break; 56 | case states.LENGTH: 57 | switch (input) { 58 | case 13: break; // CR 59 | case 10: // LF 60 | this.length = parseInt(this.length); 61 | if (isNaN(this.length)) { 62 | this.syntaxError(); 63 | this.reset(); 64 | } else { 65 | this.buffer = new Buffer(this.length); 66 | this.buffers.push(this.buffer); 67 | this.state = states.DATA; 68 | } 69 | break; 70 | default: 71 | if (input >= 48 && input <= 57) { 72 | this.length += input - 48; 73 | } else { 74 | this.syntaxError(); 75 | this.reset(); 76 | } 77 | } 78 | break; 79 | case states.DATA: 80 | if (this.offset < this.length) { 81 | // write to current buffer 82 | this.buffer[this.offset] = this.input; 83 | this.offset++; 84 | } else { 85 | switch (input) { 86 | case 13: break; // CR 87 | case 10: // LF 88 | this.state = states.DATA_END; 89 | break; 90 | default: 91 | this.syntaxError(); 92 | this.reset(); 93 | } 94 | } 95 | break; 96 | case states.DATA_END: 97 | switch (input) { 98 | case 36: // $ 99 | // more data 100 | this.state = states.LENGTH; 101 | 102 | this.offset = 0; 103 | this.buffer = null; 104 | this.length = ''; 105 | break; 106 | case 38: // & 107 | // done parsing 108 | this.gotMessage(); 109 | this.reset(); 110 | break; 111 | } 112 | break; 113 | } 114 | index++; 115 | } 116 | }; 117 | 118 | Parser.prototype.syntaxError = function () { 119 | var states = this.states; 120 | this.emit('error', 'Syntax Error - state: ' + this.state + ', input: ' + this.input); 121 | }; 122 | 123 | Parser.prototype.gotMessage = function () { 124 | this.emit('message', this.buffers); 125 | }; -------------------------------------------------------------------------------- /lib/anchor.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Anchor 3 | * 4 | * Copyright(c) 2011 Mihai Tomescu 5 | * Copyright(c) 2011 Tolga Tezel 6 | * 7 | * MIT Licensed 8 | * 9 | * Anchor listens on a port and performs updates on the local fs 10 | * when a sync request is received over http 11 | * 12 | * also watches the local fs on a specified filepath 13 | */ 14 | 15 | var EventEmitter = require('events').EventEmitter 16 | , fs = require('fs') 17 | , https = require('https') 18 | , path = require('path') 19 | , util = require('util') 20 | , Client = require('./client') 21 | 22 | // 23 | // grab credentials 24 | // 25 | var privateKey = fs.readFileSync('../credentials/privatekey.pem').toString() 26 | , certificate = fs.readFileSync('../credentials/certificate.pem').toString() 27 | 28 | // 29 | // regex to grab the resource 30 | // 31 | var rgxRoutes = /\/(rename|change)/; 32 | 33 | module.exports = Anchor 34 | 35 | // 36 | // usage: 37 | // 38 | // new Anchor({ 39 | // host: 'mydomain.myhost.com' 40 | // [, blockSize: 750 ] 41 | // [, port: 8080 ] 42 | // , roots: { 43 | // '../sync': './sync' 44 | // } 45 | // }) 46 | // 47 | function Anchor(options) { 48 | if(!options || typeof options !== 'object') throw new Error('Anchor options must be Object') 49 | if(!options.host) throw new Error('Must specify remote host to sync on') 50 | if(!options.roots || typeof options.roots !== 'object') throw new Error('Must specify sync paths') 51 | EventEmitter.call(this) 52 | 53 | options.blockSize = options.blockSize || 750 54 | options.localPort = options.localPort || 8080 55 | options.remotePort = options.remotePort || 8080 56 | 57 | var self = this 58 | , relRoot = Object.keys(options.roots)[0] 59 | 60 | options.localRoot = path.resolve(__dirname, relRoot) 61 | options.remoteRoot = options.roots[relRoot] 62 | 63 | this.options = options 64 | 65 | var creds = { key: privateKey, cert: certificate } 66 | 67 | //get the route using regex, once body is received 68 | //update the local fs 69 | function handleRequest(req, res) { 70 | var match = rgxRoutes.exec(req.url) 71 | if(!match) { 72 | res.writeHead(400) 73 | res.end() 74 | return 75 | } 76 | 77 | var action = match[1], data = '' 78 | 79 | req.setEncoding('utf8') 80 | req 81 | .on('data', function (chunk) { data += chunk }) 82 | .on('end', function () { 83 | try { 84 | var parsed = JSON.parse(data) 85 | } catch(err) { 86 | var error = new Error('ANCHOR: request is not valid JSON') 87 | error.msg = err 88 | throw error 89 | } 90 | self.updateLocal(req, res, action, parsed) 91 | }) 92 | } 93 | this.server = https.createServer(creds, handleRequest) 94 | this.server.listen(this.options.localPort) 95 | 96 | console.log('SERVER: listening @ port', this.options.localPort) 97 | 98 | this.client = new Client(options) 99 | this.client.on('ready', function () { self.emit('ready') }) 100 | } 101 | util.inherits(Anchor, EventEmitter) 102 | 103 | Anchor.prototype.updateLocal = function (req, res, action, parsed) { 104 | var self = this 105 | , localRoot = this.options.localRoot 106 | 107 | console.log('received change. updating...') 108 | 109 | switch(action) { 110 | case 'rename': 111 | var oldName = path.resolve(localRoot, parsed.oldName) 112 | , newName = path.resolve(localRoot, parsed.newName) 113 | 114 | console.log('oldNamez', oldName) 115 | console.log('newNamez', newName) 116 | 117 | //temporarily unwatch file to prevent fs.watch firing 118 | //otherwise it would be an infinite update cycle 119 | this.client.ignorePath(oldName) 120 | this.client.ignorePath(newName) 121 | 122 | //apply the rename 123 | fs.rename(oldName, newName, function (err) { 124 | if(err) { 125 | res.writeHead(500) 126 | 127 | var msg = util.format('rename from `%s` to `%s` failed.', oldName, newName) 128 | res.end(msg) 129 | throw err 130 | } 131 | 132 | //update cache 133 | fs.stat(newName, function (err, stat) { 134 | if (err) throw err 135 | self.client.cache[stat.ino] = { fpath: newName, stat: stat } 136 | self.client.unignorePath(newName) 137 | res.end() 138 | return 139 | }) 140 | }) 141 | break 142 | case 'change': 143 | console.log('file contents changed') 144 | break 145 | default: 146 | throw new Error('action `' + action + '` is not supported') 147 | } 148 | } -------------------------------------------------------------------------------- /test/tests.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | , vows = require('vows') 3 | , assert = require('assert') 4 | , rsync = require('../lib/node-rsync.js'); 5 | 6 | var clientFilePath = './lib/files/hello-client.txt' 7 | , serverFilePath = './lib/files/hello-server.txt'; 8 | 9 | //Rsnyc tests 10 | vows.describe('Rsync - updating server file') 11 | .addBatch({ 12 | 'chunk of size chunkSize = 2 in midst': { 13 | topic: function() { 14 | var self = this; 15 | 16 | rsync.chunkSize = 2; 17 | 18 | var clientContents = 'aabbqqcc' 19 | , serverContents = 'aabbcc' 20 | 21 | serverUpdate( 22 | clientFilePath 23 | , clientContents 24 | , serverFilePath 25 | , serverContents 26 | , function(err, outgoing) { 27 | if(err) throw err; 28 | self.callback(null, outgoing.data.toString()) 29 | }); 30 | }, 31 | 'diffence is just that block': function(topic) { 32 | assert.equal('qq', topic); 33 | }, 34 | 'two chunks of size chunkSize = 2 in midst': { 35 | topic: function() { 36 | var self = this 37 | , clientContents = 'aappbbqqcc' 38 | , serverContents = 'aabbcc' 39 | , changes = ['pp', 'qq']; 40 | 41 | rsync.chunkSize = 2; 42 | 43 | serverUpdate( 44 | clientFilePath 45 | , clientContents 46 | , serverFilePath 47 | , serverContents 48 | , function(err, outgoing) { 49 | if(err) throw err; 50 | self.callback(null, {outgoing: outgoing.data.toString(), changes: changes}) 51 | }); 52 | }, 53 | 'diffences = the two chunks': function(topic) { 54 | assert.include(topic.changes, topic.outgoing); 55 | delete(topic.changes[topic.changes.indexOf(topic.outgoing)]); 56 | } 57 | } 58 | } 59 | }) 60 | .addBatch({ 61 | 'chunks all over the place': { 62 | topic: function() { 63 | var self = this 64 | , clientContents = '12345678910111213141516' 65 | , serverContents = '37891012131516' 66 | , changes = ['12', '456', '11', '4']; 67 | 68 | rsync.chunkSize = 3; 69 | 70 | serverUpdate( 71 | clientFilePath 72 | , clientContents 73 | , serverFilePath 74 | , serverContents 75 | , function(err, outgoing) { 76 | if(err) throw err; 77 | self.callback(null, {outgoing: outgoing.data.toString(), changes: changes}) 78 | }); 79 | }, 80 | 'diffences are correct': function(topic) { 81 | console.log('all ova da place tests'); 82 | assert.include(topic.changes, topic.outgoing); 83 | delete(topic.changes[topic.changes.indexOf(topic.outgoing)]); 84 | } 85 | } 86 | }) 87 | .run(); 88 | 89 | 90 | //HELPERS 91 | 92 | // 93 | // Overwrite file contents of @path (string) with @contents (string) 94 | // Checks if path is an allowed path, so you can't overwrite important files 95 | // 96 | function fileOverwrite (path, contents, callback) { 97 | //allowed paths 98 | var allowed = [ 99 | clientFilePath 100 | , serverFilePath 101 | ]; 102 | 103 | if(allowed.indexOf(path) === -1) { return; } //nuh-uh! 104 | 105 | fs.open(path, 'w+', function (err, fd) { 106 | if (err) { 107 | return callback(err); 108 | } 109 | 110 | fs.truncate(fd, 0, function(err) { 111 | if (err) { 112 | return callback(err); 113 | } 114 | 115 | fs.writeFile(path, contents, callback); 116 | }); 117 | }); 118 | }; 119 | 120 | // 121 | // Helper for basic Rsync testing 122 | // Overwrite @clientPath with @clientContents 123 | // Overwrite @serverPath with@serverContents 124 | // When all this is done: 125 | // - checksum the server file 126 | // - search the client file for new blocks 127 | // - call @callback 128 | // 129 | function serverUpdate (clientPath, clientContents, serverPath, serverContents, callback) { 130 | fileOverwrite(clientPath, clientContents, function(err, written) { 131 | if(err) callback(err); 132 | 133 | fileOverwrite(serverPath, serverContents, function(err, written) { 134 | if(err) callback(err); 135 | 136 | rsync.chunk(serverPath, false, function (err, chunkData) { 137 | if(err) callback(err); 138 | 139 | var checksums = rsync.checksum(chunkData); 140 | 141 | rsync.search(clientPath, checksums, 142 | callback 143 | ); 144 | }); 145 | }); 146 | }); 147 | }; -------------------------------------------------------------------------------- /lib/client.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Node rsync client 3 | * 4 | * Copyright(c) 2011 Mihai Tomescu 5 | * Copyright(c) 2011 Tolga Tezel 6 | * 7 | * MIT Licensed 8 | * 9 | * Client watches the local fs, and performs synchronization 10 | * requests to remote to update their files 11 | * 12 | */ 13 | 14 | var EventEmitter = require('events').EventEmitter 15 | , fs = require('fs') 16 | , https = require('https') 17 | , path = require('path') 18 | , util = require('util') 19 | , findit = require('findit') 20 | 21 | module.exports = Client 22 | 23 | // 24 | // emit `ready` once all local files are watched and cached 25 | // 26 | function Client (opts) { 27 | if(!opts.localRoot) throw new Error('must specify `localRoot`') 28 | EventEmitter.call(this) 29 | 30 | this.opts = opts 31 | this.cache = {} 32 | this.ignoreChanges = [] 33 | 34 | //watch localRoot recursively and notify remote of updates 35 | this.watchPath(opts.localRoot, true) 36 | } 37 | util.inherits(Client, EventEmitter) 38 | 39 | // 40 | // watch @fpath for changes (recurses thru @fpath) 41 | // emit `ready` when done 42 | // 43 | Client.prototype.watchPath = function (fpath) { 44 | var self = this 45 | 46 | //watch @dir for renames only, call #updateRemote on a change 47 | function watchDir (dir) { 48 | return fs.watch(dir, function (event, filename) { 49 | if (!filename) return console.log('\nfs.watch: no filename emitted') 50 | if ('rename' === event) { 51 | var abspath = path.resolve(dir, filename) 52 | console.log('rename of:', abspath) 53 | if (self.ignoreChanges.indexOf(abspath) === -1) 54 | return self.updateRemote(event, abspath) 55 | } else if ('change' === event) { 56 | return console.log('file contents change to', filename) 57 | } 58 | }) 59 | } 60 | 61 | //cache top level dir's stats 62 | fs.stat(fpath, function (err, stat) { 63 | if (err) throw err 64 | self.cache[stat.ino] = { fpath: fpath, stat: stat } 65 | }) 66 | //watch top level dir 67 | watchDir(fpath) 68 | 69 | //recursively walk down @fpath and track contained files 70 | var finder = findit.find(fpath) 71 | finder.on('directory', function (dir, stat) { 72 | self.cache[stat.ino] = { fpath: dir, stat: stat } 73 | watchDir(dir) 74 | }) 75 | finder.on('file', function (file, stat) { 76 | self.cache[stat.ino] = { fpath: file, stat: stat } 77 | return fs.watch(file, function (event, filename) { 78 | console.log('watched file %s had a %s event', filename, event) 79 | if (self.ignoreChanges.indexOf(file) === -1) 80 | return self.updateRemote(event, file) 81 | }) 82 | }) 83 | finder.on('end', function () { self.emit('ready') }) 84 | return this 85 | } 86 | 87 | // 88 | // ignore changes to fpath 89 | // 90 | Client.prototype.ignorePath = function (fpath) { 91 | this.ignoreChanges.push(fpath) 92 | } 93 | 94 | Client.prototype.unignorePath = function (fpath) { 95 | var index = this.ignoreChanges.indexOf(fpath) 96 | if(index !== -1) this.ignoreChanges.splice(index, 1) 97 | } 98 | 99 | // 100 | // @event event emitted by fs.watch (`rename` or `change`) 101 | // @fname absolute path to file/folder 102 | // 103 | Client.prototype.updateRemote = function (event, fname) { 104 | var self = this 105 | , exists = path.existsSync(fname) 106 | 107 | //file has been deleted or renamed - do nothing 108 | if(!exists) return 109 | 110 | var reqOpts = { 111 | host: this.opts.host 112 | , port: this.opts.remotePort 113 | , method: 'POST' 114 | , path: '/' + event 115 | } 116 | 117 | var req = https.request(reqOpts, function (res) { 118 | if (res.statusCode !== 200) console.log('res.statusCode', res.statusCode) 119 | res.on('data', function (d) { console.log('data', d.toString()) }) 120 | }) 121 | req.on('error', function (e) { 122 | console.log('https client request error:', e) 123 | }) 124 | 125 | //send synchronization instructions to remote 126 | //then update local cache 127 | switch(event) { 128 | case 'rename': 129 | fs.stat(fname, function (err, stat) { 130 | if (err) throw err 131 | var cached = self.cache[stat.ino] 132 | if (!cached) throw new Error('renamed file was not cached') 133 | 134 | var localRoot = self.opts.localRoot 135 | 136 | var data = { 137 | oldName: path.relative(localRoot, cached.fpath) 138 | , newName: path.relative(localRoot, fname) 139 | } 140 | 141 | console.log('cached.fpath', cached.fpath) 142 | 143 | console.log('oldName', data.oldName) 144 | console.log('newName', data.newName) 145 | 146 | console.log('CLIENT: %s rename `%s` to `%s`', self.opts.localPort, data.oldName, data.newName) 147 | 148 | req.end(JSON.stringify(data)) 149 | 150 | //update cache 151 | self.cache[stat.ino] = { fpath: fname, stat: stat } 152 | }) 153 | break 154 | case 'change': 155 | console.log('CLIENT: content change') 156 | break 157 | default: 158 | throw new Error('CLIENT: event `' + event + '` is not supported') 159 | } 160 | } -------------------------------------------------------------------------------- /lib/node-rsync.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * node-rsync 3 | * 4 | * Copyright(c) 2011 Mihai Tomescu 5 | * Copyright(c) 2011 Tolga Tezel 6 | * 7 | * MIT Licensed 8 | */ 9 | 10 | var fs = require('fs') 11 | , util = require('util') 12 | , hash = require('./hash') 13 | , EventEmitter = require('events').EventEmitter; 14 | 15 | // 16 | // Algorithm functions 17 | // 18 | 19 | function createHashtable(checksums) { 20 | var hashtable = {} 21 | , len = checksums.length 22 | , i = 0; 23 | 24 | for (; i < len; i++) { 25 | var checksum = checksums[i] 26 | , weak16 = hash.weak16(checksum.weak); 27 | 28 | if (hashtable[weak16]) { 29 | hashtable[weak16].push(checksum); 30 | } else { 31 | hashtable[weak16] = [checksum]; 32 | } 33 | } 34 | return hashtable; 35 | } 36 | 37 | // rolls through data 1 byte at a time, and determines sync instructions 38 | function roll(data, checksums, chunkSize) { 39 | var results = [] 40 | , hashtable = createHashtable(checksums) 41 | , length = data.length 42 | , start = 0 43 | , end = chunkSize > length ? length : chunkSize 44 | // Updated when a block matches 45 | , lastMatchedEnd = 0 46 | // This gets updated every iteration with the previous weak 32bit hash 47 | , prevRollingWeak = null; 48 | 49 | for (; end <= length; start++, end++) { 50 | var weak = hash.weak32(data, prevRollingWeak, start, end) 51 | , weak16 = hash.weak16(weak.sum) 52 | , match = false; 53 | 54 | prevRollingWeak = weak; 55 | 56 | if (hashtable[weak16]) { 57 | var len = hashtable[weak16].length 58 | , i = 0; 59 | 60 | for (; i < len; i++) { 61 | if (hashtable[weak16][i].weak === weak.sum) { 62 | var mightMatch = hashtable[weak16][i] 63 | , chunk = data.slice(start, end) 64 | , strong = hash.md5(chunk); 65 | 66 | if (mightMatch.strong === strong) { 67 | match = mightMatch; 68 | break; 69 | } 70 | } 71 | } 72 | } 73 | 74 | if (match) { 75 | if(start < lastMatchedEnd) { 76 | var d = data.slice(lastMatchedEnd - 1, end); 77 | results.push({ 78 | data: d 79 | , index: match.index 80 | }); 81 | } else if (start - lastMatchedEnd > 0) { 82 | var d = data.slice(lastMatchedEnd, start); 83 | results.push({ 84 | data: d 85 | , index: match.index 86 | }); 87 | } else { 88 | results.push({ 89 | index: match.index 90 | }); 91 | } 92 | 93 | lastMatchedEnd = end; 94 | } else if (end === length) { 95 | // No match and last block 96 | var d = data.slice(lastMatchedEnd); 97 | results.push({ 98 | data: d 99 | }); 100 | } 101 | } 102 | 103 | return results; 104 | } 105 | 106 | // 107 | // RSync implementation 108 | // 109 | 110 | var RSync = function (size) { 111 | // block size used in checksums 112 | this.size = size; 113 | 114 | // file cache 115 | this.cache = {}; 116 | }; 117 | 118 | util.inherits(RSync, EventEmitter); 119 | 120 | RSync.prototype = { 121 | checksum: function (path, callback) { 122 | var self = this; 123 | 124 | fs.readFile(path, function (err, data) { 125 | if (err) { return callback(err); } 126 | 127 | var length = data.length 128 | , incr = self.size 129 | , start = 0 130 | , end = incr > length ? length : incr 131 | , blockIndex = 0; 132 | 133 | // cache file 134 | self.cache[path] = data; 135 | 136 | var result = ''; 137 | 138 | while (start < length) { 139 | var chunk = data.slice(start, end) 140 | , weak = hash.weak32(chunk).sum 141 | , strong = hash.md5(chunk); 142 | 143 | result += '$' + blockIndex.toString().length + '\r\n'; 144 | result += blockIndex + '\r\n' 145 | result += '$' + weak.length + '\r\n'; 146 | result += weak + '\r\n'; 147 | result += '$32\r\n'; 148 | result += strong + '\r\n'; 149 | result += '$\r\n'; 150 | 151 | // update slice indices 152 | start += incr; 153 | end = (end + incr) > length ? length : end + incr; 154 | 155 | // update block index 156 | blockIndex++; 157 | } 158 | 159 | return callback(null, result); 160 | }); 161 | }, 162 | // 163 | // Calculates instructions for synching using the given checksums 164 | // 165 | // Instructions are an array of sync objects which contain 166 | // the index of the block before which the the data must to be 167 | // inserted: 168 | // [{ index: null, data: 'cf92d' }, { index: 0, data: 'da41f' }] 169 | // 170 | diff: function (path, checksums, callback) { 171 | if (!checksums.length) { return callback(null, results); } 172 | 173 | var self = this 174 | , path = this.root + path; 175 | 176 | fs.stat(path, function(err, stats) { 177 | var filesize = stats.size; 178 | 179 | // roll through the file 180 | fs.readFile(path, 'utf-8', function (err, data) { 181 | if (err) { return callback(err); } 182 | return callback(null, roll(data, checksums, self.size)); 183 | }); 184 | }); 185 | }, 186 | // 187 | // Syncs a file based on instructions, then recalculates its 188 | // checksum 189 | // 190 | sync: function (path, diff, callback) { 191 | var self = this 192 | , path = this.root + path 193 | , raw = this.files[path] 194 | , i = 0 195 | , len = diff.length; 196 | 197 | if(typeof raw === 'undefined') { 198 | var err = new Error('must do checksum() first'); 199 | return callback(err, null); 200 | } 201 | 202 | //get slice of raw file from block's index 203 | function rawslice(index) { 204 | var start = index*self.size 205 | , end = start + self.size > raw.length 206 | ? raw.length 207 | : start + self.size; 208 | 209 | return raw.slice(start, end); 210 | } 211 | 212 | var synced = ''; 213 | 214 | for(; i < len; i++) { 215 | var chunk = diff[i]; 216 | 217 | if(typeof chunk.data === 'undefined') { //use slice of original file 218 | synced += rawslice(chunk.index).toString(); 219 | } else { 220 | synced += chunk.data.toString(); 221 | 222 | if(typeof chunk.index !== 'undefined') { 223 | synced += rawslice(chunk.index).toString(); 224 | } 225 | } 226 | } 227 | 228 | delete this.files[path]; 229 | raw = new Buffer(synced); 230 | 231 | return callback(null, raw); 232 | }, 233 | // 234 | // Files that are being synced 235 | // 236 | files: {} 237 | }; 238 | 239 | // Public API 240 | 241 | exports.version = '0.0.1'; 242 | 243 | exports.createRSync = function (root, size) { 244 | return new RSync(root, size || 750); 245 | }; 246 | --------------------------------------------------------------------------------