├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .npmrc ├── .nvmrc ├── .travis.yml ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── seq-file.js └── test ├── 00-basic.js ├── 01-crash.js ├── 02-after-crash.js ├── 03-save-non-number.js ├── 04-read-non-number.js ├── 05-test-starts-with-zero.js └── zz-cleanup.js /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: master 6 | pull_request: 7 | 8 | jobs: 9 | build: 10 | name: Build & Test 11 | runs-on: ubuntu-latest 12 | 13 | strategy: 14 | matrix: 15 | node-version: [20.x] 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | 20 | - name: Use Node.js ${{ matrix.node-version }} 21 | uses: actions/setup-node@v2 22 | with: 23 | node-version: ${{ matrix.node-version }} 24 | registry-url: 'https://registry.npmjs.org' 25 | 26 | - name: Set all tokens 27 | run: >- 28 | npm config set //registry.npmjs.org/:_authToken ${{ secrets.NPMCORP_TOKEN }} 29 | 30 | - name: Build 31 | run: npm ci 32 | 33 | - name: Lint 34 | run: npm run lint 35 | 36 | - name: Test 37 | run: npm test -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | test/*.TMP 2 | test/*.seq 3 | node_modules 4 | .nyc_output -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | @npm:registry=https://npm.pkg.github.com/ 2 | lockfile-version=3 3 | 4 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | lts/iron 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | before_script: npm install -g npm@latest 3 | node_js: 4 | - '0.10' 5 | - '0.12' 6 | - 'iojs' 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The ISC License 2 | 3 | Copyright (c) Isaac Z. Schlueter 4 | 5 | Permission to use, copy, modify, and/or distribute this software for any 6 | purpose with or without fee is hereby granted, provided that the above 7 | copyright notice and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR 15 | IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # seq-file 2 | 3 | A module for storing the ever-increasing sequence files when following 4 | couchdb _changes feeds. 5 | 6 | Saves the sequence ID in an atomic way, so that it doesn't clobber the 7 | file if it crashes mid-save. Only does a single save at a time, so 8 | you can bang on it repeatedy, and it'll avoid doing unnecessary file 9 | IO or weird cases where two writes cross paths in odd ways. 10 | 11 | ## USAGE 12 | 13 | ```javascript 14 | var SF = require('seq-file') 15 | var s = new SF('sequence.seq') 16 | 17 | s.save(10) 18 | console.log(s.seq) // 10 19 | s.save(11) // won't actually save, because still saving the 10 20 | console.log(s.seq) // 11. You get the idea. 21 | 22 | // some time in the future, another change comes in 23 | s.save(21) 24 | 25 | // oh no! crash while saving! 26 | s.save(34) 27 | throw 'pwn' // file now contains "21", not "" 28 | ``` 29 | 30 | ## OPTIONS 31 | 32 | * **frequency:** modify how frequently a sequence number is saved. 33 | 34 | ```javascript 35 | var SF = require('seq-file', { 36 | frequency: 4 37 | }) 38 | var s = new SF('sequence.seq') 39 | s.save(11) // this won't save (we only save every 4 increments). 40 | s.save(12) // this will totally save. 41 | ``` 42 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "seq-file", 3 | "version": "2.0.0", 4 | "description": "A module for storing the ever-increasing sequence files when following couchdb _changes feeds.", 5 | "main": "seq-file.js", 6 | "directories": { 7 | "test": "test" 8 | }, 9 | "dependencies": { 10 | "touch": "^3.1.0" 11 | }, 12 | "devDependencies": { 13 | "rimraf": "^3.0.2", 14 | "standard": "^17.1.0", 15 | "tap": "^21.0.0" 16 | }, 17 | "scripts": { 18 | "lint": "standard", 19 | "lint:fix": "standard --fix", 20 | "test": "tap test/*.js --jobs=1 --disable-coverage" 21 | }, 22 | "repository": { 23 | "type": "git", 24 | "url": "git://github.com/isaacs/seq-file" 25 | }, 26 | "keywords": [ 27 | "changes", 28 | "sequence", 29 | "file" 30 | ], 31 | "author": "Isaac Z. Schlueter (http://blog.izs.me/)", 32 | "license": "ISC", 33 | "bugs": { 34 | "url": "https://github.com/isaacs/seq-file/issues" 35 | }, 36 | "homepage": "https://github.com/isaacs/seq-file" 37 | } 38 | -------------------------------------------------------------------------------- /seq-file.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var touch = require('touch') 3 | 4 | module.exports = SeqFile 5 | 6 | function SeqFile (file, opts) { 7 | if (!file) { throw new TypeError('need to specify a file') } 8 | 9 | opts = opts || {} 10 | 11 | this.file = file 12 | this.saving = false 13 | this.seq = 0 14 | this.frequency = opts.frequency || 1 15 | this.delimiter = opts.delimiter || '-' 16 | } 17 | 18 | SeqFile.prototype.read = function (cb) { 19 | var _this = this 20 | 21 | touch(this.file, function () { 22 | fs.readFile(_this.file, 'ascii', _this.onRead.bind(_this, cb)) 23 | }) 24 | } 25 | 26 | SeqFile.prototype.readSync = function () { 27 | var er, data 28 | try { 29 | touch.sync(this.file) 30 | data = fs.readFileSync(this.file, 'ascii') 31 | } catch (e) { 32 | er = e 33 | } 34 | return this.onRead(null, er, data) 35 | } 36 | 37 | SeqFile.prototype.isSeqGreater = function (newSeq, oldSeq) { 38 | return Number((newSeq + '').split(this.delimiter, 1)) > Number((oldSeq + '').split(this.delimiter, 1)) 39 | } 40 | 41 | SeqFile.prototype.save = function (n) { 42 | var skip 43 | if (n) { 44 | if (this.isSeqGreater(n, this.seq)) { this.seq = n } else if (this.seq === 0 && typeof n === 'string') { this.seq = n } 45 | } 46 | 47 | skip = (n || 0) % this.frequency 48 | 49 | // only save occasionally to cut down on I/O. 50 | if (!isNaN(skip) && skip !== 0) return 51 | 52 | if (!this.saving) { 53 | this.saving = true 54 | var t = this.file + '.TMP' 55 | var data = this.seq + '\n' 56 | fs.writeFile(t, data, this.onSave.bind(this)) 57 | } 58 | } 59 | 60 | SeqFile.prototype.onSave = function (er) { 61 | var cb = this.onFinish.bind(this) 62 | if (!er) { fs.rename(this.file + '.TMP', this.file, cb) } else { fs.unlink(this.file + '.TMP', cb) } 63 | } 64 | 65 | SeqFile.prototype.onFinish = function () { 66 | this.saving = false 67 | } 68 | 69 | SeqFile.prototype.saveSync = function (n) { 70 | if (n) { 71 | this.seq = n 72 | } 73 | try { 74 | this.saving = true 75 | var t = this.file + '.TMP' 76 | var data = this.seq + '\n' 77 | fs.writeFileSync(t, data) 78 | fs.renameSync(this.file + '.TMP', this.file) 79 | } catch (e) { 80 | console.log(e) 81 | fs.unlinkSync(this.file + '.TMP') 82 | } 83 | this.saving = false 84 | } 85 | 86 | SeqFile.prototype.onRead = function (cb, er, data) { 87 | if (er && er.code === 'ENOENT') { data = 0 } else if (er) { 88 | if (cb) { return cb(er) } else { throw er } 89 | } 90 | 91 | if (data === undefined) { data = 0 } 92 | 93 | if (data.length > 1) { 94 | // remove delimiter 95 | data = (data + '').trim() 96 | if (/^\d+$/.test(data)) { data = +data } else { 97 | // compare strings 98 | this.seq = this.seq + '' 99 | } 100 | } else { 101 | data = 0 102 | } 103 | 104 | if (data > this.seq) { this.seq = data } 105 | 106 | if (cb) { cb(er, data) } else if (er) { throw er } else { return data } 107 | } 108 | -------------------------------------------------------------------------------- /test/00-basic.js: -------------------------------------------------------------------------------- 1 | var test = require('tap').test 2 | 3 | var SeqFile = require('../seq-file.js') 4 | 5 | var path = require('path') 6 | var sf = path.join(__dirname, '/test.seq') 7 | var rimraf = require('rimraf') 8 | 9 | rimraf.sync(sf) 10 | 11 | test('touch', function (t) { 12 | var s = new SeqFile(sf) 13 | 14 | // the .seq file is created 15 | // if it doesn't exist. 16 | s.read(function (er, data) { 17 | t.equal(er, null) 18 | t.equal(data, 0) 19 | t.equal(s.seq, 0) 20 | 21 | // set sequence to 10 22 | // for read tests. 23 | s.save(10) 24 | setTimeout(function () { 25 | t.end() 26 | }, 100) 27 | }) 28 | }) 29 | 30 | test('read', function (t) { 31 | var s = new SeqFile(sf) 32 | s.read(function (er, data) { 33 | t.equal(data, 10) 34 | t.equal(s.seq, 10) 35 | t.end() 36 | }) 37 | }) 38 | 39 | test('bump a few', function (t) { 40 | var s = new SeqFile(sf) 41 | s.save(11) 42 | // these few should NOT be saved, because a save is in process. 43 | s.save(12) 44 | s.save(13) 45 | setTimeout(function () { 46 | var q = s.readSync() 47 | t.equal(q, 11) 48 | t.equal(s.seq, 13) 49 | t.notOk(s.saving) 50 | s.save() 51 | setTimeout(function () { 52 | var q = s.readSync() 53 | t.equal(q, 13) 54 | t.equal(s.seq, 13) 55 | t.end() 56 | }, 100) 57 | }, 100) 58 | }) 59 | 60 | test('it should allow save frequency to be changed', function (t) { 61 | var s = new SeqFile(sf, { 62 | frequency: 4 63 | }) 64 | s.save(11) 65 | s.save(12) 66 | s.save(13) 67 | setTimeout(function () { 68 | var q = s.readSync() 69 | t.equal(q, 12) 70 | t.equal(s.seq, 13) 71 | t.notOk(s.saving) 72 | t.end() 73 | }, 100) 74 | }) 75 | -------------------------------------------------------------------------------- /test/01-crash.js: -------------------------------------------------------------------------------- 1 | var test = require('tap').test 2 | 3 | var SeqFile = require('../seq-file.js') 4 | 5 | var path = require('path') 6 | var sf = path.join(__dirname, '/test.seq') 7 | var fs = require('fs') 8 | fs.writeFileSync(sf, '10\n', 'ascii') 9 | 10 | test('setup', function (t) { 11 | var s = new SeqFile(sf) 12 | s.read(function (er, data) { 13 | if (er) throw er 14 | t.equal(data, 10) 15 | t.equal(s.seq, 10) 16 | t.end() 17 | }) 18 | }) 19 | 20 | test('try to save, but crash in the process', function (t) { 21 | console.log('crasher') 22 | fs.writeFile = function (path, data, cb) { 23 | var fd = fs.openSync(path, 'w') 24 | fs.closeSync(fd) 25 | process.nextTick(t.end.bind(t)) 26 | } 27 | var s = new SeqFile(sf) 28 | s.save(11) 29 | }) 30 | -------------------------------------------------------------------------------- /test/02-after-crash.js: -------------------------------------------------------------------------------- 1 | var test = require('tap').test 2 | 3 | var fs = require('fs') 4 | var path = require('path') 5 | var sf = path.join(__dirname, '/test.seq') 6 | 7 | test('recover from mid-save crash', function (t) { 8 | t.equal(fs.readFileSync(sf, 'ascii'), '10\n') 9 | t.end() 10 | }) 11 | -------------------------------------------------------------------------------- /test/03-save-non-number.js: -------------------------------------------------------------------------------- 1 | var test = require('tap').test 2 | 3 | var SeqFile = require('../seq-file.js') 4 | 5 | var fs = require('fs') 6 | var path = require('path') 7 | var sf = path.join(__dirname, '/test.seq') 8 | 9 | test('saves non number sequence ids', function (t) { 10 | var s = new SeqFile(sf) 11 | s.seq = '1-1110' 12 | s.save('2-1111') 13 | setTimeout(function () { 14 | t.equal(fs.readFileSync(sf, 'ascii'), '2-1111\n') 15 | t.end() 16 | }, 50) 17 | }) 18 | -------------------------------------------------------------------------------- /test/04-read-non-number.js: -------------------------------------------------------------------------------- 1 | var test = require('tap').test 2 | 3 | var SeqFile = require('../seq-file.js') 4 | 5 | var path = require('path') 6 | var sf = path.join(__dirname, '/test.seq') 7 | 8 | test('reads non number sequence ids from file', function (t) { 9 | var s = new SeqFile(sf) 10 | s.read(function (err, data) { 11 | t.equal(data, '2-1111') 12 | t.equal(s.seq, '2-1111') 13 | t.same(err, undefined) 14 | t.end() 15 | }) 16 | }) 17 | -------------------------------------------------------------------------------- /test/05-test-starts-with-zero.js: -------------------------------------------------------------------------------- 1 | var test = require('tap').test 2 | 3 | var SeqFile = require('../seq-file.js') 4 | 5 | var fs = require('fs') 6 | var path = require('path') 7 | var sf = path.join(__dirname, '/test.seq') 8 | 9 | test('saves non number sequences even if they start with 0', function (t) { 10 | var s = new SeqFile(sf) 11 | // number zero cannot be compared to strings that start with "0" 12 | s.seq = 0 13 | s.save('0-00') 14 | setTimeout(function () { 15 | t.equal(fs.readFileSync(sf) + '', '0-00\n') 16 | t.end() 17 | }, 50) 18 | }) 19 | -------------------------------------------------------------------------------- /test/zz-cleanup.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var path = require('path') 3 | var sf = path.join(__dirname, '/test.seq') 4 | var test = require('tap').test 5 | 6 | test('cleanup', function (t) { 7 | try { fs.unlinkSync(sf) } catch (er) {} 8 | try { fs.unlinkSync(sf + '.TMP') } catch (er) {} 9 | t.pass('ok') 10 | t.end() 11 | }) 12 | --------------------------------------------------------------------------------