├── .gitignore ├── .travis.yml ├── test ├── rename-fail.js ├── toolong.js ├── chown.js ├── slow-close.js ├── basic.js └── rename-eperm.js ├── LICENSE ├── package.json ├── README.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | coverage/ 3 | .nyc_output/ 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | sudo: false 3 | before_install: 4 | - "npm -g install npm" 5 | node_js: 6 | - "10" 7 | - "8" 8 | - "6" 9 | - "11" 10 | -------------------------------------------------------------------------------- /test/rename-fail.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | var fs = require('graceful-fs') 3 | var path = require('path') 4 | var test = require('tap').test 5 | var rimraf = require('rimraf') 6 | var writeStream = require('../index.js') 7 | 8 | var target = path.resolve(__dirname, 'test-rename') 9 | 10 | test('rename fails', function (t) { 11 | t.plan(1) 12 | fs.rename = function (src, dest, cb) { 13 | cb(new Error('TEST BREAK')) 14 | } 15 | var stream = writeStream(target) 16 | var hadError = false 17 | stream.on('error', function (er) { 18 | hadError = true 19 | console.log('#', er) 20 | }) 21 | stream.on('close', function () { 22 | t.is(hadError, true, 'error before close') 23 | }) 24 | stream.end() 25 | }) 26 | 27 | test('cleanup', function (t) { 28 | rimraf.sync(target) 29 | t.end() 30 | }) 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The ISC License 2 | 3 | Copyright (c) Isaac Z. Schlueter and Contributors 4 | 5 | Permission to use, copy, modify, and/or distribute this software for any 6 | purpose with or without fee is hereby granted, provided that the above 7 | copyright notice and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR 15 | IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /test/toolong.js: -------------------------------------------------------------------------------- 1 | var path = require('path') 2 | var test = require('tap').test 3 | var writeStream = require('../index.js') 4 | 5 | function repeat (times, string) { 6 | var output = '' 7 | for (var ii = 0; ii < times; ++ii) { 8 | output += string 9 | } 10 | return output 11 | } 12 | 13 | var target = path.resolve(__dirname, repeat(1000, 'test')) 14 | 15 | test('name too long', function (t) { 16 | t.plan(2) 17 | var stream = writeStream(target) 18 | var hadError = false 19 | stream.on('error', function (er) { 20 | if (!hadError) { 21 | t.is(er.code, 'ENAMETOOLONG', target.length + ' character name results in ENAMETOOLONG') 22 | hadError = true 23 | } 24 | }) 25 | stream.on('close', function () { 26 | t.ok(hadError, 'got error before close') 27 | }) 28 | stream.end() 29 | }) 30 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fs-write-stream-atomic", 3 | "version": "1.0.10", 4 | "description": "Like `fs.createWriteStream(...)`, but atomic.", 5 | "main": "index.js", 6 | "directories": { 7 | "test": "test" 8 | }, 9 | "dependencies": { 10 | "graceful-fs": "^4.1.2", 11 | "iferr": "^1.0.2", 12 | "imurmurhash": "^0.1.4", 13 | "readable-stream": "1 || 2" 14 | }, 15 | "devDependencies": { 16 | "rimraf": "^2.4.4", 17 | "standard": "^5.4.1", 18 | "tap": "^12.4.0" 19 | }, 20 | "scripts": { 21 | "test": "standard && tap --coverage test/*.js" 22 | }, 23 | "repository": { 24 | "type": "git", 25 | "url": "https://github.com/npm/fs-write-stream-atomic" 26 | }, 27 | "author": "Isaac Z. Schlueter (http://blog.izs.me/)", 28 | "license": "ISC", 29 | "bugs": { 30 | "url": "https://github.com/npm/fs-write-stream-atomic/issues" 31 | }, 32 | "homepage": "https://github.com/npm/fs-write-stream-atomic" 33 | } 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fs-write-stream-atomic 2 | 3 | Like `fs.createWriteStream(...)`, but atomic. 4 | 5 | Writes to a tmp file and does an atomic `fs.rename` to move it into 6 | place when it's done. 7 | 8 | First rule of debugging: **It's always a race condition.** 9 | 10 | ## USAGE 11 | 12 | ```javascript 13 | var fsWriteStreamAtomic = require('fs-write-stream-atomic') 14 | // options are optional. 15 | var write = fsWriteStreamAtomic('output.txt', options) 16 | var read = fs.createReadStream('input.txt') 17 | read.pipe(write) 18 | 19 | // When the write stream emits a 'finish' or 'close' event, 20 | // you can be sure that it is moved into place, and contains 21 | // all the bytes that were written to it, even if something else 22 | // was writing to `output.txt` at the same time. 23 | ``` 24 | 25 | ### `fsWriteStreamAtomic(filename, [options])` 26 | 27 | * `filename` {String} The file we want to write to 28 | * `options` {Object} 29 | * `chown` {Object} User and group to set ownership after write 30 | * `uid` {Number} 31 | * `gid` {Number} 32 | * `encoding` {String} default = 'utf8' 33 | * `mode` {Number} default = `0666` 34 | * `flags` {String} default = `'w'` 35 | 36 | -------------------------------------------------------------------------------- /test/chown.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | var fs = require('graceful-fs') 3 | var path = require('path') 4 | var test = require('tap').test 5 | var rimraf = require('rimraf') 6 | var writeStream = require('../index.js') 7 | 8 | var target = path.resolve(__dirname, 'test-chown') 9 | 10 | test('chown works', function (t) { 11 | t.plan(1) 12 | var stream = writeStream(target, {chown: {uid: process.getuid(), gid: process.getgid()}}) 13 | var hadError = false 14 | stream.on('error', function (er) { 15 | hadError = true 16 | console.log('#', er) 17 | }) 18 | stream.on('close', function () { 19 | t.is(hadError, false, 'no errors before close') 20 | }) 21 | stream.end() 22 | }) 23 | 24 | test('chown fails', function (t) { 25 | t.plan(1) 26 | fs.chown = function (file, uid, gid, cb) { 27 | cb(new Error('TEST BREAK')) 28 | } 29 | var stream = writeStream(target, {chown: {uid: process.getuid(), gid: process.getgid()}}) 30 | var hadError = false 31 | stream.on('error', function (er) { 32 | hadError = true 33 | console.log('#', er) 34 | }) 35 | stream.on('close', function () { 36 | t.is(hadError, true, 'error before close') 37 | }) 38 | stream.end() 39 | }) 40 | 41 | test('cleanup', function (t) { 42 | rimraf.sync(target) 43 | t.end() 44 | }) 45 | -------------------------------------------------------------------------------- /test/slow-close.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | var fs = require('graceful-fs') 3 | var path = require('path') 4 | var test = require('tap').test 5 | var rimraf = require('rimraf') 6 | var writeStream = require('../index.js') 7 | 8 | var target = path.resolve(__dirname, 'test-chown') 9 | 10 | test('slow close', function (t) { 11 | t.plan(2) 12 | // The goal here is to simulate the "file close" step happening so slowly 13 | // that the whole close/rename process could finish before the file is 14 | // actually closed (and thus buffers truely flushed to the OS). In 15 | // previous versions of this module, this would result in the module 16 | // emitting finish & close before the file was fully written and in 17 | // turn, could break other layers that tried to read the new file. 18 | var realEmit = fs.WriteStream.prototype.emit 19 | var reallyClosed = false 20 | fs.WriteStream.prototype.emit = function (event) { 21 | if (event !== 'close') return realEmit.apply(this, arguments) 22 | setTimeout(function () { 23 | reallyClosed = true 24 | realEmit.call(this, 'close') 25 | }.bind(this), 200) 26 | } 27 | var stream = writeStream(target) 28 | stream.on('finish', function () { 29 | t.is(reallyClosed, true, "didn't finish before target was closed") 30 | }) 31 | stream.on('close', function () { 32 | t.is(reallyClosed, true, "didn't close before target was closed") 33 | }) 34 | stream.end() 35 | }) 36 | 37 | test('cleanup', function (t) { 38 | rimraf.sync(target) 39 | t.end() 40 | }) 41 | -------------------------------------------------------------------------------- /test/basic.js: -------------------------------------------------------------------------------- 1 | var fs = require('graceful-fs') 2 | var test = require('tap').test 3 | var path = require('path') 4 | var writeStream = require('../index.js') 5 | 6 | var rename = fs.rename 7 | fs.rename = function (from, to, cb) { 8 | setTimeout(function () { 9 | rename(from, to, cb) 10 | }, 100) 11 | } 12 | 13 | test('basic', function (t) { 14 | // open 10 write streams to the same file. 15 | // then write to each of them, and to the target 16 | // and verify at the end that each of them does their thing 17 | var target = path.resolve(__dirname, 'test.txt') 18 | var n = 10 19 | 20 | // We run all of our assertions twice: 21 | // once for finish, once for close 22 | // There are 6 assertions, two fixed, plus 4 lines in the file. 23 | t.plan(n * 2 * 6) 24 | 25 | var streams = [] 26 | for (var i = 0; i < n; i++) { 27 | var s = writeStream(target) 28 | s.on('finish', verifier('finish', i)) 29 | s.on('close', verifier('close', i)) 30 | streams.push(s) 31 | } 32 | 33 | function verifier (ev, num) { 34 | return function () { 35 | if (ev === 'close') { 36 | t.equal(this.__emittedFinish, true, num + '. closed only after finish') 37 | } else { 38 | this.__emittedFinish = true 39 | t.equal(ev, 'finish', num + '. finished') 40 | } 41 | 42 | // make sure that one of the atomic streams won. 43 | var res = fs.readFileSync(target, 'utf8') 44 | var lines = res.trim().split(/\n/) 45 | lines.forEach(function (line, lineno) { 46 | var first = lines[0].match(/\d+$/)[0] 47 | var cur = line.match(/\d+$/)[0] 48 | t.equal(cur, first, num + '. line ' + lineno + ' matches') 49 | }) 50 | 51 | var resExpr = /^first write \d+\nsecond write \d+\nthird write \d+\nfinal write \d+\n$/ 52 | t.similar(res, resExpr, num + '. content matches') 53 | } 54 | } 55 | 56 | // now write something to each stream. 57 | streams.forEach(function (stream, i) { 58 | stream.write('first write ' + i + '\n') 59 | }) 60 | 61 | // wait a sec for those writes to go out. 62 | setTimeout(function () { 63 | // write something else to the target. 64 | fs.writeFileSync(target, 'brutality!\n') 65 | 66 | // write some more stuff. 67 | streams.forEach(function (stream, i) { 68 | stream.write('second write ' + i + '\n') 69 | }) 70 | 71 | setTimeout(function () { 72 | // Oops! Deleted the file! 73 | fs.unlinkSync(target) 74 | 75 | // write some more stuff. 76 | streams.forEach(function (stream, i) { 77 | stream.write('third write ' + i + '\n') 78 | }) 79 | 80 | setTimeout(function () { 81 | fs.writeFileSync(target, 'brutality TWO!\n') 82 | streams.forEach(function (stream, i) { 83 | stream.end('final write ' + i + '\n') 84 | }) 85 | }, 50) 86 | }, 50) 87 | }, 50) 88 | }) 89 | 90 | test('cleanup', function (t) { 91 | fs.readdirSync(__dirname).filter(function (f) { 92 | return f.match(/^test.txt/) 93 | }).forEach(function (file) { 94 | fs.unlinkSync(path.resolve(__dirname, file)) 95 | }) 96 | t.end() 97 | }) 98 | -------------------------------------------------------------------------------- /test/rename-eperm.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | var fs = require('graceful-fs') 3 | var path = require('path') 4 | var test = require('tap').test 5 | var rimraf = require('rimraf') 6 | var writeStream = require('../index.js') 7 | 8 | var target = path.resolve(__dirname, 'test-rename-eperm1') 9 | var target2 = path.resolve(__dirname, 'test-rename-eperm2') 10 | var target3 = path.resolve(__dirname, 'test-rename-eperm3') 11 | 12 | test('rename eperm none existing file', function (t) { 13 | t.plan(2) 14 | 15 | var _rename = fs.rename 16 | fs.existsSync = function (src) { 17 | return true 18 | } 19 | fs.rename = function (src, dest, cb) { 20 | // simulate a failure during rename where the file 21 | // is renamed successfully but the process encounters 22 | // an EPERM error and the target file does not exist 23 | _rename(src, dest, function (e) { 24 | var err = new Error('TEST BREAK') 25 | err.syscall = 'rename' 26 | err.code = 'EPERM' 27 | cb(err) 28 | }) 29 | } 30 | 31 | var stream = writeStream(target, { isWin: true }) 32 | var hadError = false 33 | var calledFinish = false 34 | stream.on('error', function (er) { 35 | hadError = true 36 | console.log('#', er) 37 | }) 38 | stream.on('finish', function () { 39 | calledFinish = true 40 | }) 41 | stream.on('close', function () { 42 | t.is(hadError, true, 'error was caught') 43 | t.is(calledFinish, false, 'finish was called before close') 44 | }) 45 | stream.end() 46 | }) 47 | 48 | // test existing file with diff. content 49 | test('rename eperm existing file different content', function (t) { 50 | t.plan(2) 51 | 52 | var _rename = fs.rename 53 | fs.existsSync = function (src) { 54 | return true 55 | } 56 | fs.rename = function (src, dest, cb) { 57 | // simulate a failure during rename where the file 58 | // is renamed successfully but the process encounters 59 | // an EPERM error and the target file that has another content than the 60 | // destination 61 | _rename(src, dest, function (e) { 62 | fs.writeFile(src, 'dest', function (writeErr) { 63 | if (writeErr) { 64 | return console.log('WRITEERR: ' + writeErr) 65 | } 66 | 67 | fs.writeFile(target2, 'target', function (writeErr) { 68 | if (writeErr) { 69 | return console.log('WRITEERR: ' + writeErr) 70 | } 71 | 72 | var err = new Error('TEST BREAK') 73 | err.syscall = 'rename' 74 | err.code = 'EPERM' 75 | cb(err) 76 | }) 77 | }) 78 | }) 79 | } 80 | 81 | var stream = writeStream(target2, { isWin: true }) 82 | var hadError = false 83 | var calledFinish = false 84 | stream.on('error', function (er) { 85 | hadError = true 86 | console.log('#', er) 87 | }) 88 | stream.on('finish', function () { 89 | calledFinish = true 90 | }) 91 | stream.on('close', function () { 92 | t.is(hadError, true, 'error was caught') 93 | t.is(calledFinish, false, 'finish was called before close') 94 | }) 95 | stream.end() 96 | }) 97 | 98 | // test existing file with the same content 99 | // test existing file with diff. content 100 | test('rename eperm existing file different content', function (t) { 101 | t.plan(2) 102 | 103 | var _rename = fs.rename 104 | fs.existsSync = function (src) { 105 | return true 106 | } 107 | fs.rename = function (src, dest, cb) { 108 | // simulate a failure during rename where the file 109 | // is renamed successfully but the process encounters 110 | // an EPERM error and the target file that has the same content than the 111 | // destination 112 | _rename(src, dest, function (e) { 113 | fs.writeFile(src, 'target2', function (writeErr) { 114 | if (writeErr) { 115 | return console.log('WRITEERR: ' + writeErr) 116 | } 117 | 118 | fs.writeFile(target3, 'target2', function (writeErr) { 119 | if (writeErr) { 120 | return console.log('WRITEERR: ' + writeErr) 121 | } 122 | 123 | var err = new Error('TEST BREAK') 124 | err.syscall = 'rename' 125 | err.code = 'EPERM' 126 | cb(err) 127 | }) 128 | }) 129 | }) 130 | } 131 | 132 | var stream = writeStream(target3, { isWin: true }) 133 | var hadError = false 134 | var calledFinish = false 135 | stream.on('error', function (er) { 136 | hadError = true 137 | console.log('#', er) 138 | }) 139 | stream.on('finish', function () { 140 | calledFinish = true 141 | }) 142 | stream.on('close', function () { 143 | t.is(hadError, false, 'error was caught') 144 | t.is(calledFinish, true, 'finish was called before close') 145 | }) 146 | stream.end() 147 | }) 148 | 149 | test('cleanup', function (t) { 150 | rimraf.sync(target) 151 | rimraf.sync(target2) 152 | rimraf.sync(target3) 153 | t.end() 154 | }) 155 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var fs = require('graceful-fs') 2 | var Writable = require('readable-stream').Writable 3 | var util = require('util') 4 | var MurmurHash3 = require('imurmurhash') 5 | var iferr = require('iferr') 6 | var crypto = require('crypto') 7 | 8 | function murmurhex () { 9 | var hash = MurmurHash3('') 10 | for (var ii = 0; ii < arguments.length; ++ii) { 11 | hash.hash('' + arguments[ii]) 12 | } 13 | return hash.result() 14 | } 15 | 16 | var invocations = 0 17 | function getTmpname (filename) { 18 | return filename + '.' + murmurhex(__filename, process.pid, ++invocations) 19 | } 20 | 21 | var setImmediate = global.setImmediate || setTimeout 22 | 23 | module.exports = WriteStreamAtomic 24 | 25 | // Requirements: 26 | // 1. Write everything written to the stream to a temp file. 27 | // 2. If there are no errors: 28 | // a. moves the temp file into its final destination 29 | // b. emits `finish` & `closed` ONLY after the file is 30 | // fully flushed and renamed. 31 | // 3. If there's an error, removes the temp file. 32 | 33 | util.inherits(WriteStreamAtomic, Writable) 34 | function WriteStreamAtomic (path, options) { 35 | if (!(this instanceof WriteStreamAtomic)) { 36 | return new WriteStreamAtomic(path, options) 37 | } 38 | Writable.call(this, options) 39 | 40 | this.__isWin = options && options.hasOwnProperty('isWin') ? options.isWin : process.platform === 'win32' 41 | 42 | this.__atomicTarget = path 43 | this.__atomicTmp = getTmpname(path) 44 | 45 | this.__atomicChown = options && options.chown 46 | 47 | this.__atomicClosed = false 48 | 49 | this.__atomicStream = fs.WriteStream(this.__atomicTmp, options) 50 | 51 | this.__atomicStream.once('open', handleOpen(this)) 52 | this.__atomicStream.once('close', handleClose(this)) 53 | this.__atomicStream.once('error', handleError(this)) 54 | } 55 | 56 | // We have to suppress default finish emitting, because ordinarily it 57 | // would happen as soon as `end` is called on us and all of the 58 | // data has been written to our target stream. So we suppress 59 | // finish from being emitted here, and only emit it after our 60 | // target stream is closed and we've moved everything around. 61 | WriteStreamAtomic.prototype.emit = function (event) { 62 | if (event === 'finish') return this.__atomicStream.end() 63 | return Writable.prototype.emit.apply(this, arguments) 64 | } 65 | 66 | WriteStreamAtomic.prototype._write = function (buffer, encoding, cb) { 67 | var flushed = this.__atomicStream.write(buffer, encoding) 68 | if (flushed) return cb() 69 | this.__atomicStream.once('drain', cb) 70 | } 71 | 72 | function handleOpen (writeStream) { 73 | return function (fd) { 74 | writeStream.emit('open', fd) 75 | } 76 | } 77 | 78 | function handleClose (writeStream) { 79 | return function () { 80 | if (writeStream.__atomicClosed) return 81 | writeStream.__atomicClosed = true 82 | if (writeStream.__atomicChown) { 83 | var uid = writeStream.__atomicChown.uid 84 | var gid = writeStream.__atomicChown.gid 85 | return fs.chown(writeStream.__atomicTmp, uid, gid, iferr(cleanup, moveIntoPlace)) 86 | } else { 87 | moveIntoPlace() 88 | } 89 | } 90 | 91 | function moveIntoPlace () { 92 | fs.rename(writeStream.__atomicTmp, writeStream.__atomicTarget, iferr(trapWindowsEPERM, end)) 93 | } 94 | 95 | function trapWindowsEPERM (err) { 96 | if (writeStream.__isWin && 97 | err.syscall && err.syscall === 'rename' && 98 | err.code && err.code === 'EPERM' 99 | ) { 100 | checkFileHashes(err) 101 | } else { 102 | cleanup(err) 103 | } 104 | } 105 | 106 | function checkFileHashes (eperm) { 107 | var inprocess = 2 108 | var tmpFileHash = crypto.createHash('sha512') 109 | var targetFileHash = crypto.createHash('sha512') 110 | 111 | fs.createReadStream(writeStream.__atomicTmp) 112 | .on('data', function (data, enc) { tmpFileHash.update(data, enc) }) 113 | .on('error', fileHashError) 114 | .on('end', fileHashComplete) 115 | fs.createReadStream(writeStream.__atomicTarget) 116 | .on('data', function (data, enc) { targetFileHash.update(data, enc) }) 117 | .on('error', fileHashError) 118 | .on('end', fileHashComplete) 119 | 120 | function fileHashError () { 121 | if (inprocess === 0) return 122 | inprocess = 0 123 | cleanup(eperm) 124 | } 125 | 126 | function fileHashComplete () { 127 | if (inprocess === 0) return 128 | if (--inprocess) return 129 | if (tmpFileHash.digest('hex') === targetFileHash.digest('hex')) { 130 | return cleanup() 131 | } else { 132 | return cleanup(eperm) 133 | } 134 | } 135 | } 136 | 137 | function cleanup (err) { 138 | fs.unlink(writeStream.__atomicTmp, function () { 139 | if (err) { 140 | writeStream.emit('error', err) 141 | writeStream.emit('close') 142 | } else { 143 | end() 144 | } 145 | }) 146 | } 147 | 148 | function end () { 149 | // We have to use our parent class directly because we suppress `finish` 150 | // events fired via our own emit method. 151 | Writable.prototype.emit.call(writeStream, 'finish') 152 | 153 | // Delay the close to provide the same temporal separation a physical 154 | // file operation would have– that is, the close event is emitted only 155 | // after the async close operation completes. 156 | setImmediate(function () { 157 | writeStream.emit('close') 158 | }) 159 | } 160 | } 161 | 162 | function handleError (writeStream) { 163 | return function (er) { 164 | cleanupSync() 165 | writeStream.emit('error', er) 166 | writeStream.__atomicClosed = true 167 | writeStream.emit('close') 168 | } 169 | function cleanupSync () { 170 | try { 171 | fs.unlinkSync(writeStream.__atomicTmp) 172 | } finally { 173 | return 174 | } 175 | } 176 | } 177 | --------------------------------------------------------------------------------