├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── bench.js ├── index.js ├── package.json └── test ├── bare.test.js └── graceful.test.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | 63 | .*DS_Store 64 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | 3 | node_js: 4 | - "11" 5 | - "10" 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Matteo Collina 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fast-write-atomic 2 | 3 | [![Build 4 | Status](https://travis-ci.com/mcollina/fast-write-atomic.svg?branch=master)](https://travis-ci.com/mcollina/fast-write-atomic) 5 | 6 | Fast way to write a file atomically, for Node.js 7 | 8 | Status: *experimental* 9 | 10 | ## Install 11 | 12 | ``` 13 | npm i fast-write-atomic 14 | ``` 15 | 16 | ## Example 17 | 18 | ```js 19 | const writeFile = require('fast-write-atomic') 20 | 21 | const data = Buffer.from('hello world') 22 | 23 | writeFile('./hello', data, function (err) { 24 | if (err) { 25 | console.log(err) 26 | return 27 | } 28 | 29 | console.log('file written') 30 | }) 31 | ``` 32 | 33 | ## Benchmarks 34 | 35 | Those benchmarks writes a 1 MB file a thousand times: 36 | 37 | ``` 38 | benchWriteFileAtomic*1000: 9830.501ms 39 | benchFastWriteAtomic*1000: 8848.916ms 40 | benchWriteFileAtomic*1000: 9944.722ms 41 | benchFastWriteAtomic*1000: 8997.108ms 42 | ``` 43 | 44 | ## License 45 | 46 | MIT 47 | 48 | -------------------------------------------------------------------------------- /bench.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const bench = require('fastbench') 4 | const { tmpdir } = require('os') 5 | const { join } = require('path') 6 | const writeFileAtomic = require('write-file-atomic') 7 | const fastWriteAtomic = require('.') 8 | 9 | const dest = join(tmpdir(), 'dest') 10 | const file = Buffer.allocUnsafe(1024 * 1024) // 1MB 11 | 12 | const run = bench([ 13 | function benchWriteFileAtomic (cb) { 14 | writeFileAtomic(dest, file, cb) 15 | }, 16 | function benchFastWriteAtomic (cb) { 17 | fastWriteAtomic(dest, file, cb) 18 | } 19 | ], 1000) 20 | 21 | run(run) 22 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { open, write, close, rename, fsync, unlink } = require('fs') 4 | const { join, dirname } = require('path') 5 | 6 | var counter = 0 7 | 8 | function cleanup (dest, err, cb) { 9 | unlink(dest, function () { 10 | cb(err) 11 | }) 12 | } 13 | 14 | function closeAndCleanup (fd, dest, err, cb) { 15 | close(fd, cleanup.bind(null, dest, err, cb)) 16 | } 17 | 18 | function writeLoop (fd, content, contentLength, offset, cb) { 19 | write(fd, content, offset, function (err, bytesWritten) { 20 | if (err) { 21 | cb(err) 22 | return 23 | } 24 | 25 | return (bytesWritten < contentLength - offset) 26 | ? writeLoop(fd, content, contentLength, offset + bytesWritten, cb) 27 | : cb(null) 28 | }) 29 | } 30 | 31 | function openLoop (dest, cb) { 32 | open(dest, 'w', function (err, fd) { 33 | if (err) { 34 | return (err.code === 'EMFILE') 35 | ? openLoop(dest, cb) 36 | : cb(err) 37 | } 38 | 39 | cb(null, fd) 40 | }) 41 | } 42 | 43 | function writeAtomic (path, content, cb) { 44 | const tmp = join(dirname(path), '.' + process.pid + '.' + counter++) 45 | openLoop(tmp, function (err, fd) { 46 | if (err) { 47 | cb(err) 48 | return 49 | } 50 | 51 | const contentLength = Buffer.byteLength(content) 52 | writeLoop(fd, content, contentLength, 0, function (err) { 53 | if (err) { 54 | closeAndCleanup(fd, tmp, err, cb) 55 | return 56 | } 57 | 58 | fsync(fd, function (err) { 59 | if (err) { 60 | closeAndCleanup(fd, tmp, err, cb) 61 | return 62 | } 63 | 64 | close(fd, function (err) { 65 | if (err) { 66 | // TODO could we possibly be leaking a file descriptor here? 67 | cleanup(tmp, err, cb) 68 | return 69 | } 70 | 71 | rename(tmp, path, (err) => { 72 | if (err) { 73 | cleanup(tmp, err, cb) 74 | return 75 | } 76 | 77 | cb(null) 78 | }) 79 | }) 80 | }) 81 | }) 82 | 83 | // clean up after oursevles, this is not needed 84 | // anymore 85 | content = null 86 | }) 87 | } 88 | 89 | module.exports = writeAtomic 90 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fast-write-atomic", 3 | "version": "0.2.1", 4 | "description": "Fast way to write a file atomically, for Node.js", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "standard | snazzy && tap test/*test.js", 8 | "cov": "tap --100 test/*test.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/mcollina/fast-write-atomic.git" 13 | }, 14 | "keywords": [ 15 | "file", 16 | "write", 17 | "atomic" 18 | ], 19 | "author": "Matteo Collina ", 20 | "license": "MIT", 21 | "bugs": { 22 | "url": "https://github.com/mcollina/fast-write-atomic/issues" 23 | }, 24 | "homepage": "https://github.com/mcollina/fast-write-atomic#readme", 25 | "devDependencies": { 26 | "fastbench": "^1.0.1", 27 | "graceful-fs": "^4.1.15", 28 | "husky": "^1.3.0", 29 | "proxyquire": "^2.1.0", 30 | "snazzy": "^8.0.0", 31 | "standard": "^12.0.1", 32 | "tap": "^12.1.1", 33 | "write-file-atomic": "^2.3.0" 34 | }, 35 | "dependencies": {}, 36 | "husky": { 37 | "hooks": { 38 | "pre-commit": "npm test" 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /test/bare.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const writeAtomic = require('..') 4 | const proxyquire = require('proxyquire') 5 | const { test, tearDown } = require('tap') 6 | const { 7 | readFile, 8 | unlink, 9 | unlinkSync, 10 | open, 11 | write, 12 | close, 13 | fsync, 14 | rename 15 | } = require('fs') 16 | const { tmpdir } = require('os') 17 | const { join } = require('path') 18 | 19 | const files = [] 20 | 21 | tearDown(() => { 22 | for (let dest of files) { 23 | try { 24 | unlinkSync(dest) 25 | } catch (_) { 26 | } 27 | } 28 | }) 29 | 30 | let nextId = 0 31 | 32 | function getDest (name) { 33 | if (!name) { 34 | name = 'hello' + nextId++ 35 | } 36 | const dest = join(tmpdir(), name) 37 | files.push(dest) 38 | return dest 39 | } 40 | 41 | test('write a file', (t) => { 42 | t.plan(3) 43 | 44 | const dest = getDest() 45 | const content = Buffer.allocUnsafe(4096) // 4 KB 46 | 47 | writeAtomic(dest, content, (err) => { 48 | t.error(err) 49 | readFile(dest, (err, data) => { 50 | t.error(err) 51 | t.equal(Buffer.compare(data, content), 0) 52 | }) 53 | }) 54 | }) 55 | 56 | test('parallel writes', (t) => { 57 | t.plan(4) 58 | 59 | const dest = getDest() 60 | const content1 = Buffer.allocUnsafe(4096).fill('AB') // 4 KB 61 | const content2 = Buffer.allocUnsafe(4096).fill('CD') // 4 KB 62 | 63 | let countdown = 2 64 | 65 | writeAtomic(dest, content1, (err) => { 66 | t.error(err) 67 | done() 68 | }) 69 | 70 | writeAtomic(dest, content2, (err) => { 71 | t.error(err) 72 | done() 73 | }) 74 | 75 | function done () { 76 | if (--countdown !== 0) { 77 | return 78 | } 79 | 80 | readFile(dest, (err, data) => { 81 | t.error(err) 82 | // we expect either content1 or content2 to be there 83 | t.equal(Buffer.compare(data, content2) === 0 || Buffer.compare(data, content1) === 0, true) 84 | }) 85 | } 86 | }) 87 | 88 | test('calls fsync', (t) => { 89 | t.plan(5) 90 | 91 | const writeAtomic = proxyquire('..', { 92 | fs: { 93 | open, 94 | write, 95 | close, 96 | fsync (fd, cb) { 97 | t.pass('fsync called') 98 | return fsync(fd, cb) 99 | }, 100 | rename (source, dest, cb) { 101 | t.pass('rename called') 102 | return rename(source, dest, cb) 103 | } 104 | } 105 | }) 106 | 107 | const dest = getDest() 108 | const content = Buffer.allocUnsafe(4096) // 4 KB 109 | 110 | writeAtomic(dest, content, (err) => { 111 | t.error(err) 112 | readFile(dest, (err, data) => { 113 | t.error(err) 114 | t.equal(Buffer.compare(data, content), 0) 115 | }) 116 | }) 117 | }) 118 | 119 | test('unlinks if it errors during rename', (t) => { 120 | t.plan(4) 121 | 122 | let _source 123 | const writeAtomic = proxyquire('..', { 124 | fs: { 125 | open, 126 | write, 127 | close, 128 | unlink (file, cb) { 129 | t.equal(file, _source) 130 | return unlink(file, cb) 131 | }, 132 | rename (source, dest, cb) { 133 | _source = source 134 | process.nextTick(cb, new Error('kaboom')) 135 | } 136 | } 137 | }) 138 | 139 | const dest = getDest() 140 | const content = Buffer.allocUnsafe(4096) // 4 KB 141 | 142 | writeAtomic(dest, content, (err) => { 143 | t.equal(err.message, 'kaboom') 144 | readFile(dest, (err) => { 145 | t.equal(err.code, 'ENOENT') 146 | }) 147 | readFile(_source, (err) => { 148 | t.equal(err.code, 'ENOENT') 149 | }) 150 | }) 151 | }) 152 | 153 | test('unlinks if it errors during write', (t) => { 154 | t.plan(4) 155 | 156 | let _source 157 | const writeAtomic = proxyquire('..', { 158 | fs: { 159 | open (dest, flags, cb) { 160 | _source = dest 161 | return open(dest, flags, cb) 162 | }, 163 | write (fd, content, offset, cb) { 164 | process.nextTick(cb, new Error('kaboom')) 165 | }, 166 | close, 167 | unlink (file, cb) { 168 | t.equal(file, _source) 169 | return unlink(file, cb) 170 | } 171 | } 172 | }) 173 | 174 | const dest = getDest() 175 | const content = Buffer.allocUnsafe(4096) // 4 KB 176 | 177 | writeAtomic(dest, content, (err) => { 178 | t.equal(err.message, 'kaboom') 179 | readFile(dest, (err) => { 180 | t.equal(err.code, 'ENOENT') 181 | }) 182 | readFile(_source, (err) => { 183 | t.equal(err.code, 'ENOENT') 184 | }) 185 | }) 186 | }) 187 | 188 | test('unlinks if it errors during fsync', (t) => { 189 | t.plan(4) 190 | 191 | let _source 192 | const writeAtomic = proxyquire('..', { 193 | fs: { 194 | open (dest, flags, cb) { 195 | _source = dest 196 | return open(dest, flags, cb) 197 | }, 198 | write, 199 | close, 200 | fsync (fd, cb) { 201 | process.nextTick(cb, new Error('kaboom')) 202 | }, 203 | unlink (file, cb) { 204 | t.equal(file, _source) 205 | return unlink(file, cb) 206 | } 207 | } 208 | }) 209 | 210 | const dest = getDest() 211 | const content = Buffer.allocUnsafe(4096) // 4 KB 212 | 213 | writeAtomic(dest, content, (err) => { 214 | t.equal(err.message, 'kaboom') 215 | readFile(dest, (err) => { 216 | t.equal(err.code, 'ENOENT') 217 | }) 218 | readFile(_source, (err) => { 219 | t.equal(err.code, 'ENOENT') 220 | }) 221 | }) 222 | }) 223 | 224 | test('retries if the write was not completed', (t) => { 225 | t.plan(5) 226 | 227 | let first = true 228 | const writeAtomic = proxyquire('..', { 229 | fs: { 230 | open, 231 | write (fd, content, offset, cb) { 232 | t.pass('fs.write') 233 | if (first) { 234 | first = false 235 | write(fd, content, 0, 16, cb) 236 | return 237 | } 238 | 239 | write(fd, content, offset, cb) 240 | }, 241 | close, 242 | unlink 243 | } 244 | }) 245 | 246 | const dest = getDest() 247 | const content = Buffer.allocUnsafe(4096) // 4 KB 248 | 249 | writeAtomic(dest, content, (err) => { 250 | t.error(err) 251 | readFile(dest, (err, data) => { 252 | t.error(err) 253 | t.equal(Buffer.compare(data, content), 0) 254 | }) 255 | }) 256 | }) 257 | 258 | test('errors if open errors', (t) => { 259 | t.plan(3) 260 | 261 | let _source 262 | const writeAtomic = proxyquire('..', { 263 | fs: { 264 | open (dest, flags, cb) { 265 | _source = dest 266 | process.nextTick(cb, new Error('kaboom')) 267 | } 268 | } 269 | }) 270 | 271 | const dest = getDest() 272 | const content = Buffer.allocUnsafe(4096) // 4 KB 273 | 274 | writeAtomic(dest, content, (err) => { 275 | t.equal(err.message, 'kaboom') 276 | readFile(dest, (err) => { 277 | t.equal(err.code, 'ENOENT') 278 | }) 279 | readFile(_source, (err) => { 280 | t.equal(err.code, 'ENOENT') 281 | }) 282 | }) 283 | }) 284 | 285 | test('retries on EMFILE', (t) => { 286 | t.plan(3) 287 | 288 | let first = true 289 | const writeAtomic = proxyquire('..', { 290 | fs: { 291 | open (dest, flags, cb) { 292 | if (first) { 293 | first = false 294 | const err = new Error('kaboom') 295 | err.code = 'EMFILE' 296 | process.nextTick(cb, err) 297 | return 298 | } 299 | 300 | return open(dest, flags, cb) 301 | }, 302 | write, 303 | close, 304 | unlink 305 | } 306 | }) 307 | 308 | const dest = getDest() 309 | const content = Buffer.allocUnsafe(4096) // 4 KB 310 | 311 | writeAtomic(dest, content, (err) => { 312 | t.error(err) 313 | readFile(dest, (err, data) => { 314 | t.error(err) 315 | t.equal(Buffer.compare(data, content), 0) 316 | }) 317 | }) 318 | }) 319 | 320 | test('write 2000 files in parallel', (t) => { 321 | const MAX = 2000 322 | let total = 0 323 | t.plan(1) 324 | 325 | for (var i = 0; i < MAX; i++) { 326 | const dest = getDest() 327 | const content = Buffer.allocUnsafe(4096) // 4 KB 328 | 329 | writeAtomic(dest, content, (err) => { 330 | if (err) { 331 | t.error(err) 332 | return 333 | } 334 | 335 | if (++total === MAX) { 336 | t.pass(`${total} writes completed`) 337 | } 338 | }) 339 | } 340 | }) 341 | 342 | test('multibyte unicode symbols', (t) => { 343 | t.plan(1) 344 | const dest = getDest() 345 | const content = '{"name":"tajné jménoed25519","id":"QmSvTNE2Eo7SxRXjmEaZnE91cNpduKjYFBtd2LYC4Rsoeq"}' 346 | writeAtomic(dest, content, (err) => { 347 | t.error(err) 348 | }) 349 | }) 350 | -------------------------------------------------------------------------------- /test/graceful.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | // Make sure to read the caveat below. 4 | const realFs = require('fs') 5 | const gracefulFs = require('graceful-fs') 6 | gracefulFs.gracefulify(realFs) 7 | 8 | require('./bare.test.js') 9 | --------------------------------------------------------------------------------